From f4fdd622d3aeafb4910bfbf64cd1a099bdf0c724 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 12 Nov 2026 01:19:07 +0700 Subject: [PATCH 1/2] fix(linux-mongodb): keep NULL apart from a missing field, export number fields as numbers and write mongosh constructors --- core/crates/core/src/dialect/sql_dialect.rs | 5 + core/crates/core/src/dialect/test_dialect.rs | 13 + .../core/src/document/document_filter.rs | 18 +- .../core/src/document/document_write.rs | 34 +-- core/crates/core/src/document/json_cells.rs | 158 ++++++++++- core/crates/core/src/document/mod.rs | 5 +- .../crates/core/src/document/shell_literal.rs | 255 ++++++++++++++++-- core/crates/core/src/document/tests.rs | 149 ++++++---- core/crates/core/src/document/value_order.rs | 2 +- core/crates/core/src/edit/mod.rs | 2 +- core/crates/core/src/edit/new_draft_cell.rs | 49 ++++ core/crates/core/src/export/csv.rs | 19 ++ .../core/src/export/encoder/json_object.rs | 18 +- .../core/src/export/encoder/tsv_encoder.rs | 3 +- core/crates/core/src/export/html.rs | 16 +- core/crates/core/src/export/json.rs | 32 ++- core/crates/core/src/export/markdown.rs | 18 +- core/crates/core/src/export/mod.rs | 24 ++ core/crates/core/src/export/sql.rs | 40 +-- core/crates/core/src/export/text_table.rs | 18 ++ core/crates/core/src/export/tsv.rs | 10 + core/crates/core/src/export/xlsx.rs | 14 + core/crates/core/src/export/xml.rs | 18 +- core/crates/core/src/import/import_plan.rs | 44 ++- .../core/src/sql_syntax/mongo/shell_access.rs | 120 ++++++++- .../src/sql_syntax/mongo/shell_helpers.rs | 4 + .../core/src/sql_syntax/mongo/shell_value.rs | 25 +- core/crates/core/tests/script_corpus.rs | 17 ++ core/crates/drivers/mongodb/src/call.rs | 26 ++ core/crates/drivers/mongodb/src/catalog.rs | 79 ++++-- core/crates/drivers/mongodb/src/connection.rs | 20 +- core/crates/drivers/mongodb/src/dialect.rs | 158 +++++++++-- .../crates/drivers/mongodb/src/field_types.rs | 54 +++- core/crates/drivers/mongodb/src/rows.rs | 33 ++- .../drivers/mongodb/src/shell_command.rs | 216 ++++++++++++++- core/crates/drivers/mongodb/src/shell_find.rs | 3 +- .../crates/drivers/mongodb/src/stream_read.rs | 106 +++++--- core/crates/drivers/mongodb/src/to_bson.rs | 10 +- .../mongodb/tests/integration/export.rs | 148 +++++++++- .../mongodb/tests/integration/replica_set.rs | 2 +- .../mongodb/tests/integration/seed_list.rs | 2 +- .../mongodb/tests/integration/statements.rs | 42 +++ .../crates/app/src/ui/data_grid/edit/paste.rs | 14 +- .../app/src/ui/data_grid/edit/row_edits.rs | 28 +- linux/crates/app/src/ui/data_grid/mod.rs | 8 +- .../app/src/ui/data_grid/tracked_values.rs | 20 +- linux/docs/adding-drivers.md | 4 +- linux/docs/drivers/mongodb.md | 12 +- linux/docs/testing.md | 2 + 49 files changed, 1780 insertions(+), 337 deletions(-) diff --git a/core/crates/core/src/dialect/sql_dialect.rs b/core/crates/core/src/dialect/sql_dialect.rs index eebf82a2b1..a9f0737888 100644 --- a/core/crates/core/src/dialect/sql_dialect.rs +++ b/core/crates/core/src/dialect/sql_dialect.rs @@ -285,4 +285,9 @@ pub trait SqlDialect: Send + Sync + std::fmt::Debug { fn update_sets_default(&self) -> bool { false } + + /// Whether a read gives NULL for a value the row does not hold, as a document's missing field. + fn reads_missing_as_null(&self) -> bool { + false + } } diff --git a/core/crates/core/src/dialect/test_dialect.rs b/core/crates/core/src/dialect/test_dialect.rs index 5b35a44ed3..24f2602359 100644 --- a/core/crates/core/src/dialect/test_dialect.rs +++ b/core/crates/core/src/dialect/test_dialect.rs @@ -33,6 +33,7 @@ pub struct TestDialect { insert_limits: InsertBatchLimits, insert_conflict: InsertConflict, update_default: bool, + missing_reads_null: bool, binds: Mutex>, } @@ -74,6 +75,7 @@ impl Default for TestDialect { insert_limits: InsertBatchLimits::DEFAULT, insert_conflict: InsertConflict::Refused, update_default: true, + missing_reads_null: false, binds: Mutex::new(Vec::new()), } } @@ -139,6 +141,13 @@ impl TestDialect { } } + pub fn reading_missing_as_null(self) -> Self { + Self { + missing_reads_null: true, + ..self + } + } + pub fn targets(&self) -> Vec { self.recorded().into_iter().map(|bind| bind.target).collect() } @@ -192,6 +201,10 @@ impl SqlDialect for TestDialect { self.update_default } + fn reads_missing_as_null(&self) -> bool { + self.missing_reads_null + } + fn quote_identifier(&self, name: &str) -> String { format!("\"{}\"", name.replace('"', "\"\"")) } diff --git a/core/crates/core/src/document/document_filter.rs b/core/crates/core/src/document/document_filter.rs index e7fcaa66a5..d7bc70043c 100644 --- a/core/crates/core/src/document/document_filter.rs +++ b/core/crates/core/src/document/document_filter.rs @@ -26,14 +26,30 @@ const JSON_OPERATORS: &[FilterOp] = &[ FilterOp::IsNotNull, ]; +/// A pattern is a `$regex`, which never matches a number. +const NUMBER_OPERATORS: &[FilterOp] = &[ + FilterOp::Eq, + FilterOp::NotEq, + FilterOp::Lt, + FilterOp::LtEq, + FilterOp::Gt, + FilterOp::GtEq, + FilterOp::Between, + FilterOp::In, + FilterOp::NotIn, + FilterOp::IsNull, + FilterOp::IsNotNull, +]; + /// The operators a filter rule takes on a field, in the order the filter /// strip lists them. A JSON column compares as well as matching text: a /// value typed for it binds as the BSON its Extended JSON names, and the /// server compares it with the field's values of that type, numbers of /// every type with each other, so a field of ints and doubles keeps `<` -/// and Between. +/// and Between. A field of numbers only compares. pub fn filter_operators(column: &ColumnInfo) -> &'static [FilterOp] { match column.column_type.kind { + ColumnKind::Json { .. } if super::json_cells::holds_numbers(&column.column_type) => NUMBER_OPERATORS, ColumnKind::Json { .. } => JSON_OPERATORS, _ => operators_for(&column.column_type.kind), } diff --git a/core/crates/core/src/document/document_write.rs b/core/crates/core/src/document/document_write.rs index db02eeb9c0..cda3825fee 100644 --- a/core/crates/core/src/document/document_write.rs +++ b/core/crates/core/src/document/document_write.rs @@ -11,9 +11,7 @@ use super::document_read::collection; use super::shell_params::ShellParams; /// A new document of the cells set. A cell left to the server is left -/// out, which for `_id` lets the server make one. A NULL cell is left out -/// too: a page reads a missing field as NULL, and an explicit `null` is a -/// field that a sparse or partial index counts. +/// out, which for `_id` lets the server make one. pub(crate) fn insert( table: &TableRef, columns: &[ColumnInfo], @@ -28,7 +26,7 @@ pub(crate) fn insert( let mut params = ShellParams::new(); let mut fields = Vec::new(); for (column, cell) in columns.iter().zip(cells) { - if let Some(value) = cell.value().filter(|value| !value.is_null()) { + if let Some(value) = cell.value() { fields.push((column.name.clone(), params.bind(value.clone(), Some(column))?)); } } @@ -38,8 +36,7 @@ pub(crate) fn insert( Ok(params.statement(&call)) } -/// Several new documents, all of the same columns, each leaving out its -/// NULL cells as `insert` does. +/// Several new documents, all of the same columns. pub(crate) fn insert_many( table: &TableRef, columns: &[&ColumnInfo], @@ -59,9 +56,7 @@ pub(crate) fn insert_many( } let mut fields = Vec::with_capacity(columns.len()); for (&column, value) in columns.iter().zip(row) { - if !value.is_null() { - fields.push((column.name.clone(), params.bind(value, Some(column))?)); - } + fields.push((column.name.clone(), params.bind(value, Some(column))?)); } documents.push(ShellValue::Object(fields)); } @@ -72,9 +67,8 @@ pub(crate) fn insert_many( } /// The fields changed on one document, with `$set`. Only the fields named -/// change; one the document lacked is added. A field set to NULL is -/// removed with `$unset`, as a new document leaves it out, so a sparse or -/// partial index stops counting it. A key keeps `$set`. +/// change; one the document lacked is added. A field given back to the +/// server is removed with `$unset`, which a key cannot be. pub(crate) fn update( table: &TableRef, columns: &[ColumnInfo], @@ -98,16 +92,16 @@ pub(crate) fn update( name: column.name.clone(), }); } - let Some(value) = input.value() else { - return Err(BuildSqlError::ServerComputedValue { - name: column.name.clone(), - }); - }; - match value { - Value::Null if !names.contains(&column.name) => { + match input { + CellInput::Value(value) => set.push((column.name.clone(), params.bind(value.clone(), Some(column))?)), + CellInput::Default if !names.contains(&column.name) => { unset.push((column.name.clone(), ShellValue::string(""))); } - _ => set.push((column.name.clone(), params.bind(value.clone(), Some(column))?)), + CellInput::Default | CellInput::Expression(_) => { + return Err(BuildSqlError::ServerComputedValue { + name: column.name.clone(), + }); + } } } let filter = key_filter(columns, identity, key, &mut params)?; diff --git a/core/crates/core/src/document/json_cells.rs b/core/crates/core/src/document/json_cells.rs index f78e934cb8..a582432936 100644 --- a/core/crates/core/src/document/json_cells.rs +++ b/core/crates/core/src/document/json_cells.rs @@ -1,6 +1,13 @@ -use crate::column::{ColumnInfo, ColumnKind}; +use std::borrow::Cow; + +use crate::column::{ColumnInfo, ColumnKind, ColumnType}; use crate::query_result::QueryResult; -use crate::value::{JsonText, Temporal, Value}; +use crate::sql_syntax::mongo::{ShellValue, parse_value}; +use crate::value::{JsonText, SqlDecimal, Temporal, Value}; + +/// Longer than any numeric wrapper: a longer cell is a document, read +/// without parsing it. +const WRAPPER_LIMIT: usize = 128; /// A page's cells in a column the grid edits as JSON, written as /// extended JSON. A field whose documents hold values of several types is @@ -27,6 +34,63 @@ pub(crate) fn json_cells(mut result: QueryResult, columns: &[ColumnInfo]) -> Que result } +/// A cell of a `NUMBER_TYPE` column that holds one number, as the +/// number: a bare JSON number, or one Extended JSON wrapper of its digits, +/// `{"$numberLong": "7"}`. A row export writes it as a number column's +/// cell, which a sheet sums. Any other value, and any other column's, +/// comes back as it is. +pub(crate) fn json_number<'a>(value: &'a Value, column: &ColumnType) -> Cow<'a, Value> { + let number = match value { + Value::Json(json) if holds_numbers(column) => numeric_json(json.as_str().trim()), + _ => None, + }; + number.map_or(Cow::Borrowed(value), Cow::Owned) +} + +/// Whether the column is the JSON column of a field of numbers. +pub(crate) fn holds_numbers(column: &ColumnType) -> bool { + column.name == super::NUMBER_TYPE && matches!(column.kind, ColumnKind::Json { .. }) +} + +fn numeric_json(text: &str) -> Option { + match text.as_bytes().first()? { + b'-' | b'0'..=b'9' => plain_number(text), + b'{' if text.len() <= WRAPPER_LIMIT => wrapped_number(text), + _ => None, + } +} + +/// A number's digits as they read. A double, which relaxed Extended JSON +/// writes in its shortest round-trip form, stays a double, so a sheet keeps +/// it a number; other digits stay exact as a decimal. +fn plain_number(text: &str) -> Option { + if let Ok(integer) = text.parse() { + return Some(Value::Int(integer)); + } + let double = text.parse::().ok().filter(|number| number.is_finite()); + match double { + Some(number) if text.contains(['e', 'E']) || format!("{number:?}") == text => Some(Value::Float64(number)), + _ => text.parse().ok().map(Value::Decimal), + } +} + +fn wrapped_number(text: &str) -> Option { + let value = parse_value(text).ok()?; + let [(key, ShellValue::String(digits))] = value.as_object()? else { + return None; + }; + match key.as_str() { + "$numberInt" => digits.parse::().ok().map(|number| Value::Int(number.into())), + "$numberLong" => digits.parse().ok().map(Value::Int), + "$numberDouble" => match digits.as_str() { + "NaN" | "Infinity" | "-Infinity" => digits.parse().ok().map(Value::Float64), + _ => plain_number(digits), + }, + "$numberDecimal" => digits.parse::().ok().map(Value::Decimal), + _ => None, + } +} + fn as_json(value: Value) -> Value { let text = match &value { Value::Null | Value::Json(_) | Value::Undecodable(_) => return value, @@ -78,8 +142,9 @@ fn double(number: f64) -> String { } } +const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + pub(super) fn base64(bytes: &[u8]) -> String { - const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4); for chunk in bytes.chunks(3) { let triple = chunk @@ -98,6 +163,22 @@ pub(super) fn base64(bytes: &[u8]) -> String { out } +/// The bytes of standard base64, `None` for text that is not. +pub(super) fn base64_bytes(text: &str) -> Option> { + let mut bytes = Vec::with_capacity(text.len() / 4 * 3); + let (mut buffer, mut bits) = (0u32, 0u32); + for digit in text.trim_end_matches('=').bytes() { + let sextet = ALPHABET.iter().position(|&letter| letter == digit)?; + buffer = ((buffer << 6) | sextet as u32) & 0xffff; + bits += 6; + if bits >= 8 { + bits -= 8; + bytes.push((buffer >> bits) as u8); + } + } + Some(bytes) +} + #[cfg(test)] mod tests { use crate::column::ResultColumn; @@ -126,6 +207,77 @@ mod tests { } } + #[test] + fn a_number_columns_cell_of_one_number_is_that_number_and_any_other_stays() { + let numbers_column = column("n", ColumnKind::Json { binary: false }).column_type; + let numbers_column = ColumnType { + name: crate::document::NUMBER_TYPE.to_owned(), + ..numbers_column + }; + let json = |text: &str| Value::Json(JsonText::parse(text.to_owned()).unwrap()); + let decimal = |text: &str| Value::Decimal(text.parse().unwrap()); + let numbers = [ + ("7", Value::Int(7)), + ("-7", Value::Int(-7)), + ("19.90", decimal("19.90")), + ("-0.1", Value::Float64(-0.1)), + ("9.99", Value::Float64(9.99)), + ("0.30000000000000004", Value::Float64(0.300_000_000_000_000_04)), + ("1e3", Value::Float64(1000.0)), + ("2.5E-7", Value::Float64(2.5e-7)), + ("123456789012345678901234", decimal("123456789012345678901234")), + (r#"{"$numberInt": "7"}"#, Value::Int(7)), + (r#"{"$numberLong": "3000000000"}"#, Value::Int(3_000_000_000)), + (r#" { "$numberDouble": "1.50" } "#, decimal("1.50")), + (r#"{"$numberDouble": "1e300"}"#, Value::Float64(1e300)), + (r#"{"$numberDouble": "-Infinity"}"#, Value::Float64(f64::NEG_INFINITY)), + (r#"{"$numberDecimal": "9.990"}"#, decimal("9.990")), + ]; + for (text, number) in numbers { + assert_eq!(json_number(&json(text), &numbers_column).into_owned(), number, "{text}"); + } + assert!(matches!( + json_number(&json(r#"{"$numberDouble": "NaN"}"#), &numbers_column).into_owned(), + Value::Float64(number) if number.is_nan() + )); + + for kept in [ + json(r#""7""#), + json("null"), + json("[1]"), + json(r#"{"$numberLong": 7}"#), + json(r#"{"$numberLong": "x"}"#), + json(r#"{"$numberInt": "3000000000"}"#), + json(r#"{"$numberLong": "1", "a": 1}"#), + json(r#"{"$oid": "5f1d7a0c9b1e8b2a3c4d5e6f"}"#), + Value::Text("7".to_owned()), + ] { + assert!( + matches!(json_number(&kept, &numbers_column), Cow::Borrowed(_)), + "{kept:?}" + ); + } + let jsonb = ColumnType { + name: "jsonb".to_owned(), + ..numbers_column.clone() + }; + let mixed = column("m", ColumnKind::Json { binary: false }).column_type; + let text_named_number = ColumnType { + kind: ColumnKind::Text { + length: TextLength::Unbounded, + }, + ..numbers_column + }; + for other in [jsonb, mixed, text_named_number] { + for cell in [json("19.90"), json(r#"{"$numberLong": "7"}"#)] { + assert!(matches!(json_number(&cell, &other), Cow::Borrowed(_)), "{other:?}"); + } + } + assert_eq!(base64_bytes("aGkh"), Some(b"hi!".to_vec())); + assert_eq!(base64_bytes("aGk="), Some(b"hi".to_vec())); + assert_eq!(base64_bytes("a*"), None); + } + #[test] fn a_json_column_shows_each_value_as_json_and_the_others_stay() { let columns = [ diff --git a/core/crates/core/src/document/mod.rs b/core/crates/core/src/document/mod.rs index c9aebfd6a2..4b16b9b6ea 100644 --- a/core/crates/core/src/document/mod.rs +++ b/core/crates/core/src/document/mod.rs @@ -21,7 +21,7 @@ pub use document_filter::filter_operators; pub(crate) use document_filter::{filter_document, typed_document}; pub(crate) use document_read::{count, export, page}; pub(crate) use document_write::{delete, insert, insert_many, key_probe, update}; -pub(crate) use json_cells::json_cells; +pub(crate) use json_cells::{json_cells, json_number}; pub use shell_literal::shell_literal; pub(crate) use shell_literal::{insert_document, insert_start}; pub(crate) use shell_params::ShellParams; @@ -30,6 +30,9 @@ pub use shell_params::bind_param; use crate::dialect::SqlDialect; use crate::sql_syntax::SqlGrammar; +/// The type of a JSON column of numbers of more than one BSON type. +pub const NUMBER_TYPE: &str = "number"; + /// Whether the dialect's statements are shell calls rather than SQL. pub(crate) fn speaks_shell(dialect: &dyn SqlDialect) -> bool { dialect.grammar() == SqlGrammar::Mongo diff --git a/core/crates/core/src/document/shell_literal.rs b/core/crates/core/src/document/shell_literal.rs index 6414de11da..a98adcde64 100644 --- a/core/crates/core/src/document/shell_literal.rs +++ b/core/crates/core/src/document/shell_literal.rs @@ -1,8 +1,10 @@ +use chrono::{DateTime, Datelike}; + use crate::column::ColumnType; use crate::dialect::LiteralError; use crate::meta::TableRef; -use crate::sql_syntax::mongo::ShellValue; -use crate::value::{Temporal, Value}; +use crate::sql_syntax::mongo::{ShellConstructor, ShellNumber, ShellValue, parse_value}; +use crate::value::{JsonText, Temporal, Value}; use super::document_read::collection; @@ -23,10 +25,12 @@ pub fn shell_literal(value: &Value, column: &ColumnType) -> Result format!("NumberDecimal({})", quoted(&number.to_string())), Value::Decimal(number) => format!("NumberDecimal({})", quoted(&number.to_string())), - Value::Float64(number) => double(*number), - Value::Float32(number) => double(f64::from(*number)), + Value::Float64(number) => double(*number).to_string(), + Value::Float32(number) => double(f64::from(*number)).to_string(), Value::Text(text) => quoted(text), - Value::Json(json) => json.as_str().to_owned(), + Value::Json(json) => json_literal(json).ok_or(LiteralError::Unrepresentable { + found: value.variant_name(), + })?, Value::Uuid(uuid) => format!("UUID({})", quoted(&uuid.to_string())), Value::Bytes(bytes) => format!("BinData(0, {})", quoted(&super::json_cells::base64(bytes))), Value::TimestampTz(Temporal::Finite(stamp)) => format!( @@ -61,21 +65,157 @@ pub fn shell_literal(value: &Value, column: &ColumnType) -> Result String { - if number.is_nan() { - return "NaN".to_owned(); +/// Extended JSON written with the shell's helpers, at any depth, so +/// `{"$numberLong": "7"}` reaches mongosh as a long rather than as a +/// document with a `$numberLong` key. +fn json_literal(json: &JsonText) -> Option { + parse_value(json.as_str()) + .ok() + .map(|value| with_helpers(value).to_string()) +} + +fn with_helpers(value: ShellValue) -> ShellValue { + match value { + ShellValue::Object(fields) => match fields.as_slice() { + [(key, inner)] => wrapped(key, inner).unwrap_or_else(|| ShellValue::Object(helped_fields(fields))), + _ => ShellValue::Object(helped_fields(fields)), + }, + ShellValue::Array(items) => ShellValue::Array(items.into_iter().map(with_helpers).collect()), + // A fraction or an exponent marks a double, which mongosh stores + // as an int when it is whole. + ShellValue::Number(number) if number.as_integer().is_none() => { + number.as_f64().map_or(ShellValue::Number(number), double) + } + other => other, } - if number.is_infinite() { - return if number < 0.0 { "-Infinity" } else { "Infinity" }.to_owned(); +} + +fn helped_fields(fields: Vec<(String, ShellValue)>) -> Vec<(String, ShellValue)> { + fields + .into_iter() + .map(|(key, value)| (key, with_helpers(value))) + .collect() +} + +/// The helper an Extended JSON wrapper stands for, or `None` for a +/// document or a wrapper the shell has no helper for, such as `$symbol`. +fn wrapped(key: &str, inner: &ShellValue) -> Option { + let text = inner.as_str(); + let helper = |kind, args| Some(ShellValue::Constructor { kind, args }); + match key { + "$oid" => text + .filter(|hex| hex.len() == 24 && hex.bytes().all(|digit| digit.is_ascii_hexdigit())) + .and_then(|hex| helper(ShellConstructor::ObjectId, vec![ShellValue::string(hex)])), + "$numberInt" => { + let number = text?.parse::().ok()?; + helper(ShellConstructor::Int32, vec![ShellValue::integer(number.into())]) + } + "$numberLong" => { + let number = text?.parse::().ok()?; + helper(ShellConstructor::Int64, vec![ShellValue::string(number.to_string())]) + } + "$numberDouble" => Some(double(ShellNumber::new(text?).as_f64()?)), + "$numberDecimal" => helper(ShellConstructor::Decimal128, vec![ShellValue::string(text?)]), + "$date" => date(inner), + "$binary" => binary(inner), + "$uuid" => text + .filter(|uuid| uuid::Uuid::parse_str(uuid).is_ok()) + .and_then(|uuid| helper(ShellConstructor::Uuid, vec![ShellValue::string(uuid)])), + "$timestamp" => { + let part = |name| match inner.field(name) { + Some(ShellValue::Number(number)) => number.as_integer().filter(|part| u32::try_from(*part).is_ok()), + _ => None, + }; + let args = vec![ShellValue::integer(part("t")?), ShellValue::integer(part("i")?)]; + helper(ShellConstructor::Timestamp, args) + } + "$regularExpression" => Some(regex( + inner.field("pattern")?.as_str()?, + inner.field("options")?.as_str()?, + )), + "$minKey" => helper(ShellConstructor::MinKey, Vec::new()), + "$maxKey" => helper(ShellConstructor::MaxKey, Vec::new()), + "$code" => helper(ShellConstructor::Code, vec![ShellValue::string(text?)]), + _ => None, + } +} + +/// `ISODate` of the ISO text, or of milliseconds for a date mongosh's +/// ISO form cannot spell, before year 0 or past 9999. +fn date(inner: &ShellValue) -> Option { + let millis = match inner { + ShellValue::String(text) => return Some(iso_date(ShellValue::string(text))), + ShellValue::Number(number) => number.as_integer()?, + _ => match inner.as_object()? { + [(key, ShellValue::String(digits))] if key == "$numberLong" => digits.parse().ok()?, + _ => return None, + }, + }; + let argument = match DateTime::from_timestamp_millis(millis).filter(|date| (0..=9999).contains(&date.year())) { + Some(date) => ShellValue::string(date.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()), + None => ShellValue::integer(millis), + }; + Some(iso_date(argument)) +} + +fn iso_date(argument: ShellValue) -> ShellValue { + ShellValue::Constructor { + kind: ShellConstructor::Date, + args: vec![argument], } - let text = format!("{number:?}"); - match text.contains(['.', 'e', 'E']) { - true => text, - false => format!("{text}.0"), +} + +/// `UUID()` for a UUID, subtype 4, and `BinData()` for any other bytes. +fn binary(inner: &ShellValue) -> Option { + let base64 = inner.field("base64")?.as_str()?; + let subtype = u8::from_str_radix(inner.field("subType")?.as_str()?, 16).ok()?; + let uuid = (subtype == 4) + .then(|| super::json_cells::base64_bytes(base64)) + .flatten() + .and_then(|bytes| uuid::Uuid::from_slice(&bytes).ok()); + let (kind, args) = match uuid { + Some(uuid) => (ShellConstructor::Uuid, vec![ShellValue::string(uuid.to_string())]), + None => ( + ShellConstructor::BinData, + vec![ShellValue::integer(subtype.into()), ShellValue::string(base64)], + ), + }; + Some(ShellValue::Constructor { kind, args }) +} + +/// `BSONRegExp()`, which mongosh stores as the pattern and options +/// given. A literal or `RegExp()` is a JavaScript expression first, which +/// drops `s` and refuses `x`, `l` and patterns only PCRE reads. +fn regex(pattern: &str, options: &str) -> ShellValue { + ShellValue::Constructor { + kind: ShellConstructor::BsonRegExp, + args: vec![ShellValue::string(pattern), ShellValue::string(options)], } } +/// A double as text mongosh stores as one. It stores a whole number in +/// the int32 range, `-0` among them, as an int, so that one goes in +/// `Double()`; any other keeps a fraction or an exponent. +fn double(number: f64) -> ShellValue { + let text = if number.is_nan() { + "NaN".to_owned() + } else if number.is_infinite() { + if number < 0.0 { "-Infinity" } else { "Infinity" }.to_owned() + } else if number.fract() == 0.0 && (f64::from(i32::MIN)..=f64::from(i32::MAX)).contains(&number) { + return ShellValue::Constructor { + kind: ShellConstructor::Double, + args: vec![ShellValue::Number(ShellNumber::new(format!("{number}")))], + }; + } else { + let text = format!("{number:?}"); + match text.contains(['.', 'e', 'E']) { + true => text, + false => format!("{text}.0"), + } + }; + ShellValue::Number(ShellNumber::new(text)) +} + /// What every row's `insertOne` shares, up to its document. pub(crate) fn insert_start(table: &TableRef) -> String { format!("{}.insertOne(", collection(table)) @@ -115,7 +255,13 @@ mod tests { (Value::Int(5), "int", "5"), (Value::Int(5), "long", r#"NumberLong("5")"#), (Value::Int(5_000_000_000), "int", r#"NumberLong("5000000000")"#), - (Value::Float64(2.0), "double", "2.0"), + (Value::Float64(2.0), "double", "Double(2)"), + (Value::Float64(-0.0), "double", "Double(-0)"), + (Value::Float64(-2_147_483_648.0), "double", "Double(-2147483648)"), + (Value::Float64(2_147_483_648.0), "double", "2147483648.0"), + (Value::Float64(2.5), "double", "2.5"), + (Value::Float64(1e300), "double", "1e300"), + (Value::Float32(3.0), "double", "Double(3)"), (Value::Text("a\"b".to_owned()), "string", r#""a\"b""#), ( Value::Other(Box::new(OtherValue { @@ -133,7 +279,7 @@ mod tests { ( Value::Json(JsonText::parse(r#"{"a": {"$numberLong": "1"}}"#.to_owned()).unwrap()), "object", - r#"{"a": {"$numberLong": "1"}}"#, + r#"{"a": NumberLong("1")}"#, ), (Value::Null, "string", "null"), ]; @@ -145,6 +291,81 @@ mod tests { assert!(shell_literal(&Value::Array(Vec::new()), &typed("array")).is_err()); } + #[test] + fn extended_json_is_written_with_the_shell_helpers_at_any_depth() { + let cases = [ + (r#"{"$numberLong": "3000000000"}"#, r#"NumberLong("3000000000")"#), + (r#"{"$numberInt": "-7"}"#, "NumberInt(-7)"), + (r#"{"$numberDecimal": "9.99"}"#, r#"NumberDecimal("9.99")"#), + (r#"{"$numberDouble": "NaN"}"#, "NaN"), + (r#"{"$numberDouble": "Infinity"}"#, "Infinity"), + (r#"{"$numberDouble": "-Infinity"}"#, "-Infinity"), + (r#"{"$numberDouble": "2"}"#, "Double(2)"), + (r#"{"$numberDouble": "-0.0"}"#, "Double(-0)"), + (r#"{"$numberDouble": "0.5"}"#, "0.5"), + ( + r#"{"$date": "2024-01-02T03:04:05.006Z"}"#, + r#"ISODate("2024-01-02T03:04:05.006Z")"#, + ), + ( + r#"{"$date": {"$numberLong": "-315619200000"}}"#, + r#"ISODate("1960-01-01T00:00:00.000Z")"#, + ), + ( + r#"{"$date": {"$numberLong": "253402300800000"}}"#, + "ISODate(253402300800000)", + ), + ( + r#"{"$oid": "5f1d7a0c9b1e8b2a3c4d5e6f"}"#, + r#"ObjectId("5f1d7a0c9b1e8b2a3c4d5e6f")"#, + ), + ( + r#"{"$binary": {"base64": "VQ6EAOKbQdSnFkRmVUQAAA==", "subType": "04"}}"#, + r#"UUID("550e8400-e29b-41d4-a716-446655440000")"#, + ), + ( + r#"{"$binary": {"base64": "aGkh", "subType": "80"}}"#, + r#"BinData(128, "aGkh")"#, + ), + ( + r#"{"$timestamp": {"t": 1700000000, "i": 3}}"#, + "Timestamp(1700000000, 3)", + ), + ( + r#"{"$regularExpression": {"pattern": "^a\\.b", "options": "i"}}"#, + r#"BSONRegExp("^a\\.b", "i")"#, + ), + ( + r#"{"$regularExpression": {"pattern": "a/b", "options": "lsx"}}"#, + r#"BSONRegExp("a/b", "lsx")"#, + ), + (r#"{"$minKey": 1}"#, "MinKey()"), + (r#"{"$maxKey": 1}"#, "MaxKey()"), + ( + r#"{"a": [{"$numberLong": "1"}, {"b": {"$oid": "5f1d7a0c9b1e8b2a3c4d5e6f"}}], "c": 1.5}"#, + r#"{"a": [NumberLong("1"), {"b": ObjectId("5f1d7a0c9b1e8b2a3c4d5e6f")}], "c": 1.5}"#, + ), + ( + r#"{"whole": 2.0, "zero": -0.0, "big": 3000000000.0, "tiny": 1e-7, "int": 2}"#, + r#"{"whole": Double(2), "zero": Double(-0), "big": 3000000000.0, "tiny": 1e-7, "int": 2}"#, + ), + (r#"{"$numberLong": "x"}"#, r#"{"$numberLong": "x"}"#), + ( + r#"{"$numberLong": "1", "b": {"$numberInt": "2"}}"#, + r#"{"$numberLong": "1", "b": NumberInt(2)}"#, + ), + (r#"{"$symbol": "s"}"#, r#"{"$symbol": "s"}"#), + ]; + for (json, expected) in cases { + let cell = Value::Json(JsonText::parse(json.to_owned()).unwrap()); + + let literal = shell_literal(&cell, &typed("mixed")).expect("a literal"); + + assert_eq!(literal, expected, "{json}"); + assert!(parse_value(&literal).is_ok(), "{literal}"); + } + } + #[test] fn a_row_is_an_insert_one_of_its_fields() { let start = insert_start(&TableRef::new(Some("shop".to_owned()), "orders")); diff --git a/core/crates/core/src/document/tests.rs b/core/crates/core/src/document/tests.rs index c5f438c3ab..0094b908cb 100644 --- a/core/crates/core/src/document/tests.rs +++ b/core/crates/core/src/document/tests.rs @@ -6,7 +6,7 @@ use crate::column::{ }; use crate::dialect::TestDialect; use crate::dml::{BuildSqlError, build_delete, build_insert, build_multi_row_insert, build_update}; -use crate::edit::CellInput; +use crate::edit::{CellExpression, CellInput}; use crate::filter::{BuildFilterError, Combinator, FilterOp, FilterRule, FilterSet, FilterValue, build_filter}; use crate::meta::{PagePosition, PageStability, RowIdentity, TableRef}; use crate::sql_syntax::SqlGrammar; @@ -288,6 +288,55 @@ fn a_json_field_compares_as_well_as_matching_text() { ); } +#[test] +fn a_field_of_numbers_compares_and_takes_no_pattern() { + let mut columns = columns(); + columns.push(column( + "qty_mixed", + crate::document::NUMBER_TYPE, + ColumnKind::Json { binary: false }, + )); + let offered = crate::document::filter_operators(&columns[4]); + for op in [ + FilterOp::Eq, + FilterOp::NotEq, + FilterOp::Lt, + FilterOp::GtEq, + FilterOp::Between, + FilterOp::In, + FilterOp::NotIn, + FilterOp::IsNull, + FilterOp::IsNotNull, + ] { + assert!(offered.contains(&op), "{op:?} in {offered:?}"); + } + let pattern = FilterSet { + rules: vec![FilterRule { + column: "qty_mixed".to_owned(), + op: FilterOp::Contains, + value: Some(FilterValue::Single("1".to_owned())), + }], + ..FilterSet::default() + }; + + assert_eq!( + build_filter(&mut StatementBuilder::new(), &shell(), &columns, &pattern), + Err(BuildFilterError::OperatorNotAllowed { + column: "qty_mixed".to_owned(), + op: FilterOp::Contains, + }) + ); + for op in [ + FilterOp::StartsWith, + FilterOp::EndsWith, + FilterOp::Like, + FilterOp::NotLike, + FilterOp::Ilike, + ] { + assert!(!offered.contains(&op), "{op:?} in {offered:?}"); + } +} + #[test] fn a_filter_that_is_not_one_document_or_names_a_dotted_field_is_refused() { let mut builder = StatementBuilder::new(); @@ -511,14 +560,34 @@ fn a_change_names_its_document_by_id_and_sets_only_the_fields_changed() { } #[test] -fn a_change_to_null_removes_the_field_and_a_key_keeps_set() { +fn a_null_is_set_and_a_field_given_back_to_the_server_is_unset() { let change = |assignments: &[(usize, CellInput)]| { build_update(&shell(), &users(), &columns(), &by_id(), &[object_id()], assignments).expect("an update") }; + let null = change(&[(2, CellInput::Value(Value::Null))]); + assert_eq!( + checked(&null), + r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $2}, {"$set": {"qty": $1}})"# + ); + assert_eq!(null.params()[0].value(), &Value::Null); + assert_eq!( + null.params()[0].param_type(), + &ParamType::Int32, + "a null keeps its field's type" + ); + let json_null = change(&[( + 1, + CellInput::Value(Value::Json(JsonText::parse("null".to_owned()).expect("JSON"))), + )]); + assert_eq!( + checked(&json_null), + r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $2}, {"$set": {"name": $1}})"# + ); + let both = change(&[ (1, CellInput::Value(Value::Text("ann".to_owned()))), - (2, CellInput::Value(Value::Null)), + (2, CellInput::Default), ]); assert_eq!( checked(&both), @@ -526,37 +595,16 @@ fn a_change_to_null_removes_the_field_and_a_key_keeps_set() { ); assert_eq!(both.params().len(), 2); - let removed = change(&[(2, CellInput::Value(Value::Null))]); + let removed = change(&[(2, CellInput::Default)]); assert_eq!( checked(&removed), r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $1}, {"$unset": {"qty": ""}})"# ); assert_eq!(removed.params().len(), 1); - - let key = change(&[(0, CellInput::Value(Value::Null))]); - assert_eq!( - checked(&key), - r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $2}, {"$set": {"_id": $1}})"# - ); - assert_eq!(key.params()[0].value(), &Value::Null); - assert!( - matches!(key.params()[0].param_type(), ParamType::ServerText(_)), - "a null keeps its field's type" - ); - - let json_null = change(&[( - 1, - CellInput::Value(Value::Json(JsonText::parse("null".to_owned()).expect("JSON"))), - )]); - assert_eq!( - checked(&json_null), - r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $2}, {"$set": {"name": $1}})"#, - "a typed null is a value" - ); } #[test] -fn a_new_document_leaves_out_every_null_cell() { +fn a_new_document_writes_a_null_and_leaves_out_a_default() { let dialect = shell(); let table = users(); let columns = columns(); @@ -570,27 +618,16 @@ fn a_new_document_leaves_out_every_null_cell() { CellInput::Default, CellInput::Value(Value::Text("ann".to_owned())), null(), - null(), + CellInput::Default, ], ) .expect("an insert"); assert_eq!( checked(&insert), - r#"db.getSiblingDB("shop").getCollection("users").insertOne({"name": $1})"# - ); - assert_eq!(insert.params().len(), 1); - - let empty = build_insert( - &dialect, - &table, - &columns, - &[CellInput::Default, null(), null(), null()], - ) - .expect("an insert of no field"); - assert_eq!( - checked(&empty), - r#"db.getSiblingDB("shop").getCollection("users").insertOne({})"# + r#"db.getSiblingDB("shop").getCollection("users").insertOne({"name": $1, "qty": $2})"# ); + assert_eq!(insert.params()[1].value(), &Value::Null); + assert_eq!(insert.params()[1].param_type(), &ParamType::Int32); let many = build_multi_row_insert( &dialect, @@ -598,16 +635,19 @@ fn a_new_document_leaves_out_every_null_cell() { &[&columns[1], &columns[2]], vec![ vec![Value::Text("a".to_owned()), Value::Null], - vec![Value::Null, Value::Int(2)], vec![Value::Null, Value::Null], ], ) - .expect("an insert of three"); + .expect("an insert of two"); assert_eq!( checked(&many), - r#"db.getSiblingDB("shop").getCollection("users").insertMany([{"name": $1}, {"qty": $2}, {}])"# + r#"db.getSiblingDB("shop").getCollection("users").insertMany([{"name": $1, "qty": $2}, {"name": $3, "qty": $4}])"# + ); + assert!( + many.params()[1..].iter().all(|param| param.value() == &Value::Null), + "{:?}", + many.params() ); - assert_eq!(many.params().len(), 2); } #[test] @@ -629,19 +669,20 @@ fn a_dotted_field_cannot_be_updated_by_name() { #[test] fn a_document_field_is_never_set_to_a_value_the_server_computes() { - let refused = build_update( - &shell(), - &users(), - &columns(), - &by_id(), - &[object_id()], - &[(1, CellInput::Default)], - ); + const NOW: CellExpression = CellExpression::new("NOW()", &[], &[]); + let refused = |assignment: (usize, CellInput)| { + build_update(&shell(), &users(), &columns(), &by_id(), &[object_id()], &[assignment]) + }; assert_eq!( - refused, + refused((1, CellInput::Expression(NOW))), Err(BuildSqlError::ServerComputedValue { name: "name".to_owned() }) ); + assert_eq!( + refused((0, CellInput::Default)), + Err(BuildSqlError::ServerComputedValue { name: "_id".to_owned() }), + "a key was unset" + ); } diff --git a/core/crates/core/src/document/value_order.rs b/core/crates/core/src/document/value_order.rs index 63f8aa9c23..40e1afac43 100644 --- a/core/crates/core/src/document/value_order.rs +++ b/core/crates/core/src/document/value_order.rs @@ -83,7 +83,7 @@ fn shell_bracket(value: &ShellValue) -> Option { ShellConstructor::Uuid | ShellConstructor::BinData | ShellConstructor::HexData => "binData", ShellConstructor::MinKey => "minKey", ShellConstructor::MaxKey => "maxKey", - ShellConstructor::RegExp => "regex", + ShellConstructor::RegExp | ShellConstructor::BsonRegExp => "regex", ShellConstructor::Code => return None, }, ShellValue::Object(fields) => match fields.first().map(|(key, _)| key.as_str()) { diff --git a/core/crates/core/src/edit/mod.rs b/core/crates/core/src/edit/mod.rs index fb4ba8a9b3..4bdd3bbc2e 100644 --- a/core/crates/core/src/edit/mod.rs +++ b/core/crates/core/src/edit/mod.rs @@ -21,7 +21,7 @@ pub use edit_parse_error::{EditParseError, ReadOnlyReason}; pub use edit_text::{EditText, edit_text, read_only_reason}; pub use empty_input_meaning::{EmptyInputMeaning, empty_input_meaning}; pub use is_unchanged::is_unchanged; -pub use new_draft_cell::new_draft_cell; +pub use new_draft_cell::{new_draft_cell, read_cell}; pub use parse_edit_text::parse_edit_text; pub use parse_literal_text::{ parse_filter_text, parse_literal_text, parse_time_with_offset, parse_timestamp_with_offset, diff --git a/core/crates/core/src/edit/new_draft_cell.rs b/core/crates/core/src/edit/new_draft_cell.rs index 1b42944319..5bae2f9559 100644 --- a/core/crates/core/src/edit/new_draft_cell.rs +++ b/core/crates/core/src/edit/new_draft_cell.rs @@ -1,4 +1,5 @@ use crate::column::ColumnInfo; +use crate::dialect::SqlDialect; use crate::value::Value; use super::CellInput; @@ -11,3 +12,51 @@ pub fn new_draft_cell(column: &ColumnInfo) -> CellInput { false => CellInput::Value(Value::Null), } } + +/// A value the app read as a new row's cell, as Duplicate Rows copies one +/// and a CSV import reads its null marker. Where the dialect reads a value +/// a row does not hold as NULL, a NULL takes the column's default, which +/// leaves the value out as the row it came from did. A NULL the user types +/// is a value and never comes through here. +pub fn read_cell(value: Value, column: &ColumnInfo, dialect: &dyn SqlDialect) -> CellInput { + match value.is_null() && dialect.reads_missing_as_null() { + true => new_draft_cell(column), + false => CellInput::Value(value), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::column::{ColumnDefault, ColumnType, SqlExpression}; + use crate::dialect::TestDialect; + + #[test] + fn a_null_read_where_a_missing_value_reads_as_null_is_left_to_the_default() { + let mut field = ColumnInfo { + name: "email".to_owned(), + column_type: ColumnType::untyped_text("string"), + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + partition_scoped: false, + default: ColumnDefault::None, + comment: None, + }; + let documents = TestDialect::default().reading_missing_as_null(); + let rows = TestDialect::default(); + + assert_eq!( + read_cell(Value::Null, &field, &documents), + CellInput::Value(Value::Null) + ); + field.default = ColumnDefault::Expression(SqlExpression::from_catalog_text("$$REMOVE")); + assert_eq!(read_cell(Value::Null, &field, &documents), CellInput::Default); + assert_eq!( + read_cell(Value::Int(1), &field, &documents), + CellInput::Value(Value::Int(1)) + ); + assert_eq!(read_cell(Value::Null, &field, &rows), CellInput::Value(Value::Null)); + } +} diff --git a/core/crates/core/src/export/csv.rs b/core/crates/core/src/export/csv.rs index 55ad66485b..a48cf765ee 100644 --- a/core/crates/core/src/export/csv.rs +++ b/core/crates/core/src/export/csv.rs @@ -64,6 +64,7 @@ fn collapse_line_breaks(text: String, opts: &CsvOptions) -> String { } pub(super) fn csv_cell(value: &Value, column: &ColumnType, opts: &CsvOptions) -> String { + let value = &*crate::document::json_number(value, column); let Some(text) = value_text(value, column) else { return if opts.null_to_empty { String::new() @@ -191,6 +192,24 @@ mod tests { } } + #[test] + fn a_number_fields_json_is_written_as_the_number_and_other_json_as_it_is() { + let (columns, rows) = super::super::test_columns::mixed_json(); + let opts = CsvOptions { + header_row: false, + decimal: CsvDecimal::Comma, + delimiter: CsvDelimiter::Semicolon, + ..plain() + }; + + let out = render_csv(&columns, &rows, &opts).unwrap(); + + assert_eq!( + out, + "3000000000;19,90;\"{\"\"$oid\"\": \"\"5f1d7a0c9b1e8b2a3c4d5e6f\"\"}\";19.90;\"{\"\"$numberLong\"\": \"\"7\"\"}\"\n" + ); + } + // An importer reads the quote a sanitized field gains as part of the value. #[test] fn csv_defaults_write_values_as_they_are() { diff --git a/core/crates/core/src/export/encoder/json_object.rs b/core/crates/core/src/export/encoder/json_object.rs index 4e61791764..c2ed539545 100644 --- a/core/crates/core/src/export/encoder/json_object.rs +++ b/core/crates/core/src/export/encoder/json_object.rs @@ -51,13 +51,17 @@ impl JsonObjectWriter { } out.extend_from_slice(key); match cells.get(index).copied() { - Some(value) => write_json( - out, - &JsonCell { - value, - kind: &column_type(&self.columns, index).kind, - }, - ), + Some(value) => { + let column = column_type(&self.columns, index); + let value = crate::document::json_number(value, column); + write_json( + out, + &JsonCell { + value: &value, + kind: &column.kind, + }, + ); + } // A row shorter than its header: the cell is missing, // which is not the same as a NULL the row holds. None => out.extend_from_slice(b"null"), diff --git a/core/crates/core/src/export/encoder/tsv_encoder.rs b/core/crates/core/src/export/encoder/tsv_encoder.rs index 245012465e..7087c58d3b 100644 --- a/core/crates/core/src/export/encoder/tsv_encoder.rs +++ b/core/crates/core/src/export/encoder/tsv_encoder.rs @@ -42,7 +42,8 @@ impl RowEncoder for TsvEncoder { self.writer.write( out, cells.iter().enumerate().map(|(index, value)| { - value_text(value, column_type(&self.columns, index)).unwrap_or_else(|| TSV_NULL.to_owned()) + let column = column_type(&self.columns, index); + value_text(&crate::document::json_number(value, column), column).unwrap_or_else(|| TSV_NULL.to_owned()) }), ); Ok(()) diff --git a/core/crates/core/src/export/html.rs b/core/crates/core/src/export/html.rs index 6fe31c0270..d40c75519d 100644 --- a/core/crates/core/src/export/html.rs +++ b/core/crates/core/src/export/html.rs @@ -37,7 +37,8 @@ pub(super) fn html_head(out: &mut String, columns: &[ResultColumn], title: &str) pub(super) fn html_row<'a>(out: &mut String, columns: &[ResultColumn], row: impl IntoIterator) { out.push_str(""); for (index, value) in row.into_iter().enumerate() { - match value_text(value, column_type(columns, index)) { + let column = column_type(columns, index); + match value_text(&crate::document::json_number(value, column), column) { Some(text) => { out.push_str(""); escape_into(out, &text); @@ -103,6 +104,19 @@ mod tests { assert!(!html.contains(""), "a column name reached the document as markup"); } + #[test] + fn a_number_fields_json_is_written_as_the_number_and_other_json_as_it_is() { + let (columns, rows) = super::super::test_columns::mixed_json(); + + let html = render_html(&columns, &rows, "t"); + + assert!(html.contains("300000000019.90{"), "{html}"); + assert!( + html.contains("19.90{"$numberLong": "7"}"), + "{html}" + ); + } + #[test] fn a_null_is_told_apart_from_the_text_null() { let rows = vec![vec![Value::Null, Value::Text("NULL".into())]]; diff --git a/core/crates/core/src/export/json.rs b/core/crates/core/src/export/json.rs index 61e89b7faa..5e333673b6 100644 --- a/core/crates/core/src/export/json.rs +++ b/core/crates/core/src/export/json.rs @@ -21,13 +21,17 @@ impl Serialize for JsonRow<'_> { let mut map = serializer.serialize_map(Some(self.names.len()))?; for (index, name) in self.names.iter().enumerate() { match self.row.get(index) { - Some(value) => map.serialize_entry( - name, - &JsonCell { - value, - kind: &column_type(self.columns, index).kind, - }, - )?, + Some(value) => { + let column = column_type(self.columns, index); + let value = crate::document::json_number(value, column); + map.serialize_entry( + name, + &JsonCell { + value: &value, + kind: &column.kind, + }, + )?; + } // A row shorter than its header: the cell is missing, // which is not the same as a NULL the row holds. None => map.serialize_entry(name, &Option::<()>::None)?, @@ -156,6 +160,20 @@ mod tests { ); } + #[test] + fn a_number_fields_json_goes_out_as_the_number_and_other_json_as_it_is() { + let (columns, rows) = super::super::test_columns::mixed_json(); + + let text = render_json(&columns, &rows).expect("the array"); + let json: serde_json::Value = serde_json::from_str(&text).expect("JSON"); + + assert_eq!(json[0]["long"], serde_json::json!(3_000_000_000_i64)); + assert!(text.contains(r#""price":19.90,"#), "{text}"); + assert_eq!(json[0]["doc"]["$oid"], serde_json::json!("5f1d7a0c9b1e8b2a3c4d5e6f")); + assert!(text.contains(r#""pg_price":19.90,"#), "{text}"); + assert_eq!(json[0]["pg_long"]["$numberLong"], serde_json::json!("7")); + } + #[test] fn json_number_vs_string_handling() { let columns = cols(&["i", "f", "d", "s"]); diff --git a/core/crates/core/src/export/markdown.rs b/core/crates/core/src/export/markdown.rs index b4adff2e0b..ceb81467b9 100644 --- a/core/crates/core/src/export/markdown.rs +++ b/core/crates/core/src/export/markdown.rs @@ -5,7 +5,9 @@ use super::encoder::{EncodeError, MarkdownEncoder, encode_rows}; use super::value_text::value_text; pub(super) fn markdown_cell(value: &Value, column: &ColumnType) -> String { - markdown_text(&value_text(value, column).unwrap_or_else(|| "NULL".to_string())) + markdown_text( + &value_text(&crate::document::json_number(value, column), column).unwrap_or_else(|| "NULL".to_string()), + ) } /// A pipe or a line break, in a cell or a column name alike, would end @@ -35,6 +37,20 @@ mod tests { assert_eq!(out, "| a |\n| --- |\n| has\\|pipe
and newline |"); } + #[test] + fn a_number_fields_json_is_written_as_the_number_and_other_json_as_it_is() { + let (columns, rows) = super::super::test_columns::mixed_json(); + + let out = render_markdown(&columns, &rows).expect("the table"); + + assert!( + out.ends_with( + r#"| 3000000000 | 19.90 | {"$oid": "5f1d7a0c9b1e8b2a3c4d5e6f"} | 19.90 | {"$numberLong": "7"} |"# + ), + "{out}" + ); + } + #[test] fn markdown_escapes_a_pipe_in_a_column_name() { let columns = cols(&["a|b", "c"]); diff --git a/core/crates/core/src/export/mod.rs b/core/crates/core/src/export/mod.rs index 44f8123118..d4f28a061a 100644 --- a/core/crates/core/src/export/mod.rs +++ b/core/crates/core/src/export/mod.rs @@ -130,4 +130,28 @@ mod test_columns { pub fn cols(names: &[&str]) -> Vec { names.iter().map(|name| col(name)).collect() } + + /// A MongoDB field of numbers of several types holding a long and a + /// fraction, one of mixed values holding a document, and two SQL JSON + /// columns holding a number and a wrapper, each a JSON cell. + pub fn mixed_json() -> (Vec, Vec>) { + let json = + |text: &str| crate::value::Value::Json(crate::value::JsonText::parse(text.to_owned()).expect("JSON")); + let json_kind = || ColumnKind::Json { binary: false }; + let columns = vec![ + typed("long", crate::document::NUMBER_TYPE, json_kind()), + typed("price", crate::document::NUMBER_TYPE, json_kind()), + typed("doc", "mixed", json_kind()), + typed("pg_price", "jsonb", json_kind()), + typed("pg_long", "jsonb", json_kind()), + ]; + let rows = vec![vec![ + json(r#"{"$numberLong": "3000000000"}"#), + json("19.90"), + json(r#"{"$oid": "5f1d7a0c9b1e8b2a3c4d5e6f"}"#), + json("19.90"), + json(r#"{"$numberLong": "7"}"#), + ]]; + (columns, rows) + } } diff --git a/core/crates/core/src/export/sql.rs b/core/crates/core/src/export/sql.rs index 5c57444dac..9f831016ae 100644 --- a/core/crates/core/src/export/sql.rs +++ b/core/crates/core/src/export/sql.rs @@ -128,9 +128,7 @@ impl InsertWriter { Some((column, written.as_ref()?, cell)) }) .collect(); - // A document of no field is `{}`, while a SQL row needs a column. - let leaves_out_nulls = - !dialect.insert_names_null_cells() && (self.document || cells.iter().any(|(_, _, cell)| !is_null(*cell))); + let leaves_out_nulls = !dialect.insert_names_null_cells() && cells.iter().any(|(_, _, cell)| !is_null(*cell)); let cells: Vec<(&ResultColumn, &WrittenColumn, Option<&Value>)> = cells .into_iter() .filter(|(_, _, cell)| !(leaves_out_nulls && is_null(*cell))) @@ -291,7 +289,6 @@ fn document_fields( for (index, column) in columns.iter().enumerate() { let value = match cells.get(index) { Some(CellInput::Default | CellInput::Expression(_)) | None => continue, - Some(CellInput::Value(Value::Null)) if !dialect.insert_names_null_cells() => continue, Some(CellInput::Value(value)) => dialect.literal(value, &column.column_type)?, }; fields.push((column.name.clone(), value)); @@ -806,41 +803,6 @@ mod tests { ); } - #[test] - fn a_document_of_only_nulls_is_empty_and_a_row_of_only_nulls_names_them() { - let rows = [vec![Value::Int(1), Value::Null], vec![Value::Null, Value::Null]]; - let render = |dialect: &dyn SqlDialect| { - render_sql_insert( - dialect, - &TableRef::unqualified("t"), - &cols(&["a", "b"]), - &[], - &[], - &rows, - ) - }; - let documents = Numbering { - inner: TestDialect::default().parsing_as(SqlGrammar::Mongo), - null_cells_left_out: true, - ..Numbering::default() - }; - let rows_of_sql = Numbering { - null_cells_left_out: true, - ..Numbering::default() - }; - - assert_eq!( - render(&documents), - "db.getCollection(\"t\").insertOne({\"a\": 1});\n\ - db.getCollection(\"t\").insertOne({});\n" - ); - assert_eq!( - render(&rows_of_sql), - "INSERT INTO \"t\" (\"a\") VALUES (1);\n\ - INSERT INTO \"t\" (\"a\", \"b\") VALUES (NULL, NULL);\n" - ); - } - /// Every call's output taken as soon as it is written, as an export /// sink takes it. fn streamed(mut encoder: SqlInsertEncoder, rows: &[Vec]) -> String { diff --git a/core/crates/core/src/export/text_table.rs b/core/crates/core/src/export/text_table.rs index e95113fabf..732542bf16 100644 --- a/core/crates/core/src/export/text_table.rs +++ b/core/crates/core/src/export/text_table.rs @@ -67,6 +67,7 @@ where .enumerate() .map(|(index, value)| { let column = column_type(columns, index); + let value = &*crate::document::json_number(value, column); if let Some(number) = numbers.get_mut(index) { number.take(value); } @@ -252,6 +253,23 @@ mod tests { ); } + #[test] + fn text_table_writes_a_number_fields_json_as_the_number_and_other_json_as_it_is() { + let (columns, rows) = super::super::test_columns::mixed_json(); + let kept = [0, 1, 3]; + let columns: Vec = kept.iter().map(|index| columns[*index].clone()).collect(); + let rows = vec![kept.iter().map(|index| rows[0][*index].clone()).collect::>()]; + + assert_eq!( + table(&columns, &rows), + "+------------+-------+----------+\n\ + | long | price | pg_price |\n\ + +------------+-------+----------+\n\ + | 3000000000 | 19.90 | 19.90 |\n\ + +------------+-------+----------+" + ); + } + #[test] fn text_table_lines_up_an_untyped_column_of_numbers_as_the_grid_does() { let columns = vec![ diff --git a/core/crates/core/src/export/tsv.rs b/core/crates/core/src/export/tsv.rs index 5a967c1189..09ec23f93e 100644 --- a/core/crates/core/src/export/tsv.rs +++ b/core/crates/core/src/export/tsv.rs @@ -71,6 +71,16 @@ mod tests { ); } + #[test] + fn a_number_fields_json_is_written_as_the_number_and_other_json_as_it_is() { + let (columns, rows) = super::super::test_columns::mixed_json(); + + assert_eq!( + render_tsv(&columns, &rows, false).unwrap(), + "3000000000\t19.90\t\"{\"\"$oid\"\": \"\"5f1d7a0c9b1e8b2a3c4d5e6f\"\"}\"\t19.90\t\"{\"\"$numberLong\"\": \"\"7\"\"}\"\n" + ); + } + #[test] fn tsv_leaves_ordinary_values_bare() { let columns = cols(&["a"]); diff --git a/core/crates/core/src/export/xlsx.rs b/core/crates/core/src/export/xlsx.rs index 5ee065d76d..93a67de68b 100644 --- a/core/crates/core/src/export/xlsx.rs +++ b/core/crates/core/src/export/xlsx.rs @@ -148,6 +148,7 @@ fn write_cell( value: &Value, formats: &CellFormats, ) -> Result<(), RenderError> { + let value = &*crate::document::json_number(value, super::column_type(columns, usize::from(column))); let written = match value { // A blank cell, not an empty string: Excel's COUNT and AVERAGE // read the two differently, and NULL is the absent one. @@ -531,6 +532,19 @@ mod tests { ); } + #[test] + fn a_number_fields_json_is_a_number_a_sheet_sums_and_other_json_is_text() { + let (columns, rows) = crate::export::test_columns::mixed_json(); + + let xml = sheet_xml(&render_xlsx(&columns, &rows, "Results").expect("render")); + + assert!(xml.contains("3000000000"), "{xml}"); + assert!(xml.contains("19.9"), "{xml}"); + assert!(xml.contains("5f1d7a0c9b1e8b2a3c4d5e6f"), "{xml}"); + assert!(xml.contains("19.90"), "{xml}"); + assert!(xml.contains(r#"{"$numberLong": "7"}"#), "{xml}"); + } + #[test] fn a_date_before_1900_is_written_as_text() { let day = chrono::NaiveDate::from_ymd_opt(1899, 12, 31).expect("a date"); diff --git a/core/crates/core/src/export/xml.rs b/core/crates/core/src/export/xml.rs index e19e1cd8db..e31f997089 100644 --- a/core/crates/core/src/export/xml.rs +++ b/core/crates/core/src/export/xml.rs @@ -36,7 +36,8 @@ pub(super) fn xml_row<'a>( }; out.push_str(" { out.push_str("\">"); escape_into(out, &text, Position::Text).map_err(|_| refused())?; @@ -114,6 +115,21 @@ mod tests { assert!(xml.contains("a < b & "c""), "{xml}"); } + #[test] + fn a_number_fields_json_is_written_as_the_number_and_other_json_as_it_is() { + let (columns, rows) = super::super::test_columns::mixed_json(); + + let xml = render_xml(&columns, &rows).expect("the document"); + + assert!(xml.contains("3000000000"), "{xml}"); + assert!(xml.contains("19.90"), "{xml}"); + assert!(xml.contains("19.90"), "{xml}"); + assert!( + xml.contains("{"$numberLong": "7"}"), + "{xml}" + ); + } + #[test] fn a_null_is_told_apart_from_the_empty_string() { let rows = vec![vec![Value::Null, Value::Text(String::new())]]; diff --git a/core/crates/core/src/import/import_plan.rs b/core/crates/core/src/import/import_plan.rs index 5f6f4daec9..7146f3d8e6 100644 --- a/core/crates/core/src/import/import_plan.rs +++ b/core/crates/core/src/import/import_plan.rs @@ -3,7 +3,7 @@ use std::ops::Range; use crate::column::ColumnInfo; use crate::dialect::{InsertConflict, SqlDialect}; use crate::dml::{BuildSqlError, build_insert, build_multi_row_insert}; -use crate::edit::CellInput; +use crate::edit::{CellInput, read_cell}; use crate::meta::TableRef; use crate::save::insert_step; use crate::value::Value; @@ -31,7 +31,7 @@ pub struct ImportPlan { /// have missed, and SQL Server counts trigger rows in its own, so only /// an engine that skips a taken key checks the count. An engine that /// replaces a taken key writes a row per step, each after a probe for -/// its key. +/// its key. A NULL the file held is one the import read (`read_cell`). pub fn plan_import( dialect: &dyn SqlDialect, table: &TableRef, @@ -46,7 +46,11 @@ pub fn plan_import( let infos: Vec = columns.iter().map(|column| column.info.clone()).collect(); let mut steps = Vec::new(); let mut step_rows = Vec::new(); - let mut rows = rows.into_iter().enumerate().peekable(); + let mut rows = rows + .into_iter() + .map(|cells| read_cells(cells, &infos, dialect)) + .enumerate() + .peekable(); while let Some((first, cells)) = rows.next() { let shape = written_columns(&infos, &cells); let width = shape.iter().filter(|written| **written).count(); @@ -103,6 +107,17 @@ pub fn plan_import( }) } +fn read_cells(cells: Vec, columns: &[ColumnInfo], dialect: &dyn SqlDialect) -> Vec { + cells + .into_iter() + .enumerate() + .map(|(index, cell)| match (cell, columns.get(index)) { + (CellInput::Value(value), Some(column)) => read_cell(value, column, dialect), + (cell, _) => cell, + }) + .collect() +} + /// Which columns a row writes: every one it has a value for that the /// server does not compute. fn written_columns(columns: &[ColumnInfo], cells: &[CellInput]) -> Vec { @@ -330,6 +345,29 @@ mod tests { ); } + #[test] + fn a_null_the_file_held_is_left_to_a_default_where_a_missing_value_reads_as_null() { + let mut columns = columns(); + columns[1].info.default = + ColumnDefault::Expression(crate::column::SqlExpression::from_catalog_text("$$REMOVE")); + let null_name = || vec![CellInput::Value(Value::Int(1)), CellInput::Value(Value::Null)]; + let plan = + |dialect: &TestDialect| plan_import(dialect, &table(), &columns, vec![null_name()]).expect("the plan"); + + let documents = plan(&TestDialect::default().reading_missing_as_null()); + let rows = plan(&TestDialect::default()); + + assert_eq!( + documents.batch.steps[0].statement.sql(), + r#"INSERT INTO "public"."people" ("id") VALUES ($1)"# + ); + assert_eq!( + rows.batch.steps[0].statement.sql(), + r#"INSERT INTO "public"."people" ("id", "name") VALUES ($1, $2)"# + ); + assert_eq!(rows.batch.steps[0].statement.params()[1].value(), &Value::Null); + } + #[test] fn a_generated_column_is_never_written() { let mut columns = columns(); diff --git a/core/crates/core/src/sql_syntax/mongo/shell_access.rs b/core/crates/core/src/sql_syntax/mongo/shell_access.rs index 5817f38533..ceb1670138 100644 --- a/core/crates/core/src/sql_syntax/mongo/shell_access.rs +++ b/core/crates/core/src/sql_syntax/mongo/shell_access.rs @@ -187,8 +187,15 @@ fn call_access(call: &ShellCall) -> StatementAccess { "status" | "conf" | "config" | "hello" | "isMaster" | "printReplicationInfo" => StatementAccess::Read, _ => StatementAccess::Write, }, - (CallRoot::Db, Some(_)) => collection_access(&operation.name, &operation.args), - (CallRoot::Db, None) => database_access(&operation.name, &operation.args), + (CallRoot::Db, Some(collection)) => collection_access( + &operation.name, + &operation.args, + Source { + database: call.database.as_deref(), + collection, + }, + ), + (CallRoot::Db, None) => database_access(&operation.name, &operation.args, call.database.as_deref()), }; match shaped { true => access, @@ -196,9 +203,11 @@ fn call_access(call: &ShellCall) -> StatementAccess { } } -fn collection_access(method: &str, args: &[ShellValue]) -> StatementAccess { +fn collection_access(method: &str, args: &[ShellValue], source: Source<'_>) -> StatementAccess { match method { - "aggregate" => args.first().map_or(StatementAccess::Read, pipeline_access), + "aggregate" => args.first().map_or(StatementAccess::Read, |pipeline| { + pipeline_access(pipeline, Some(source)) + }), _ if COLLECTION_READS.contains(&method) => StatementAccess::Read, _ if COLLECTION_DROPS.contains(&method) => StatementAccess::Destructive, // `dropTarget` drops the collection the rename replaces. @@ -245,10 +254,18 @@ fn is_true(value: &ShellValue) -> bool { } } -fn database_access(method: &str, args: &[ShellValue]) -> StatementAccess { +fn database_access(method: &str, args: &[ShellValue], database: Option<&str>) -> StatementAccess { + let database = match method { + "adminCommand" => Some("admin"), + _ => database, + }; match method { - "runCommand" | "adminCommand" => args.first().map_or(StatementAccess::Write, command_access), - "aggregate" => args.first().map_or(StatementAccess::Read, pipeline_access), + "runCommand" | "adminCommand" => args + .first() + .map_or(StatementAccess::Write, |command| command_access(command, database)), + "aggregate" => args + .first() + .map_or(StatementAccess::Read, |pipeline| pipeline_access(pipeline, None)), _ if DATABASE_READS.contains(&method) => StatementAccess::Read, _ if DATABASE_DROPS.contains(&method) => StatementAccess::Destructive, _ => StatementAccess::Write, @@ -256,8 +273,9 @@ fn database_access(method: &str, args: &[ShellValue]) -> StatementAccess { } /// A command document by its first key, the command's name, which the -/// server matches without regard to case for most commands. -fn command_access(command: &ShellValue) -> StatementAccess { +/// server matches without regard to case for most commands. `database` +/// is the one it runs in, `None` for the current one. +fn command_access(command: &ShellValue, database: Option<&str>) -> StatementAccess { let Some(name) = command .as_object() .and_then(|fields| fields.first()) @@ -267,11 +285,17 @@ fn command_access(command: &ShellValue) -> StatementAccess { }; let name = name.to_ascii_lowercase(); let set = |option: &str| command.field(option).is_some_and(is_true); - let pipeline = command.field("pipeline").map_or(StatementAccess::Read, pipeline_access); + let source = (name == "aggregate") + .then(|| command.field("aggregate").and_then(ShellValue::as_str)) + .flatten() + .map(|collection| Source { database, collection }); + let pipeline = command + .field("pipeline") + .map_or(StatementAccess::Read, |pipeline| pipeline_access(pipeline, source)); let writes = pipeline > StatementAccess::Read || command .field("explain") - .is_some_and(|inner| command_access(inner) > StatementAccess::Read) + .is_some_and(|inner| command_access(inner, database) > StatementAccess::Read) || (name == "validate" && VALIDATE_WRITES.into_iter().any(set)); let destructive = pipeline == StatementAccess::Destructive || COMMAND_DROPS.contains(&name.as_str()) @@ -320,11 +344,20 @@ fn reaches_every_document(name: &str, command: &ShellValue) -> bool { .any(|entry| every(entry) && entry.field(filter).is_none_or(matches_every_document)) } +/// The collection an aggregate reads, in `database`, `None` for the +/// current one. +#[derive(Clone, Copy)] +struct Source<'a> { + database: Option<&'a str>, + collection: &'a str, +} + /// What a pipeline does, by the stages it holds at any depth: a -/// `$lookup` or `$facet` holds pipelines of its own. -fn pipeline_access(pipeline: &ShellValue) -> StatementAccess { +/// `$lookup` or `$facet` holds pipelines of its own. A `$merge` into the +/// collection it reads rewrites that collection's documents. +fn pipeline_access(pipeline: &ShellValue, source: Option>) -> StatementAccess { let stages = pipeline.keys(); - if stages.contains(&REPLACING_STAGE) { + if stages.contains(&REPLACING_STAGE) || source.is_some_and(|source| merges_into(pipeline, source)) { StatementAccess::Destructive } else if stages.contains(&MERGING_STAGE) { StatementAccess::Write @@ -333,6 +366,39 @@ fn pipeline_access(pipeline: &ShellValue) -> StatementAccess { } } +/// Whether a `$merge` at any depth may write into `source`. Its `into` +/// names a collection of the current database, or `{db, coll}`. A `db` +/// named where the call names none, which may be the current one, and a +/// target that is not a name count as the source, the stricter reading. +fn merges_into(value: &ShellValue, source: Source<'_>) -> bool { + match value { + ShellValue::Object(fields) => fields + .iter() + .any(|(key, value)| (key == MERGING_STAGE && merge_names(value, source)) || merges_into(value, source)), + ShellValue::Array(items) | ShellValue::Constructor { args: items, .. } => { + items.iter().any(|item| merges_into(item, source)) + } + _ => false, + } +} + +fn merge_names(stage: &ShellValue, source: Source<'_>) -> bool { + let into = match stage.as_object() { + Some(_) => stage.field("into"), + None => Some(stage), + }; + let (database, collection) = match into { + Some(ShellValue::String(collection)) => (None, Some(collection.as_str())), + Some(target @ ShellValue::Object(_)) => (target.field("db"), target.field("coll").and_then(ShellValue::as_str)), + _ => (None, None), + }; + let other_database = matches!( + (database, source.database), + (Some(ShellValue::String(named)), Some(read)) if named != read + ); + !other_database && collection.is_none_or(|collection| collection == source.collection) +} + /// Whether a `mapReduce` output replaces a collection: a collection's /// name or `{replace: ...}`. `{merge: ...}` and `{reduce: ...}` write into /// one, and `{inline: 1}` writes nowhere. Anything else but an object @@ -432,6 +498,32 @@ mod tests { } } + #[test] + fn a_merge_into_the_collection_it_reads_is_destructive() { + for text in [ + "db.users.aggregate([{$match: {}}, {$merge: {into: 'users', whenMatched: 'replace'}}])", + "db.users.aggregate([{$merge: 'users'}])", + "db.users.aggregate([{$merge: {into: {coll: 'users'}}}])", + "db.getSiblingDB('shop').users.aggregate([{$merge: {into: {db: 'shop', coll: 'users'}}}])", + "db.users.aggregate([{$merge: {into: {db: 'shop', coll: 'users'}}}])", + "db.getCollection('users').aggregate([{$merge: {whenMatched: 'replace'}}])", + "db.runCommand({aggregate: 'users', pipeline: [{$merge: {into: 'users'}}], cursor: {}})", + ] { + assert_eq!(script_access(text), StatementAccess::Destructive, "{text}"); + } + for text in [ + "db.users.aggregate([{$merge: {into: 'users_copy', whenMatched: 'replace'}}])", + "db.users.aggregate([{$merge: 'Users'}])", + "db.getSiblingDB('shop').users.aggregate([{$merge: {into: {db: 'archive', coll: 'users'}}}])", + "db.runCommand({aggregate: 'users', pipeline: [{$merge: 'totals'}], cursor: {}})", + "db.adminCommand({aggregate: 'users', pipeline: [{$merge: {into: {db: 'shop', coll: 'users'}}}], cursor: {}})", + "db.aggregate([{$documents: [{}]}, {$merge: 'users'}])", + "db.runCommand({explain: {aggregate: 'users', pipeline: [{$merge: 'users'}]}})", + ] { + assert_eq!(script_access(text), StatementAccess::Write, "{text}"); + } + } + #[test] fn an_aggregate_that_replaces_a_collection_is_destructive() { for text in [ diff --git a/core/crates/core/src/sql_syntax/mongo/shell_helpers.rs b/core/crates/core/src/sql_syntax/mongo/shell_helpers.rs index 5f4543396a..0a8abc6c20 100644 --- a/core/crates/core/src/sql_syntax/mongo/shell_helpers.rs +++ b/core/crates/core/src/sql_syntax/mongo/shell_helpers.rs @@ -37,6 +37,10 @@ pub(crate) fn check_helper(kind: ShellConstructor, args: &[ShellValue], at: usiz matches!(args, [String(_)] | [String(_), String(_)]), "a pattern and flags in quotes", ), + ShellConstructor::BsonRegExp => ( + matches!(args, [String(_)] | [String(_), String(_)]), + "a pattern and options in quotes", + ), ShellConstructor::Code => (matches!(args, [String(_)]), "JavaScript in quotes"), }; match accepted { diff --git a/core/crates/core/src/sql_syntax/mongo/shell_value.rs b/core/crates/core/src/sql_syntax/mongo/shell_value.rs index b079049f8a..8b64285972 100644 --- a/core/crates/core/src/sql_syntax/mongo/shell_value.rs +++ b/core/crates/core/src/sql_syntax/mongo/shell_value.rs @@ -161,10 +161,11 @@ impl ShellNumber { "Infinity" | "+Infinity" => Some(f64::INFINITY), "-Infinity" => Some(f64::NEG_INFINITY), "NaN" | "-NaN" | "+NaN" => Some(f64::NAN), - text => self - .as_integer() - .map(|integer| integer as f64) - .or_else(|| text.parse().ok()), + text => match self.as_integer() { + Some(0) if text.starts_with('-') => Some(-0.0), + Some(integer) => Some(integer as f64), + None => text.parse().ok(), + }, } } } @@ -185,6 +186,9 @@ pub enum ShellConstructor { MinKey, MaxKey, RegExp, + /// A BSON regular expression as it is stored, which mongosh keeps + /// whole where a JavaScript `RegExp` drops or refuses some options. + BsonRegExp, Code, } @@ -206,6 +210,7 @@ impl ShellConstructor { "MinKey" => Self::MinKey, "MaxKey" => Self::MaxKey, "RegExp" => Self::RegExp, + "BSONRegExp" => Self::BsonRegExp, "Code" => Self::Code, _ => return None, }) @@ -227,6 +232,7 @@ impl ShellConstructor { Self::MinKey => "MinKey", Self::MaxKey => "MaxKey", Self::RegExp => "RegExp", + Self::BsonRegExp => "BSONRegExp", Self::Code => "Code", } } @@ -247,6 +253,12 @@ mod tests { assert_eq!(ShellNumber::new("99999999999999999999").as_integer(), None); assert_eq!(ShellNumber::new("1e3").as_f64(), Some(1000.0)); assert!(ShellNumber::new("NaN").as_f64().is_some_and(f64::is_nan)); + assert!( + ShellNumber::new("-0") + .as_f64() + .is_some_and(|zero| zero == 0.0 && zero.is_sign_negative()) + ); + assert!(ShellNumber::new("0").as_f64().is_some_and(f64::is_sign_positive)); } #[test] @@ -273,6 +285,11 @@ mod tests { assert_eq!(ShellConstructor::named("ISODate"), Some(ShellConstructor::Date)); assert_eq!(ShellConstructor::named("NumberLong"), Some(ShellConstructor::Int64)); assert_eq!(ShellConstructor::named("ObjectID"), Some(ShellConstructor::ObjectId)); + assert_eq!( + ShellConstructor::named("BSONRegExp"), + Some(ShellConstructor::BsonRegExp) + ); + assert_eq!(ShellConstructor::BsonRegExp.name(), "BSONRegExp"); assert_eq!(ShellConstructor::named("eval"), None); assert_eq!(ShellConstructor::Decimal128.name(), "NumberDecimal"); } diff --git a/core/crates/core/tests/script_corpus.rs b/core/crates/core/tests/script_corpus.rs index d938bd78aa..9ed007bfba 100644 --- a/core/crates/core/tests/script_corpus.rs +++ b/core/crates/core/tests/script_corpus.rs @@ -927,6 +927,23 @@ fn mongo_what_replaces_a_collection_is_destructive() { } } +#[test] +fn mongo_a_merge_into_its_own_collection_is_destructive() { + for text in [ + "db.users.aggregate([{$match: {}}, {$merge: {into: 'users', whenMatched: 'replace'}}])", + "db.users.aggregate([{$merge: 'users'}])", + "db.getSiblingDB('shop').users.aggregate([{$merge: {into: {db: 'shop', coll: 'users'}}}])", + ] { + assert_eq!(mongo_access(text), StatementAccess::Destructive, "{text}"); + } + for text in [ + "db.users.aggregate([{$merge: {into: 'users_archive', whenMatched: 'replace'}}])", + "db.getSiblingDB('shop').users.aggregate([{$merge: {into: {db: 'archive', coll: 'users'}}}])", + ] { + assert_eq!(mongo_access(text), StatementAccess::Write, "{text}"); + } +} + #[test] fn mongo_a_bulk_write_that_reaches_every_document_is_destructive() { for ops in [ diff --git a/core/crates/drivers/mongodb/src/call.rs b/core/crates/drivers/mongodb/src/call.rs index 0ad58d12f0..f82237a312 100644 --- a/core/crates/drivers/mongodb/src/call.rs +++ b/core/crates/drivers/mongodb/src/call.rs @@ -138,6 +138,17 @@ impl ReadStop { false => Ok(()), } } + + /// Once the call is stopped, send no new command: no stop reaches one + /// sent after it. The read still waits for the stop's answer, so the + /// session goes back to the pool and the link reads as live. + pub(crate) async fn before_command(&self) -> Result<(), DriverError> { + if !self.call.is_requested() { + return Ok(()); + } + self.kill.answered.cancelled().await; + Err(stopped(&self.call)) + } } fn refused_by_server(error: &mongodb::error::Error) -> bool { @@ -330,6 +341,21 @@ mod tests { ); } + #[tokio::test(start_paused = true)] + async fn a_stopped_call_sends_no_new_command_once_the_stop_is_answered() { + let (kill, call) = (SessionKill::new(), StopRequest::new()); + let read = kill.follow(&call); + assert_eq!(read.before_command().await, Ok(())); + + call.request(StopReason::Cancelled); + let waited = tokio::time::timeout(Duration::from_secs(1), read.before_command()).await; + assert!(waited.is_err(), "the command went out before the answer"); + + kill.answered.cancel(); + assert_eq!(read.before_more().await, Ok(()), "a killed cursor's own error ends it"); + assert_eq!(read.before_command().await, Err(DriverError::Cancelled)); + } + #[tokio::test(start_paused = true)] async fn a_stop_that_sends_nothing_lets_the_read_go_on() { let (kill, call) = (SessionKill::new(), StopRequest::new()); diff --git a/core/crates/drivers/mongodb/src/catalog.rs b/core/crates/drivers/mongodb/src/catalog.rs index 85ac67f322..4b1e80a7bf 100644 --- a/core/crates/drivers/mongodb/src/catalog.rs +++ b/core/crates/drivers/mongodb/src/catalog.rs @@ -5,9 +5,12 @@ use bson::spec::BinarySubtype; use bson::{Binary, Bson, Document, doc}; use mongodb::{Client, ClientSession}; use tablepro_core::column::ColumnKind; +use tablepro_core::document::NUMBER_TYPE; use tablepro_core::meta::{IndexInfo, IndexKey, ObjectInfo, ObjectKind, RowCountEstimate, SchemaInfo, TableEstimate}; use tablepro_core::{ColumnInfo, DriverError, LossPhase}; +use crate::call::ReadStop; +use crate::cursor_rows::Read; use crate::endpoint::MongoEndpoint; use crate::field_types::{field_column, type_alias}; use crate::shell_command::Plan; @@ -20,17 +23,20 @@ const CENSUS_DOCUMENTS: i32 = 1_000; /// Databases MongoDB keeps for itself. const SYSTEM_DATABASES: [&str; 3] = ["admin", "config", "local"]; -/// What one catalogue read needs: the client, the call's own session and -/// the server's time limit for it. +/// What one catalogue read needs: the client, the call's own session, the +/// server's time limit for it and the call's stop, which every command +/// and `getMore` checks first. pub(crate) struct Reader<'a> { pub(crate) endpoint: &'a MongoEndpoint, pub(crate) client: &'a Client, pub(crate) session: &'a mut ClientSession, pub(crate) max_time: Option, + pub(crate) stop: &'a ReadStop, } impl Reader<'_> { async fn command(&mut self, database: &str, mut command: Document) -> Result { + self.stop.before_command().await?; if let Some(millis) = self.max_time { command.insert("maxTimeMS", Bson::Int64(millis)); } @@ -47,6 +53,7 @@ impl Reader<'_> { } async fn documents(&mut self, database: &str, mut command: Document) -> Result, DriverError> { + self.stop.before_command().await?; if let Some(millis) = self.max_time { command.insert("maxTimeMS", Bson::Int64(millis)); } @@ -56,18 +63,16 @@ impl Reader<'_> { .run_cursor_command(command) .session(&mut *self.session) .await; - let mut cursor = match opened { + let cursor = match opened { Ok(cursor) => cursor, Err(error) => return Err(self.endpoint.explain(&error, LossPhase::Statement).await), }; + let mut read = Read::new(self.endpoint, cursor, &mut *self.session, self.stop); let mut documents = Vec::new(); - loop { - match cursor.advance(self.session).await { - Ok(true) => documents.push(crate::nested_document::current_document(&cursor)?), - Ok(false) => return Ok(documents), - Err(error) => return Err(self.endpoint.explain(&error, LossPhase::Statement).await), - } + while let Some(document) = read.next().await? { + documents.push(document); } + Ok(documents) } /// The databases the login may see, `default` among them even @@ -187,21 +192,39 @@ impl Reader<'_> { } } -/// Each collection's fields that its columns show as JSON, from the last -/// census of its fields, so a page or an export of it shows them as -/// Extended JSON straight from the BSON, which keeps an `int` apart from -/// a `long`. +/// Each collection's columns as this connection last read them, so a +/// page or an export of it shows the JSON ones as Extended JSON straight +/// from the BSON, which keeps an `int` apart from a `long`, and an export +/// keeps the tab's column order. #[derive(Debug, Default)] -pub(crate) struct JsonFields(Mutex>>); +pub(crate) struct KnownFields(Mutex>); + +/// A collection's columns, as the last census of its fields found them. +#[derive(Debug, Clone, Default)] +pub(crate) struct Fields { + /// Every column's field, in the columns' order. + pub(crate) order: Vec, + /// The fields shown as JSON. + pub(crate) json: BTreeSet, + /// The JSON fields of numbers of more than one type, `NUMBER_TYPE`. + pub(crate) numbers: BTreeSet, +} -impl JsonFields { - /// Keep the fields among a collection's columns that are JSON. - pub(crate) fn remember(&self, database: &str, collection: &str, columns: &[ColumnInfo]) -> BTreeSet { - let fields: BTreeSet = columns - .iter() - .filter(|column| matches!(column.column_type.kind, ColumnKind::Json { .. })) - .map(|column| column.name.clone()) - .collect(); +impl KnownFields { + pub(crate) fn remember(&self, database: &str, collection: &str, columns: &[ColumnInfo]) -> Fields { + let fields = Fields { + order: columns.iter().map(|column| column.name.clone()).collect(), + json: columns + .iter() + .filter(|column| matches!(column.column_type.kind, ColumnKind::Json { .. })) + .map(|column| column.name.clone()) + .collect(), + numbers: columns + .iter() + .filter(|column| column.column_type.name == NUMBER_TYPE) + .map(|column| column.name.clone()) + .collect(), + }; self.0 .lock() .unwrap_or_else(PoisonError::into_inner) @@ -209,13 +232,13 @@ impl JsonFields { fields } - /// The JSON fields of the collection `planned` reads. A connection - /// that has not read them, as a new one after a reconnect has not, - /// takes the census first in the same call, so it shows the fields the - /// columns it was handed show as JSON. - pub(crate) async fn of(&self, planned: &Plan, reader: &mut Reader<'_>) -> Result, DriverError> { + /// The columns of the collection `planned` reads. A connection that + /// has not read them, as a new one after a reconnect has not, takes the + /// census first in the same call, so it shows the fields the columns it + /// was handed show as JSON. + pub(crate) async fn of(&self, planned: &Plan, reader: &mut Reader<'_>) -> Result { let Some((database, collection)) = planned.collection() else { - return Ok(BTreeSet::new()); + return Ok(Fields::default()); }; let known = self .0 diff --git a/core/crates/drivers/mongodb/src/connection.rs b/core/crates/drivers/mongodb/src/connection.rs index 1207e95cfe..26b2f01cd6 100644 --- a/core/crates/drivers/mongodb/src/connection.rs +++ b/core/crates/drivers/mongodb/src/connection.rs @@ -26,7 +26,7 @@ use tokio::time::Instant; use tokio_util::sync::{CancellationToken, WaitForCancellationFutureOwned}; use crate::call::{ClientCall, KillState, ReadStop, SessionKill, generated, release, remaining_millis, session}; -use crate::catalog::{CollectionType, JsonFields, Reader}; +use crate::catalog::{CollectionType, KnownFields, Reader}; use crate::dialect::MongoDialect; use crate::editor_session::MongoEditorSession; use crate::endpoint::MongoEndpoint; @@ -62,7 +62,7 @@ pub(crate) struct MongoConnection { dialect: Arc, link: LinkWatch, closed: AtomicBool, - json_fields: Arc, + known_fields: Arc, } impl MongoConnection { @@ -85,7 +85,7 @@ impl MongoConnection { dialect: Arc::new(MongoDialect), link: LinkWatch::new(), closed: AtomicBool::new(false), - json_fields: Arc::new(JsonFields::default()), + known_fields: Arc::new(KnownFields::default()), } } @@ -321,7 +321,7 @@ impl Connection for MongoConnection { }) }) .await?; - self.json_fields.remember(&key.0, &key.1, &columns); + self.known_fields.remember(&key.0, &key.1, &columns); Ok(columns) } @@ -450,13 +450,14 @@ impl Connection for MongoConnection { options: CallOptions, ) -> Result { let mut planned = self.generated_read(statement)?; - let (json_fields, deadline) = (self.json_fields.clone(), options.deadline()); + let (known_fields, deadline) = (self.known_fields.clone(), options.deadline()); self.read(options, move |context, session| { Box::pin(async move { - let shown = json_fields.of(&planned, &mut reader(&context, session)).await?; - planned.show_as_json(&shown); + let known = known_fields.of(&planned, &mut reader(&context, session)).await?; + planned.show_as_json(&known); // After the census, which the page's deadline also bounds. planned.limit_time(remaining_millis(deadline)); + context.stop.before_command().await?; let source = start(&context.endpoint, &context.client, session, planned).await?; collect(&context.endpoint, source, session, limit, &context.stop).await }) @@ -487,7 +488,7 @@ impl Connection for MongoConnection { ) -> Result { self.ensure_open()?; let (read, endpoint, database) = (read.clone(), self.endpoint.clone(), self.database.clone()); - let (json_fields, max_time) = (self.json_fields.clone(), remaining_millis(options.deadline())); + let (known_fields, max_time) = (self.known_fields.clone(), remaining_millis(options.deadline())); let kill = SessionKill::new(); let armed = kill.clone(); let stop = kill.clone(); @@ -500,7 +501,7 @@ impl Connection for MongoConnection { move |call, request: StopRequest| async move { let scope = ExportScope { database: &database, - json_fields: &json_fields, + known_fields: &known_fields, max_time, }; let result = @@ -596,5 +597,6 @@ fn reader<'a>(context: &'a CallContext, session: &'a mut ClientSession) -> Reade client: &context.client, session, max_time: context.max_time, + stop: &context.stop, } } diff --git a/core/crates/drivers/mongodb/src/dialect.rs b/core/crates/drivers/mongodb/src/dialect.rs index 88fa529e5e..16fa411734 100644 --- a/core/crates/drivers/mongodb/src/dialect.rs +++ b/core/crates/drivers/mongodb/src/dialect.rs @@ -109,22 +109,32 @@ impl SqlDialect for MongoDialect { None } - /// A page reads a missing field as NULL, so a NULL cell is a field the - /// document leaves out, as `tablepro_core::document` inserts it. - fn insert_names_null_cells(&self) -> bool { - false + /// A saved document's field given back to the server is removed with + /// `$unset`, as a new document leaves it out. + fn update_sets_default(&self) -> bool { + true + } + + fn reads_missing_as_null(&self) -> bool { + true } } #[cfg(test)] mod tests { use tablepro_core::column::ResultColumn; + use tablepro_core::dml::build_update; + use tablepro_core::edit::CellInput; use tablepro_core::export::render_sql_insert; - use tablepro_core::meta::TableRef; + use tablepro_core::meta::{RowIdentity, TableRef}; use tablepro_core::sql_syntax::WithholdReason; + use tablepro_core::sql_syntax::mongo::{ShellStatement, parse_statement, parse_value}; use tablepro_core::statement::ParamType; + use tablepro_core::value::JsonText; use super::*; + use crate::extended_json::extended_json; + use crate::to_bson::to_bson; #[test] fn an_import_writes_a_document_a_step() { @@ -132,7 +142,7 @@ mod tests { } #[test] - fn an_exported_insert_leaves_a_null_field_out() { + fn an_exported_insert_writes_a_null_field_as_null() { let columns = ["_id", "name", "email"].map(|name| ResultColumn::new(name, ColumnType::unknown())); let rows = [vec![Value::Int(1), Value::Text("ann".to_owned()), Value::Null]]; @@ -147,25 +157,143 @@ mod tests { assert_eq!( sql, - "db.getSiblingDB(\"shop\").getCollection(\"users\").insertOne({\"_id\": 1, \"name\": \"ann\"});\n" + "db.getSiblingDB(\"shop\").getCollection(\"users\").insertOne({\"_id\": 1, \"name\": \"ann\", \"email\": null});\n" + ); + } + + #[test] + fn an_exported_json_cell_runs_back_as_the_bson_it_names() { + let cells = [ + r#"{"$numberLong": "7"}"#, + r#"{"$numberInt": "7"}"#, + r#"{"$numberDecimal": "9.990"}"#, + r#"{"$numberDouble": "NaN"}"#, + r#"{"$numberDouble": "-Infinity"}"#, + r#"{"$date": "2024-01-02T03:04:05.006Z"}"#, + r#"{"$date": {"$numberLong": "-315619200000"}}"#, + r#"{"$date": {"$numberLong": "253402300800000"}}"#, + r#"{"$oid": "5f1d7a0c9b1e8b2a3c4d5e6f"}"#, + r#"{"$binary": {"base64": "VQ6EAOKbQdSnFkRmVUQAAA==", "subType": "04"}}"#, + r#"{"$binary": {"base64": "aGkh", "subType": "00"}}"#, + r#"{"$binary": {"base64": "aGkh", "subType": "80"}}"#, + r#"{"$timestamp": {"t": 1700000000, "i": 3}}"#, + r#"{"$regularExpression": {"pattern": "^a\\.b", "options": "im"}}"#, + r#"{"$regularExpression": {"pattern": "a/b", "options": "s"}}"#, + r#"{"$regularExpression": {"pattern": "(?i)a\\Z", "options": "lsx"}}"#, + "2.0", + "-0.0", + "-2147483648.0", + "2147483648.0", + "0.5", + r#"{"$numberDouble": "1e300"}"#, + r#"{"$minKey": 1}"#, + r#"{"$maxKey": 1}"#, + r#"{"a": [{"$numberLong": "1"}, {"b": {"$oid": "5f1d7a0c9b1e8b2a3c4d5e6f"}}], "c": 1.5}"#, + r#"{"whole": 2.0, "zero": -0.0, "n": 2}"#, + ]; + let column = crate::field_types::field_column("v", &["long", "string"].into_iter().collect()); + let columns = [ResultColumn::new("v", column.column_type)]; + let rows: Vec> = cells + .iter() + .map(|cell| vec![Value::Json(JsonText::parse((*cell).to_owned()).expect("JSON"))]) + .collect(); + + let script = render_sql_insert(&MongoDialect, &TableRef::unqualified("t"), &columns, &[], &[], &rows); + + assert_eq!(script.lines().count(), cells.len()); + for (line, cell) in script.lines().zip(cells) { + let Ok(ShellStatement::Call(call)) = parse_statement(line.trim_end_matches(';')) else { + panic!("{line} does not parse"); + }; + let written = call + .operation() + .and_then(|insert| insert.arg(0)) + .and_then(|document| document.field("v")) + .expect("the field"); + let named = parse_value(cell).expect("the cell"); + + assert_eq!( + to_bson(written, &[]).map(|bson| extended_json(&bson)), + to_bson(&named, &[]).map(|bson| extended_json(&bson)), + "{line}" + ); + assert!(!written.keys().iter().any(|key| key.starts_with('$')), "{line}"); + } + } + + #[test] + fn a_whole_double_runs_back_as_a_double() { + let column = crate::field_types::field_column("price", &["double"].into_iter().collect()); + let columns = [ResultColumn::new("price", column.column_type)]; + let rows = [2.0, -0.0, 2.5].map(|number| vec![Value::Float64(number)]); + + let script = render_sql_insert(&MongoDialect, &TableRef::unqualified("t"), &columns, &[], &[], &rows); + + assert!( + script.contains("Double(2)") && script.contains("Double(-0)"), + "{script}" + ); + let read: Vec = script + .lines() + .map(|line| { + let Ok(ShellStatement::Call(call)) = parse_statement(line.trim_end_matches(';')) else { + panic!("{line} does not parse"); + }; + let written = call + .operation() + .and_then(|insert| insert.arg(0)) + .and_then(|document| document.field("price")) + .expect("the field"); + to_bson(written, &[]).map(|bson| extended_json(&bson)).expect("BSON") + }) + .collect(); + assert_eq!(read, ["2.0", "-0.0", "2.5"]); + } + + #[test] + fn a_saved_field_given_back_to_the_server_is_unset() { + let columns = [ + crate::field_types::field_column("_id", &["int"].into_iter().collect()), + crate::field_types::field_column("email", &["string"].into_iter().collect()), + ]; + let identity = RowIdentity::UniqueKey { + columns: vec!["_id".to_owned()], + }; + let update = |cell: CellInput| { + build_update( + &MongoDialect, + &TableRef::new(Some("shop".to_owned()), "users"), + &columns, + &identity, + &[Value::Int(1)], + &[(1, cell)], + ) + .expect("an update") + }; + + assert!(MongoDialect.update_sets_default()); + assert_eq!( + update(CellInput::Default).sql(), + r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $1}, {"$unset": {"email": ""}})"# + ); + assert_eq!( + update(CellInput::Value(Value::Null)).sql(), + r#"db.getSiblingDB("shop").getCollection("users").updateOne({"_id": $2}, {"$set": {"email": $1}})"# ); } #[test] - fn a_field_of_ints_and_doubles_keeps_the_comparisons() { + fn a_field_of_ints_and_doubles_keeps_the_comparisons_and_no_pattern() { let price = crate::field_types::field_column("price", &["int", "double"].into_iter().collect()); let qty = crate::field_types::field_column("qty", &["int"].into_iter().collect()); + let mixed = crate::field_types::field_column("code", &["int", "string"].into_iter().collect()); let offered = MongoDialect.filter_operators(&price); - for op in [ - FilterOp::Lt, - FilterOp::GtEq, - FilterOp::Between, - FilterOp::Contains, - FilterOp::IsNull, - ] { + for op in [FilterOp::Lt, FilterOp::GtEq, FilterOp::Between, FilterOp::IsNull] { assert!(offered.contains(&op), "{op:?} in {offered:?}"); } + assert!(!offered.contains(&FilterOp::Contains), "{offered:?}"); + assert!(MongoDialect.filter_operators(&mixed).contains(&FilterOp::Contains)); assert_eq!( MongoDialect.filter_operators(&qty), tablepro_core::filter::operators_for(&qty.column_type.kind) diff --git a/core/crates/drivers/mongodb/src/field_types.rs b/core/crates/drivers/mongodb/src/field_types.rs index d1551c8dc9..ceb074f23c 100644 --- a/core/crates/drivers/mongodb/src/field_types.rs +++ b/core/crates/drivers/mongodb/src/field_types.rs @@ -7,6 +7,7 @@ use tablepro_core::column::{ BoolStorage, ColumnDefault, ColumnKind, ColumnType, FloatWidth, IntegerKind, IntegerWidth, ReadForm, SqlExpression, TemporalZone, TextLength, }; +use tablepro_core::document::NUMBER_TYPE; /// The `$type` alias of a value, with a UUID told apart from other /// binary data since the grid shows and edits it as a UUID. @@ -41,7 +42,9 @@ pub(crate) fn type_alias(value: &Bson) -> &'static str { /// more than one type, numbers of two BSON types among them, or of /// documents, arrays or binary data other than UUIDs, is JSON, which /// writes each value as the type it is, so an edit leaves an `int` an -/// `int` beside a `long`. A null names no type: it fits any. +/// `int` beside a `long`. Numbers of more than one type are +/// `NUMBER_TYPE`, which an export writes as numbers. A null names no +/// type: it fits any. pub(crate) fn shared_type(aliases: &BTreeSet<&str>) -> &'static str { let typed: BTreeSet<&str> = aliases.iter().copied().filter(|alias| *alias != "null").collect(); match typed.iter().next() { @@ -60,6 +63,13 @@ pub(crate) fn shared_type(aliases: &BTreeSet<&str>) -> &'static str { "array" => "array", _ => "mixed", }, + Some(_) + if typed + .iter() + .all(|alias| ["int", "long", "double", "decimal"].contains(alias)) => + { + NUMBER_TYPE + } _ => "mixed", } } @@ -105,7 +115,10 @@ pub(crate) fn column_type(name: &str) -> ColumnType { /// A field as a column of the collection's grid. `_id` names the /// document: every document has one, and it is unique. A new document /// left without one gets an ObjectId, so a new row leaves it out unless -/// the user types one. +/// the user types one. Any other field a new row leaves out until a value +/// or NULL is typed, so its default is `$$REMOVE`, MongoDB's name for a +/// missing value. An insert leaves a defaulted field out, so neither +/// text is evaluated. pub(crate) fn field_column(name: &str, aliases: &BTreeSet<&str>) -> ColumnInfo { let id = name == "_id"; ColumnInfo { @@ -116,16 +129,18 @@ pub(crate) fn field_column(name: &str, aliases: &BTreeSet<&str>) -> ColumnInfo { is_auto_increment: false, is_generated: false, partition_scoped: false, - default: match id { - true => ColumnDefault::Expression(SqlExpression::from_catalog_text("ObjectId()")), - false => ColumnDefault::None, - }, + default: ColumnDefault::Expression(SqlExpression::from_catalog_text(match id { + true => "ObjectId()", + false => "$$REMOVE", + })), comment: None, } } #[cfg(test)] mod tests { + use tablepro_core::edit::{CellInput, new_draft_cell}; + use super::*; fn shared(aliases: &[&'static str]) -> &'static str { @@ -136,9 +151,10 @@ mod tests { fn a_field_takes_the_type_its_values_share() { assert_eq!(shared(&["int"]), "int"); assert_eq!(shared(&["long", "null"]), "long"); - assert_eq!(shared(&["int", "long"]), "mixed"); - assert_eq!(shared(&["int", "double", "null"]), "mixed"); - assert_eq!(shared(&["decimal", "int"]), "mixed"); + assert_eq!(shared(&["int", "long"]), NUMBER_TYPE); + assert_eq!(shared(&["int", "double", "null"]), NUMBER_TYPE); + assert_eq!(shared(&["decimal", "int", "long", "double"]), NUMBER_TYPE); + assert_eq!(shared(&["long", "string"]), "mixed"); assert_eq!(shared(&["uuid", "null"]), "uuid"); assert_eq!(shared(&["uuid", "binData"]), "mixed"); assert_eq!(shared(&["string", "null"]), "string"); @@ -146,6 +162,9 @@ mod tests { assert_eq!(shared(&["object"]), "object"); assert_eq!(shared(&["null"]), "mixed"); assert_eq!(shared(&["timestamp"]), "mixed"); + let numbers = column_type(NUMBER_TYPE); + assert_eq!(numbers.name, NUMBER_TYPE); + assert!(matches!(numbers.kind, ColumnKind::Json { .. })); } #[test] @@ -159,4 +178,21 @@ mod tests { ColumnKind::Json { .. } )); } + + #[test] + fn a_new_row_leaves_every_field_out_until_one_is_typed() { + let field = field_column("email", &["string"].into_iter().collect()); + + assert!(field.nullable && !field.primary_key); + assert!(field.default.needs_server_evaluation(), "an insert leaves it out"); + assert_eq!(new_draft_cell(&field), CellInput::Default); + assert!( + !field.is_assigned_on_insert(std::slice::from_ref(&field)), + "a copied row keeps the value" + ); + assert_eq!( + new_draft_cell(&field_column("_id", &["objectId"].into_iter().collect())), + CellInput::Default + ); + } } diff --git a/core/crates/drivers/mongodb/src/rows.rs b/core/crates/drivers/mongodb/src/rows.rs index 008022f7a9..d2706be01d 100644 --- a/core/crates/drivers/mongodb/src/rows.rs +++ b/core/crates/drivers/mongodb/src/rows.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use bson::{Bson, Document}; use mongodb::ClientSession; use tablepro_core::column::{ColumnType, ResultColumn}; +use tablepro_core::document::NUMBER_TYPE; use tablepro_core::value::Value; use tablepro_core::{DriverError, QueryResult, ResultEvent, ResultSink, RowLimit, ServerNotice, SinkFlow, Truncation}; @@ -29,6 +30,8 @@ pub(crate) struct Layout { names: Vec, /// Each column shown as Extended JSON whatever its value's type. json: Vec, + /// Each JSON column of numbers of more than one type. + numbers: Vec, /// Each column's types so far, for the type the column shows. types: Vec>, discovered: bool, @@ -44,6 +47,7 @@ impl Layout { Self { types: vec![BTreeSet::new(); named.len()], json: named.iter().map(|column| column.json).collect(), + numbers: named.iter().map(|column| column.numbers).collect(), names: named.iter().map(|column| column.name.clone()).collect(), keys: named.into_iter().map(|column| column.key).collect(), discovered: matches!(columns, Columns::Discovered), @@ -62,6 +66,7 @@ impl Layout { self.keys.push(key.clone()); self.names.push(key.clone()); self.json.push(false); + self.numbers.push(false); self.types.push(BTreeSet::new()); outside = true; self.keys.len() - 1 @@ -93,10 +98,11 @@ impl Layout { self.names .iter() .zip(&self.types) - .zip(&self.json) - .map(|((name, types), json)| match json { - true => ResultColumn::new(name.clone(), column_type("object")), - false => ResultColumn::new(name.clone(), column_kind(types)), + .zip(self.json.iter().zip(&self.numbers)) + .map(|((name, types), shown)| match shown { + (true, true) => ResultColumn::new(name.clone(), column_type(NUMBER_TYPE)), + (true, false) => ResultColumn::new(name.clone(), column_type("object")), + (false, _) => ResultColumn::new(name.clone(), column_kind(types)), }) .collect() } @@ -301,18 +307,23 @@ mod tests { fn a_named_layout_keeps_its_columns() { let aliased = NamedColumn { key: "__field_2".to_owned(), - name: "a.b".to_owned(), - json: false, + ..NamedColumn::field("a.b") }; let nested = NamedColumn { json: true, ..NamedColumn::field("n") }; + let numbers = NamedColumn { + json: true, + numbers: true, + ..NamedColumn::field("q") + }; let mut layout = Layout::new(&Columns::Named(vec![ NamedColumn::field("b"), NamedColumn::field("a"), aliased, nested, + numbers, ])); assert!(layout.learn(&doc! { "a": 1_i32, "c": 2_i32 }), "c is outside"); @@ -323,10 +334,14 @@ mod tests { Value::Null, Value::Int(1), Value::Int(3), - Value::Json(tablepro_core::value::JsonText::parse(r#"{"$numberLong": "4"}"#.to_owned()).expect("JSON")) + Value::Json(tablepro_core::value::JsonText::parse(r#"{"$numberLong": "4"}"#.to_owned()).expect("JSON")), + Value::Null, ] ); - let names: Vec = layout.columns().iter().map(|column| column.name.clone()).collect(); - assert_eq!(names, ["b", "a", "a.b", "n"]); + let columns = layout.columns(); + let names: Vec<&str> = columns.iter().map(|column| column.name.as_str()).collect(); + assert_eq!(names, ["b", "a", "a.b", "n", "q"]); + let types: Vec<&str> = columns.iter().map(|column| column.column_type.name.as_str()).collect(); + assert_eq!(types[3..], ["object", NUMBER_TYPE]); } } diff --git a/core/crates/drivers/mongodb/src/shell_command.rs b/core/crates/drivers/mongodb/src/shell_command.rs index 0a577f78fe..39412a482e 100644 --- a/core/crates/drivers/mongodb/src/shell_command.rs +++ b/core/crates/drivers/mongodb/src/shell_command.rs @@ -1,10 +1,11 @@ -use std::collections::BTreeSet; - use bson::{Bson, Document, doc}; -use tablepro_core::sql_syntax::mongo::{CallRoot, ShellCall, ShellMethod, ShellStatement, ShellValue, ShowTarget}; +use tablepro_core::sql_syntax::mongo::{ + CallRoot, ShellCall, ShellMethod, ShellStatement, ShellValue, ShowTarget, is_addressable_field, +}; use tablepro_core::statement::BoundParam; use tablepro_core::{DriverError, UnsupportedFeature}; +use crate::catalog::Fields; use crate::errors::value_error; use crate::to_bson::{ValueError, to_bson}; @@ -31,6 +32,9 @@ pub(crate) struct NamedColumn { /// shows a field of nested or mixed values, so a `long` stays one /// through an edit. pub(crate) json: bool, + /// A JSON column of numbers of more than one BSON type, typed + /// `NUMBER_TYPE`, which an export writes as numbers. + pub(crate) numbers: bool, } impl NamedColumn { @@ -40,6 +44,7 @@ impl NamedColumn { key: name.clone(), name, json: false, + numbers: false, } } } @@ -107,15 +112,18 @@ impl Plan { Some((database, collection)) } - /// Show `fields` as Extended JSON. - pub(crate) fn show_as_json(&mut self, fields: &BTreeSet) { + /// Show the fields `known` shows as JSON as Extended JSON, typed as + /// it types them. A column already JSON, as an export's census over + /// its documents finds one, keeps the census's type. + pub(crate) fn show_as_json(&mut self, known: &Fields) { if let Self::Cursor { columns: Columns::Named(columns), .. } = self { - for column in columns { - column.json |= fields.contains(&column.name); + for column in columns.iter_mut().filter(|column| !column.json) { + column.json = known.json.contains(&column.name); + column.numbers = known.numbers.contains(&column.name); } } } @@ -139,10 +147,13 @@ impl Plan { } } - /// The aggregate with `stages` after its pipeline, its columns found - /// from what they return. `None` for a cursor that is no aggregate, or - /// one whose pipeline ends in a write, which no stage may follow. - pub(crate) fn followed_by(&self, stages: impl IntoIterator) -> Option { + /// The aggregate with the tally `stages` after its pipeline, its + /// columns found from what they return. A tally reads documents in any + /// order, so a `$sort` that no stage after it reads the order of is left + /// out, rather than sort the whole collection for it. `None` for a + /// cursor that is no aggregate, or one whose pipeline ends in a write, + /// which no stage may follow. + pub(crate) fn tallied_by(&self, stages: impl IntoIterator) -> Option { let Self::Cursor { database, command, @@ -155,10 +166,11 @@ impl Plan { if !command.contains_key("aggregate") { return None; } - let mut pipeline = command.get_array("pipeline").ok()?.clone(); - if ends_in_write(&pipeline) { + let pipeline = command.get_array("pipeline").ok()?; + if ends_in_write(pipeline) { return None; } + let mut pipeline = unsorted(pipeline); pipeline.extend(stages.into_iter().map(Bson::Document)); let mut command = command.clone(); command.insert("pipeline", pipeline); @@ -170,6 +182,26 @@ impl Plan { }) } + /// Fix `named` as the columns of an aggregate, and end its pipeline + /// with a `$project` of exactly their fields, so a field none of them + /// names, such as one another client adds after the census, is left + /// out. A field whose name has a dot or starts with `$` is read with + /// `$getField` under an alias no column's name starts with, since a + /// projection key would read it as a path. With no columns there is no + /// projection to write, so they are only named. + pub(crate) fn project_columns(&mut self, mut named: Vec) { + let Self::Cursor { command, columns, .. } = self else { + return; + }; + if let Ok(pipeline) = command.get_array_mut("pipeline") + && !named.is_empty() + && !ends_in_write(pipeline) + { + pipeline.push(Bson::Document(doc! { "$project": exact_projection(&mut named) })); + } + *columns = Columns::Named(named); + } + /// Bound the command by the server's own time limit, which stops it /// there when the deadline passes. pub(crate) fn limit_time(&mut self, millis: Option) { @@ -602,6 +634,72 @@ fn ends_in_write(pipeline: &[Bson]) -> bool { .is_some_and(|stage| stage == "$out" || stage == "$merge") } +/// Stages that pass each document on by itself, so the order documents +/// reach them in changes nothing about what they pass on. +const PER_DOCUMENT_STAGES: [&str; 9] = [ + "$addFields", + "$match", + "$project", + "$replaceRoot", + "$replaceWith", + "$set", + "$sort", + "$unset", + "$unwind", +]; + +/// `pipeline` without each `$sort` that only per-document stages follow. +/// A `$sort` before a `$limit`, a `$skip` or a `$group` decides which +/// documents or values pass, so it stays. +fn unsorted(pipeline: &[Bson]) -> Vec { + let mut order_unread = true; + let mut kept = Vec::with_capacity(pipeline.len()); + for stage in pipeline.iter().rev() { + let name = stage + .as_document() + .and_then(|stage| stage.keys().next()) + .map(String::as_str); + if order_unread && name == Some("$sort") { + continue; + } + order_unread &= name.is_some_and(|name| PER_DOCUMENT_STAGES.contains(&name)); + kept.push(stage.clone()); + } + kept.reverse(); + kept +} + +/// A `$project` of exactly `columns`' fields, each read under its +/// column's key, which this sets: the field's own name where a projection +/// can write it, else an alias. `_id`, which a projection keeps unless +/// told not to, is left out when no column names it. +fn exact_projection(columns: &mut [NamedColumn]) -> Document { + let mut prefix = String::from("__field_"); + while columns.iter().any(|column| column.name.starts_with(&prefix)) { + prefix.insert(0, '_'); + } + let mut projection = Document::new(); + if !columns.iter().any(|column| column.name == "_id") { + projection.insert("_id", 0_i32); + } + for (index, column) in columns.iter_mut().enumerate() { + match is_addressable_field(&column.name) { + true => { + column.key = column.name.clone(); + projection.insert(column.name.clone(), 1_i32); + } + false => { + column.key = format!("{prefix}{index}"); + projection.insert( + column.key.clone(), + doc! { "$getField": { "field": { "$literal": column.name.clone() }, "input": "$$ROOT" } }, + ); + } + } + } + projection +} + #[cfg(test)] mod tests { use tablepro_core::sql_syntax::mongo::parse_statement; @@ -674,12 +772,102 @@ mod tests { assert_eq!(collection("db.a.aggregate([]).itcount()"), None); } + fn stage_names(command: &Document) -> Vec { + command + .get_array("pipeline") + .expect("a pipeline") + .iter() + .filter_map(|stage| stage.as_document()?.keys().next().cloned()) + .collect() + } + + #[test] + fn a_tally_leaves_out_each_sort_that_decides_nothing_it_counts() { + let tallied = |text: &str| { + let Some(Plan::Cursor { command, .. }) = planned(text) + .ok() + .and_then(|plan| plan.tallied_by([doc! { "$count": "n" }])) + else { + panic!("not an aggregate: {text}"); + }; + stage_names(&command) + }; + + assert_eq!( + tallied("db.a.aggregate([{$match: {x: 1}}, {$sort: {b: 1, _id: 1}}], {allowDiskUse: true})"), + ["$match", "$count"] + ); + assert_eq!( + tallied("db.a.aggregate([{$sort: {b: 1}}, {$match: {x: 1}}, {$sort: {_id: -1}}, {$project: {b: 1}}])"), + ["$match", "$project", "$count"] + ); + assert_eq!( + tallied("db.a.aggregate([{$sort: {b: -1}}, {$limit: 10}, {$sort: {_id: 1}}])"), + ["$sort", "$limit", "$count"], + "the first sort picks the ten" + ); + assert_eq!( + tallied("db.a.aggregate([{$sort: {at: 1}}, {$group: {_id: '$u', last: {$last: '$v'}}}])"), + ["$sort", "$group", "$count"] + ); + } + + #[test] + fn an_export_projects_exactly_the_columns_its_census_found() { + let Ok(mut plan) = planned("db.a.aggregate([{$match: {x: 1}}])") else { + panic!("not a plan"); + }; + let mut nested = NamedColumn::field("n"); + nested.json = true; + plan.project_columns(vec![ + NamedColumn::field("_id"), + NamedColumn::field("a.b"), + nested, + NamedColumn::field("__field_9"), + ]); + + let Plan::Cursor { + command, + columns: Columns::Named(columns), + .. + } = &plan + else { + panic!("no named columns"); + }; + let keys: Vec<&str> = columns.iter().map(|column| column.key.as_str()).collect(); + assert_eq!(keys, ["_id", "___field_1", "n", "__field_9"]); + assert!(columns[2].json, "a JSON column stays one"); + let pipeline = command.get_array("pipeline").expect("a pipeline"); + assert_eq!( + pipeline.last().and_then(Bson::as_document), + Some(&doc! { "$project": { + "_id": 1_i32, + "___field_1": { "$getField": { "field": { "$literal": "a.b" }, "input": "$$ROOT" } }, + "n": 1_i32, + "__field_9": 1_i32, + } }) + ); + + let Ok(mut idless) = planned("db.a.aggregate([{$project: {_id: 0, a: 1}}, {$match: {}}])") else { + panic!("not a plan"); + }; + idless.project_columns(vec![NamedColumn::field("a")]); + let Plan::Cursor { command, .. } = &idless else { + panic!("not a cursor"); + }; + assert_eq!( + command.get_array("pipeline").ok().and_then(|pipeline| pipeline.last()), + Some(&Bson::Document(doc! { "$project": { "_id": 0_i32, "a": 1_i32 } })), + "_id is left out where no column names it" + ); + } + #[test] fn stages_follow_an_aggregate_and_keep_its_options() { let followed = |text: &str| { planned(text) .ok() - .and_then(|plan| plan.followed_by([doc! { "$count": "n" }])) + .and_then(|plan| plan.tallied_by([doc! { "$count": "n" }])) }; let Some(Plan::Cursor { command, columns, .. }) = diff --git a/core/crates/drivers/mongodb/src/shell_find.rs b/core/crates/drivers/mongodb/src/shell_find.rs index 12fa660266..038e8939cc 100644 --- a/core/crates/drivers/mongodb/src/shell_find.rs +++ b/core/crates/drivers/mongodb/src/shell_find.rs @@ -146,8 +146,7 @@ pub(crate) fn projected_columns(projection: &Document) -> Columns { let column = match aliased_field(value) { Some(name) => NamedColumn { key: key.clone(), - name, - json: false, + ..NamedColumn::field(name) }, None => NamedColumn::field(key.split('.').next().unwrap_or(key)), }; diff --git a/core/crates/drivers/mongodb/src/stream_read.rs b/core/crates/drivers/mongodb/src/stream_read.rs index 0f84914dd2..063e0ad1ed 100644 --- a/core/crates/drivers/mongodb/src/stream_read.rs +++ b/core/crates/drivers/mongodb/src/stream_read.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use bson::Document; use mongodb::{Client, ClientSession}; use tablepro_core::column::ColumnKind; +use tablepro_core::document::NUMBER_TYPE; use tablepro_core::sql_syntax::StatementAccess; use tablepro_core::sql_syntax::mongo::{ShellStatement, access_of, parse_statement, statement_ranges}; use tablepro_core::{ @@ -13,7 +14,7 @@ use tablepro_core::{ }; use crate::call::{ReadStop, SessionKill, generated, session}; -use crate::catalog::{JsonFields, Reader, census_stages, columns_from}; +use crate::catalog::{KnownFields, Reader, census_stages, columns_from}; use crate::cursor_rows::Read; use crate::endpoint::{ClientUse, MongoEndpoint}; use crate::errors::syntax_error; @@ -22,11 +23,11 @@ use crate::rows::emit; use crate::shell_command::{NamedColumn, Plan, plan}; /// Where an export reads: the connection's database, and the -/// collections' JSON fields with the server's time limit for a census of -/// them. +/// collections' columns as it last read them, with the server's time +/// limit for a census of them. pub(crate) struct ExportScope<'a> { pub(crate) database: &'a str, - pub(crate) json_fields: &'a JsonFields, + pub(crate) known_fields: &'a KnownFields, pub(crate) max_time: Option, } @@ -85,39 +86,40 @@ async fn stream_in( let mut sets = 0; let mut current = scope.database.to_owned(); for planned in plans { - let (mut planned, from_tab) = match planned { - Planned::Ready(planned) => (planned, true), - Planned::Typed(statement) => (plan(&statement, ¤t, &[])?, false), - }; - if planned.discovers_columns() { - let mut columns = census(endpoint, client, session, &planned, stop).await?; - if columns.is_empty() - && from_tab - && let Some((database, collection)) = planned.collection() - { - // No document matched: the collection's columns still head the export. + let (mut planned, known) = match planned { + Planned::Ready(planned) => { let mut reader = Reader { endpoint, client, session: &mut *session, max_time: scope.max_time, + stop, + }; + let known = scope.known_fields.of(&planned, &mut reader).await?; + (planned, Some(known)) + } + Planned::Typed(statement) => (plan(&statement, ¤t, &[])?, None), + }; + if planned.discovers_columns() { + let (mut columns, tallied) = census(endpoint, client, session, &planned, stop).await?; + if let Some(known) = &known { + columns = match columns.is_empty() { + // No document matched: the collection's columns still head the export. + true => known.order.iter().map(NamedColumn::field).collect(), + false => in_tab_order(columns, &known.order), }; - columns = reader.fields(database, collection).await?.iter().map(named).collect(); } - planned.name_columns(columns); + match tallied { + true => planned.project_columns(columns), + false => planned.name_columns(columns), + } } // A column the tab shows as JSON stays JSON where the documents // the export matches hold one type. - if from_tab { - let mut reader = Reader { - endpoint, - client, - session: &mut *session, - max_time: scope.max_time, - }; - let shown = scope.json_fields.of(&planned, &mut reader).await?; - planned.show_as_json(&shown); + if let Some(known) = &known { + planned.show_as_json(known); } + stop.before_command().await?; let source = match start(endpoint, client, session, planned).await? { Source::Used(next) => { current = next; @@ -135,22 +137,23 @@ async fn stream_in( } /// The columns of what `planned` reads, found before its first row, as -/// an export writes its columns first. The server tallies an aggregate's -/// documents with the census a collection's columns come from, over every -/// document, so a field a late document brings is a column and a field -/// of mixed values is JSON. A `find` has no pipeline to tally in, so its -/// documents are read here, for the fields in the order they first show. +/// an export writes its columns first, and whether the server tallied +/// them. The server tallies an aggregate's documents with the census a +/// collection's columns come from, over every document, so a field a late +/// document brings is a column and a field of mixed values is JSON. A +/// `find` has no pipeline to tally in, so its documents are read here, for +/// the fields in the order they first show. async fn census( endpoint: &MongoEndpoint, client: &Client, session: &mut ClientSession, planned: &Plan, stop: &ReadStop, -) -> Result, DriverError> { - if let Some(tally) = planned.followed_by(census_stages()) { +) -> Result<(Vec, bool), DriverError> { + if let Some(tally) = planned.tallied_by(census_stages()) { let mut tallies = Vec::new(); each(endpoint, client, session, tally, stop, |tally| tallies.push(tally)).await?; - return Ok(columns_from(&tallies).iter().map(named).collect()); + return Ok((columns_from(&tallies).iter().map(named).collect(), true)); } let mut fields = Vec::new(); let mut seen = BTreeSet::new(); @@ -162,7 +165,20 @@ async fn census( } }) .await?; - Ok(fields) + Ok((fields, false)) +} + +/// The census's columns in the order the connection last read the +/// collection's, then the ones it did not read in the census's order, so +/// another client's documents do not move a column the tab shows. +fn in_tab_order(mut columns: Vec, order: &[String]) -> Vec { + columns.sort_by_key(|column| { + order + .iter() + .position(|name| *name == column.name) + .unwrap_or(order.len()) + }); + columns } /// Hand each document a plan's cursor returns to `take`. @@ -174,6 +190,7 @@ async fn each( stop: &ReadStop, mut take: impl FnMut(Document) + Send, ) -> Result<(), DriverError> { + stop.before_command().await?; let Source::Cursor { cursor, .. } = start(endpoint, client, session, planned).await? else { return Ok(()); }; @@ -188,6 +205,7 @@ async fn each( fn named(column: &ColumnInfo) -> NamedColumn { NamedColumn { json: matches!(column.column_type.kind, ColumnKind::Json { .. }), + numbers: column.column_type.name == NUMBER_TYPE, ..NamedColumn::field(column.name.clone()) } } @@ -225,3 +243,21 @@ fn plans(read: &ReadStatement, database: &str) -> Result, DriverErr } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn an_export_keeps_the_tabs_column_order_with_other_fields_after_it() { + let census = ["_id", "x", "a", "late", "z", "b"].map(NamedColumn::field).to_vec(); + let tab = ["_id", "z", "a", "gone", "b"].map(str::to_owned); + + let names: Vec = in_tab_order(census, &tab) + .into_iter() + .map(|column| column.name) + .collect(); + + assert_eq!(names, ["_id", "z", "a", "b", "x", "late"]); + } +} diff --git a/core/crates/drivers/mongodb/src/to_bson.rs b/core/crates/drivers/mongodb/src/to_bson.rs index aa490ffd85..1090ec311e 100644 --- a/core/crates/drivers/mongodb/src/to_bson.rs +++ b/core/crates/drivers/mongodb/src/to_bson.rs @@ -225,7 +225,7 @@ fn constructed(kind: ShellConstructor, args: &[ShellValue]) -> Converted { } ShellConstructor::MinKey => Bson::MinKey, ShellConstructor::MaxKey => Bson::MaxKey, - ShellConstructor::RegExp => regex( + ShellConstructor::RegExp | ShellConstructor::BsonRegExp => regex( text_arg(args, 0).unwrap_or_default(), text_arg(args, 1).unwrap_or_default(), )?, @@ -426,6 +426,9 @@ mod tests { assert_eq!(shell("NumberLong(7)"), Bson::Int64(7)); assert_eq!(shell("NumberInt('7')"), Bson::Int32(7)); assert_eq!(shell("NumberDecimal('1.50')").to_string(), "1.50"); + assert_eq!(shell("Double(2)"), Bson::Double(2.0)); + assert!(matches!(shell("Double(-0)"), Bson::Double(zero) if zero == 0.0 && zero.is_sign_negative())); + assert!(matches!(shell("{$numberDouble: '-0.0'}"), Bson::Double(zero) if zero.is_sign_negative())); } #[test] @@ -441,6 +444,11 @@ mod tests { shell("/a.b/xi"), shell("{$regularExpression: {pattern: 'a.b', options: 'ix'}}") ); + assert_eq!( + shell("BSONRegExp('(?i)a++b', 'xsl')"), + shell("{$regularExpression: {pattern: '(?i)a++b', options: 'lsx'}}") + ); + assert_eq!(shell("new BSONRegExp('a')"), shell("RegExp('a', '')")); assert_eq!(shell("Timestamp(5, 2)"), shell("{$timestamp: {t: 5, i: 2}}")); assert_eq!(shell("BinData(0, 'aGk=')"), shell("HexData(0, '6869')")); assert!( diff --git a/core/crates/drivers/mongodb/tests/integration/export.rs b/core/crates/drivers/mongodb/tests/integration/export.rs index 0a09b15208..32b57305cc 100644 --- a/core/crates/drivers/mongodb/tests/integration/export.rs +++ b/core/crates/drivers/mongodb/tests/integration/export.rs @@ -5,13 +5,17 @@ use std::sync::{Arc, Mutex, PoisonError}; use async_trait::async_trait; use tablepro_core::browse::{ExportFields, ExportRequest, build_export}; +use tablepro_core::document::NUMBER_TYPE; +use tablepro_core::export::{CsvOptions, render_csv}; use tablepro_core::filter::{Combinator, FilterOp, FilterRule, FilterSet, FilterValue}; use tablepro_core::meta::TableRef; -use tablepro_core::{Connection, ReadStatement, ResultEvent, ResultSink, SinkFlow, Value}; +use tablepro_core::{Connection, EditorSession, ReadStatement, ResultEvent, ResultSink, RunEnd, SinkFlow, Value}; -use crate::support::{CALL, DATABASE, column_names, connect, rows, start, unique, within}; +use crate::support::{CALL, DATABASE, column_names, connect, editor, rows, run, start, unique, within}; -struct Keep(Arc>>); +type Events = Arc>>; + +struct Keep(Events); #[async_trait] impl ResultSink for Keep { @@ -21,9 +25,42 @@ impl ResultSink for Keep { } } +/// Keeps what an export sends, and has another client run `late` once +/// the export has read its first rows, after its census. +struct WriteOnceStarted { + kept: Events, + writer: Arc, + late: Option, + wrote: Arc>>, +} + +#[async_trait] +impl ResultSink for WriteOnceStarted { + async fn accept(&mut self, event: ResultEvent) -> SinkFlow { + if matches!(event, ResultEvent::ResultSetStarted { .. }) + && let Some(late) = self.late.take() + { + let (summary, _) = run(self.writer.as_ref(), &late).await; + *self.wrote.lock().unwrap_or_else(PoisonError::into_inner) = Some(summary.end); + } + self.kept.lock().unwrap_or_else(PoisonError::into_inner).push(event); + SinkFlow::Continue + } +} + +fn taken(events: &Events) -> Vec { + std::mem::take(&mut *events.lock().unwrap_or_else(PoisonError::into_inner)) +} + /// Every event an export of `table` under `filter` sent, with the /// columns and identity the tab reads. async fn export(connection: &dyn Connection, table: &TableRef, filter: &FilterSet) -> Vec { + let kept = Arc::new(Mutex::new(Vec::new())); + export_into(connection, table, filter, Box::new(Keep(kept.clone()))).await; + taken(&kept) +} + +async fn export_into(connection: &dyn Connection, table: &TableRef, filter: &FilterSet, sink: Box) { let columns = connection.fetch_columns(table, within(CALL)).await.expect("the fields"); let identity = connection.row_identity(table, within(CALL)).await.expect("an identity"); let request = ExportRequest { @@ -35,16 +72,10 @@ async fn export(connection: &dyn Connection, table: &TableRef, filter: &FilterSe fields: ExportFields::Every, }; let statement = build_export(&*connection.dialect(), &request).expect("an export statement"); - let kept = Arc::new(Mutex::new(Vec::new())); connection - .stream_read( - &ReadStatement::Generated(statement), - Box::new(Keep(kept.clone())), - within(CALL), - ) + .stream_read(&ReadStatement::Generated(statement), sink, within(CALL)) .await .expect("the export"); - std::mem::take(&mut *kept.lock().unwrap_or_else(PoisonError::into_inner)) } fn assert_json(cell: &Value, expected: &str) { @@ -109,3 +140,100 @@ async fn a_table_export_writes_the_fields_and_types_past_the_first_thousand_docu "the collection's columns head an empty export" ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_field_another_client_adds_while_a_table_export_reads_is_left_out() { + let fixture = start().await; + let connection = connect(&fixture).await; + let collection = unique("live_export"); + // About 100 KB each, so the server's batches hold fewer documents than + // the export reads before its first rows, and a `getMore` follows them. + let filled = fixture + .shell(&format!( + "db.getSiblingDB('{DATABASE}').getCollection('{collection}').insertMany(\ + Array.from({{length: 400}}, (_, id) => ({{_id: id, a: id, pad: 'x'.repeat(100000)}}))).acknowledged" + )) + .await + .expect("mongosh ran"); + assert!(filled.succeeded(), "{}", filled.stderr_text()); + let table = TableRef::new(Some(DATABASE.to_owned()), collection.clone()); + let (kept, wrote) = (Arc::new(Mutex::new(Vec::new())), Arc::new(Mutex::new(None))); + let sink = WriteOnceStarted { + kept: kept.clone(), + writer: editor(connection.as_ref()).await, + late: Some(format!( + "db.getSiblingDB('{DATABASE}').getCollection('{collection}').insertOne({{_id: 400, a: 400, late: true}})" + )), + wrote: wrote.clone(), + }; + + export_into(connection.as_ref(), &table, &FilterSet::default(), Box::new(sink)).await; + + assert_eq!( + *wrote.lock().unwrap_or_else(PoisonError::into_inner), + Some(RunEnd::Completed) + ); + let events = taken(&kept); + assert_eq!(column_names(&events), ["_id", "a", "pad"]); + let written = rows(&events); + assert_eq!(written.len(), 401, "the export read the late document"); + assert_eq!(written[400], [Value::Int(400), Value::Int(400), Value::Null]); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_table_export_keeps_the_tabs_column_order() { + let fixture = start().await; + let connection = connect(&fixture).await; + let collection = unique("tab_order"); + // Past the tab's census, a document puts `a` where the others put `b`. + let filled = fixture + .shell(&format!( + "const target = db.getSiblingDB('{DATABASE}').getCollection('{collection}');\n\ + target.insertMany(Array.from({{length: 1000}}, (_, id) => ({{_id: id, b: id, a: id}})));\n\ + target.insertOne({{_id: 1000, a: 1000, c: 1}}).acknowledged" + )) + .await + .expect("mongosh ran"); + assert!(filled.succeeded(), "{}", filled.stderr_text()); + let table = TableRef::new(Some(DATABASE.to_owned()), collection); + + let events = export(connection.as_ref(), &table, &FilterSet::default()).await; + + assert_eq!(column_names(&events), ["_id", "b", "a", "c"]); + assert_eq!( + rows(&events)[1000], + [Value::Int(1000), Value::Null, Value::Int(1000), Value::Int(1)] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_field_of_numbers_of_several_types_exports_as_numbers() { + let fixture = start().await; + let connection = connect(&fixture).await; + let collection = unique("number_export"); + let filled = fixture + .shell(&format!( + "db.getSiblingDB('{DATABASE}').getCollection('{collection}').insertMany([\ + {{_id: 1, q: 1, m: 1}}, {{_id: 2, q: NumberLong('3000000000'), m: 'x'}}, {{_id: 3, q: 19.9}}]).acknowledged" + )) + .await + .expect("mongosh ran"); + assert!(filled.succeeded(), "{}", filled.stderr_text()); + let table = TableRef::new(Some(DATABASE.to_owned()), collection); + + let events = export(connection.as_ref(), &table, &FilterSet::default()).await; + + let Some(ResultEvent::ResultSetStarted { columns }) = events + .iter() + .find(|event| matches!(event, ResultEvent::ResultSetStarted { .. })) + else { + panic!("no result set in {events:?}"); + }; + let types: Vec<&str> = columns.iter().map(|column| column.column_type.name.as_str()).collect(); + assert_eq!(types, ["int", NUMBER_TYPE, "object"]); + let csv = render_csv(columns, &rows(&events), &CsvOptions::default()).expect("the CSV"); + assert_eq!(csv, "_id,q,m\n1,1,1\n2,3000000000,\"\"\"x\"\"\"\n3,19.9,\n"); +} diff --git a/core/crates/drivers/mongodb/tests/integration/replica_set.rs b/core/crates/drivers/mongodb/tests/integration/replica_set.rs index 7bfb860304..778913ddb0 100644 --- a/core/crates/drivers/mongodb/tests/integration/replica_set.rs +++ b/core/crates/drivers/mongodb/tests/integration/replica_set.rs @@ -14,7 +14,7 @@ use crate::support::{CALL, DATABASE, connect_to, direct, editor, first_page, pas const MEMBER: &str = "localhost.test.build.10gen.cc:27017"; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires docker"] +#[ignore = "requires docker, public DNS and host port 27017"] async fn a_replica_set_is_found_from_its_seeds_and_read_through_its_primary() { let fixture = MongoFixture::start_on_test_records() .await diff --git a/core/crates/drivers/mongodb/tests/integration/seed_list.rs b/core/crates/drivers/mongodb/tests/integration/seed_list.rs index 7ee5697748..3e5033b021 100644 --- a/core/crates/drivers/mongodb/tests/integration/seed_list.rs +++ b/core/crates/drivers/mongodb/tests/integration/seed_list.rs @@ -12,7 +12,7 @@ use crate::support::{CALL, connect_to, direct, editor, first_page, password, run const SEED_LIST: &str = "test5.test.build.10gen.cc"; #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[ignore = "requires docker"] +#[ignore = "requires docker, public DNS and host port 27017"] async fn a_seed_list_logs_in_where_its_txt_record_says_then_pages_and_saves() { let fixture = MongoFixture::start_on_test_records() .await diff --git a/core/crates/drivers/mongodb/tests/integration/statements.rs b/core/crates/drivers/mongodb/tests/integration/statements.rs index c7bf0246de..8e233a4711 100644 --- a/core/crates/drivers/mongodb/tests/integration/statements.rs +++ b/core/crates/drivers/mongodb/tests/integration/statements.rs @@ -9,6 +9,7 @@ use tablepro_core::browse::{PageRead, PageRequest, read_page}; use tablepro_core::column::ColumnKind; use tablepro_core::edit::CellInput; use tablepro_core::filter::FilterSet; +use tablepro_core::import::CsvImportOptions; use tablepro_core::meta::{ObjectKind, ObjectScope, PagePosition, RowIdentity, TableRef}; use tablepro_core::save::{ChangeSet, RowChange, RowUpdate, plan_save}; use tablepro_core::value::{JsonText, OtherValue, SqlDecimal, Temporal}; @@ -17,6 +18,7 @@ use tablepro_core::{ RunOptions, ServerCode, Truncation, Value, WorkspaceSettings, }; use tablepro_driver_mongodb::MongoDriver; +use tablepro_test_support::import_csv; use tokio_util::sync::CancellationToken; use crate::support::{ @@ -630,3 +632,43 @@ async fn page_with(connection: &dyn Connection, table: &TableRef, columns: &[Col .await .expect("a page") } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn an_imported_null_leaves_its_field_out_of_a_sparse_unique_index() { + let fixture = start().await; + let connection = connect(&fixture).await; + let collection = unique("sparse_import"); + let filled = fixture + .shell(&format!( + "const target = db.getSiblingDB('{DATABASE}').getCollection('{collection}');\n\ + target.createIndex({{email: 1}}, {{unique: true, sparse: true}});\n\ + target.insertOne({{_id: 0, email: 'ann@example.com'}}).acknowledged" + )) + .await + .expect("mongosh ran"); + assert!(filled.succeeded(), "{}", filled.stderr_text()); + let table = TableRef::new(Some(DATABASE.to_owned()), collection.clone()); + let options = CsvImportOptions { + null_marker: "NULL".to_owned(), + ..CsvImportOptions::default() + }; + + let report = import_csv( + connection.as_ref(), + &table, + b"_id,email\n1,NULL\n2,NULL\n", + &options, + within(CALL), + ) + .await; + + assert!(report.is_ok(), "{report:?}"); + let missing = fixture + .shell(&format!( + "db.getSiblingDB('{DATABASE}').getCollection('{collection}').countDocuments({{email: {{$exists: false}}}})" + )) + .await + .expect("mongosh ran"); + assert_eq!(missing.stdout_text().trim(), "2", "{}", missing.stderr_text()); +} diff --git a/linux/crates/app/src/ui/data_grid/edit/paste.rs b/linux/crates/app/src/ui/data_grid/edit/paste.rs index c9016072fe..55fab92b2c 100644 --- a/linux/crates/app/src/ui/data_grid/edit/paste.rs +++ b/linux/crates/app/src/ui/data_grid/edit/paste.rs @@ -2,7 +2,7 @@ use gtk4::prelude::*; use gtk4::{gdk, glib}; use libadwaita as adw; use libadwaita::prelude::*; -use tablepro_core::edit::{CellInput, EditParseError}; +use tablepro_core::edit::{CellInput, EditParseError, read_cell}; use tablepro_core::import::{CsvRowError, ImportError, PastedRows, PastedTable}; use tablepro_core::{ColumnInfo, Value}; @@ -160,7 +160,17 @@ impl EditController { } fn insert_pasted(&self, grid: &DataGrid, pasted: PastedRows) { - let PastedRows { rows, skipped } = pasted; + let PastedRows { mut rows, skipped } = pasted; + // A pasted NULL is a value read elsewhere, which a document engine reads for a field a row lacks. + if let (Some(dialect), Some(columns)) = (grid.dialect(), catalog_columns(grid)) { + for row in &mut rows { + for (cell, column) in row.iter_mut().zip(&columns) { + if matches!(cell, CellInput::Value(Value::Null)) { + *cell = read_cell(Value::Null, column, &*dialect); + } + } + } + } let count = rows.len(); if count == 0 { grid.toast(gettext("The clipboard has no rows to paste.")); diff --git a/linux/crates/app/src/ui/data_grid/edit/row_edits.rs b/linux/crates/app/src/ui/data_grid/edit/row_edits.rs index 62bebac19d..0a118bfa11 100644 --- a/linux/crates/app/src/ui/data_grid/edit/row_edits.rs +++ b/linux/crates/app/src/ui/data_grid/edit/row_edits.rs @@ -2,7 +2,7 @@ use gtk4::prelude::*; use gtk4::{self as gtk, glib}; use tablepro_core::Value; use tablepro_core::column::{BoolStorage, ColumnKind}; -use tablepro_core::edit::{CellInput, EditParseError, new_draft_cell, parse_edit_text}; +use tablepro_core::edit::{CellInput, EditParseError, new_draft_cell, parse_edit_text, read_cell}; use super::{EditController, edit_start}; use crate::i18n::{gettext, ngettext_f}; @@ -148,7 +148,8 @@ impl EditController { } /// Copies of the rows as the user sees them, pending edits included. - /// What the database owns (key, identity, generated) is left to it. + /// What the database owns (key, identity, generated) is left to it, + /// and so is a NULL read where it can stand for a missing value. pub(super) fn duplicate_rows(&self, grid: &DataGrid) { if self.policy().is_none_or(|policy| policy.duplicate.is_err()) { return; @@ -158,6 +159,7 @@ impl EditController { }; let columns = grid.grid_columns(); let tracked = self.tracked(); + let dialect = grid.dialect(); let copies = target .rows .iter() @@ -165,14 +167,24 @@ impl EditController { let source = tracked.inputs(row); columns .iter() - .map(|column| match &column.catalog { - Some(catalog) if catalog.primary_key || catalog.is_auto_increment || catalog.is_generated => { - new_draft_cell(catalog) - } - Some(_) | None => source + .map(|column| { + let input = source .get(column.index) .cloned() - .unwrap_or(CellInput::Value(Value::Null)), + .unwrap_or(CellInput::Value(Value::Null)); + match (&column.catalog, input, dialect.as_deref()) { + (Some(catalog), _, _) + if catalog.primary_key || catalog.is_auto_increment || catalog.is_generated => + { + new_draft_cell(catalog) + } + (Some(catalog), CellInput::Value(value), Some(dialect)) + if tracked.shows_read_value(row, column.index) => + { + read_cell(value, catalog, dialect) + } + (_, input, _) => input, + } }) .collect() }) diff --git a/linux/crates/app/src/ui/data_grid/mod.rs b/linux/crates/app/src/ui/data_grid/mod.rs index 849e5f37a9..e8c3c94ad7 100644 --- a/linux/crates/app/src/ui/data_grid/mod.rs +++ b/linux/crates/app/src/ui/data_grid/mod.rs @@ -142,10 +142,14 @@ impl DataGrid { } } + /// The connection's dialect, SQL or not. + pub(crate) fn dialect(&self) -> Option> { + (self.config()?.dialect.as_ref()?)() + } + /// The connection's dialect, where the rows have an SQL spelling. pub(crate) fn sql_dialect(&self) -> Option> { - let dialect = (self.config()?.dialect.as_ref()?)()?; - dialect.grammar().is_sql().then_some(dialect) + self.dialect().filter(|dialect| dialect.grammar().is_sql()) } pub fn connect_request(&self, f: F) -> glib::SignalHandlerId { diff --git a/linux/crates/app/src/ui/data_grid/tracked_values.rs b/linux/crates/app/src/ui/data_grid/tracked_values.rs index fbc83996d2..4ad39aec60 100644 --- a/linux/crates/app/src/ui/data_grid/tracked_values.rs +++ b/linux/crates/app/src/ui/data_grid/tracked_values.rs @@ -242,17 +242,23 @@ impl TrackedValues { RowSnapshot::new(cells, pending.into_iter().collect()) } - /// Like `snapshot`, except a draft's cell left to the server stays - /// `Default`. + /// Like `snapshot`, except a cell left to the server stays `Default`, + /// a draft's or a saved row's pending one. pub fn inputs(&self, row: &GridRow) -> Vec { if let (GridRowId::Draft(draft_id), Some(tracker)) = (row.id(), self.tracker.as_ref()) { return tracker.draft_inputs(draft_id).unwrap_or_default(); } - self.snapshot(row) - .to_values() - .into_iter() - .map(CellInput::Value) - .collect() + let width = row.cells().map_or(0, |cells| cells.len()); + (0..width).map(|column| self.edit_input(row, column)).collect() + } + + /// Whether the cell shows the value as it was read, with no edit over + /// it. + pub fn shows_read_value(&self, row: &GridRow, column: usize) -> bool { + !matches!(row.id(), GridRowId::Draft(_)) + && self + .tracked_key(row) + .is_none_or(|(tracker, key)| tracker.cell_overlay(&key, column).pending.is_none()) } } diff --git a/linux/docs/adding-drivers.md b/linux/docs/adding-drivers.md index 39ce3216b7..4f3079b717 100644 --- a/linux/docs/adding-drivers.md +++ b/linux/docs/adding-drivers.md @@ -314,7 +314,7 @@ A run streams `ResultEvent`s into a `ResultSink`: - `Notice` passes on a server notice or warning. - `NoticeLimitReached { retained }` comes once per run when the driver kept fewer notices than the server sent, after the `retained` ones it passed on. -`ResultSink::accept` is awaited before the next rows are read off the wire, so a grid that cannot keep up slows the read instead of growing a queue the size of the result. One exception: an editor result whose columns a document engine finds from its documents, since a later document can add a column. It reads the documents it keeps, at most the row limit and one more, before `ResultSetStarted`, then sends them under the same backpressure and closes the cursor. `SinkFlow::ConsumerGone` ends the run with `RunEnd::Cancelled`. PostgreSQL, MySQL, SQL Server and ClickHouse then stop the statement on the server, SQLite stops stepping it, DuckDB stops fetching its streamed result, and the Cassandra driver asks for no more pages. `stream_read` uses the same sink and events, without a row limit. +`ResultSink::accept` is awaited before the next rows are read off the wire, so a grid that cannot keep up slows the read instead of growing a queue the size of the result. One exception: an editor result whose columns a document engine finds from its documents, since a later document can add a column. It reads the documents it keeps, at most the row limit and one more, and closes the cursor before `ResultSetStarted`, then sends them under the same backpressure. `SinkFlow::ConsumerGone` ends the run with `RunEnd::Cancelled`. PostgreSQL, MySQL, SQL Server and ClickHouse then stop the statement on the server, SQLite stops stepping it, DuckDB stops fetching its streamed result, and the Cassandra driver asks for no more pages. `stream_read` uses the same sink and events, without a row limit. ## The row-limit rule @@ -332,7 +332,7 @@ A run streams `ResultEvent`s into a `ResultSink`: A table whose scan has no order to page by, as a CQL scan in token order has none, returns `TablePaging::ServerState` from `table_paging`: the browse tab then pages by the state the server hands back, through `Connection::query_server_page`, offers no sort, and reaches a page by number by skipping rows. `filter_operators`, `filter_combinators` and `filtered_read_suffix` narrow the filter strip to what the engine evaluates and add what it needs after a WHERE clause. `capabilities().insert_conflict` says what an INSERT of a taken key does, which decides how a save proves an insert: refused (`Refused`), skipped (`Skipped`, proved by the count), or replaced (`Replaced`, proved by a probe for the key first). A column that holds one value per partition, a CQL partition key or static column, is `ColumnInfo::partition_scoped`: an update of such columns alone names only the partition, is spelled by `partition_update_statement`, and is proved by a probe for its row. -`SqlDialect::cell_expressions` lists the SQL a date, time or timestamp cell may be set to, such as `NOW()`, each with the column kinds it fills and the other spellings typed for it. The grid matches what the user types against this list and the save writes the listed spelling into the statement, so only list an expression after running it against the engine for every kind it names; `check_cell_expressions` in `tablepro-test-support` does that from a driver's integration tests. `update_sets_default` says whether `UPDATE ... SET c = DEFAULT` parses, which lets a saved row's cell take the column's default. +`SqlDialect::cell_expressions` lists the SQL a date, time or timestamp cell may be set to, such as `NOW()`, each with the column kinds it fills and the other spellings typed for it. The grid matches what the user types against this list and the save writes the listed spelling into the statement, so only list an expression after running it against the engine for every kind it names; `check_cell_expressions` in `tablepro-test-support` does that from a driver's integration tests. `update_sets_default` says whether `UPDATE ... SET c = DEFAULT` parses, which lets a saved row's cell take the column's default. `reads_missing_as_null` says a read gives NULL for a value the row does not hold, as a document's missing field: a NULL Duplicate Rows copies or a CSV import's null marker names then takes the column's default (`edit::read_cell`), while NULL typed into a cell is written. `SqlDialect::grammar()` names the engine's `SqlGrammar`, which the editor's split and completion, filter and default validation, and the history check read. `SqlGrammar` is closed: an engine with its own lexical rules adds a variant, and the compiler lists every match to extend, starting with its rules in `core/crates/core/src/sql_syntax/script/script_rules.rs`. A language that is not SQL adds a variant too, with no SQL rules ([Engines without SQL](#engines-without-sql)). Each rule gets a named case in `core/crates/core/tests/script_corpus.rs` ([testing.md](testing.md#the-script-planner)). An engine with its own error numbering adds a `ServerCode` variant the same way ([error-handling.md](error-handling.md#serverdiagnostics)). diff --git a/linux/docs/drivers/mongodb.md b/linux/docs/drivers/mongodb.md index e6b72b5356..e8ac3aefeb 100644 --- a/linux/docs/drivers/mongodb.md +++ b/linux/docs/drivers/mongodb.md @@ -46,11 +46,11 @@ Databases are the sidebar's schemas, collections its tables and views its views. A collection's columns are its top-level fields, found by the server from the first 1,000 documents in their natural order: the fields, the BSON types each holds, and where each first appears. No document crosses the wire for it. `_id` comes first, then the fields in the order the documents place them. -A field's column type is its values' type: one numeric type is a number, a string is text, a date a timestamp with a zone, an `objectId` its own type, and UUIDs (binary subtype 4) a UUID. `$type` names every binary subtype `binData`, so the census tells a UUID by where it sorts: binary data orders by length, then subtype, then bytes. A field that holds documents, arrays, other binary data, numbers of more than one type (`int` and `long`, or `int` and `double`) or more than one kind of value is a JSON column, written as relaxed Extended JSON from the BSON itself, with each `long` in its canonical `{"$numberLong": "..."}` form, so the type survives an edit, a copy and an export. A field missing from a document reads as NULL. +A field's column type is its values' type: one numeric type is a number, a string is text, a date a timestamp with a zone, an `objectId` its own type, and UUIDs (binary subtype 4) a UUID. `$type` names every binary subtype `binData`, so the census tells a UUID by where it sorts: binary data orders by length, then subtype, then bytes. A field that holds documents, arrays, other binary data, numbers of more than one type (`int` and `long`, or `int` and `double`) or more than one kind of value is a JSON column, written as relaxed Extended JSON from the BSON itself, with each `long` in its canonical `{"$numberLong": "..."}` form, so the type survives an edit, a copy and an export. A field of numbers alone, of more than one type, is typed `number` (`tablepro_core::document::NUMBER_TYPE`), MongoDB's `$type` name for any number. A field missing from a document reads as NULL. A connection that has not read a collection's fields reads them before its first page or export of it, so a page after a reconnect shows the same JSON columns. -`_id` is the primary key and names a document for every save. A new row leaves `_id` out unless one is typed, and the document gets an ObjectId, as every driver gives it. A cell left NULL in a new row, a duplicated row or an imported one is a field left out, since a page reads a missing field as NULL: the document holds only the fields given a value, so a sparse or partial index and a validator's optional field see what `insertOne` in mongosh would give them. The statements an export or Copy As writes leave them out too, and an edit that sets a field to NULL removes it with `$unset`. To store an explicit `null`, type `null` in a JSON cell or write the statement in the editor. A view has no key, and neither has a time-series collection, whose `_id` the server does not keep unique: their pages are counted off in the order they give, and their rows read and never write. Indexes come from `listIndexes`, `_id`'s first. The row count is the collection's metadata count, shown as an estimate. +`_id` is the primary key and names a document for every save. A new row leaves `_id` out unless one is typed, and the document gets an ObjectId, as every driver gives it. A cell of a new row starts as DEFAULT, which leaves the field out of the document (the Structure tab names that default `$$REMOVE`, MongoDB's word for a missing value), so a sparse or partial index and a validator's optional field see what `insertOne` in mongosh would give them. NULL typed into a cell stores `null`, which a validator that requires a nullable field takes. On a saved row, Set to Default removes the field with `$unset`. A page reads a missing field as NULL (`SqlDialect::reads_missing_as_null`), so Duplicate Rows leaves out a field read as NULL, and a CSV import one its null marker names, where the column has a default; an export writes `null` for it. A view has no key, and neither has a time-series collection, whose `_id` the server does not keep unique: their pages are counted off in the order they give, and their rows read and never write. Indexes come from `listIndexes`, `_id`'s first. The row count is the collection's metadata count, shown as an estimate. ## Pages, filters and saves @@ -66,7 +66,7 @@ A collection opens in the table tab, and every statement the tab builds is a mon Every value is a bound parameter, never spliced into the text. A sort always ends with `_id`, so a page is stable, and paging past the first pages walks `_id` by key: across `_id`s of more than one type, in the order MongoDB compares types. -A filter rule becomes a `$match` condition: equality, comparisons, `$in`, a pattern as an anchored `$regex`, NULL as missing or null. The raw filter takes a query document, `{ qty: { $gt: 1 }, status: 'A' }`, checked before it is sent. A field whose name holds a `.` or starts with `$` is read through `$getField` and shown, but cannot be filtered, sorted or edited, since a query cannot name it. A JSON column filters with comparisons as well as patterns: MongoDB orders any two values, by type and then by value, and a filter value is the BSON its Extended JSON names, so `price > 10` works on a field of `int` and `double`. +A filter rule becomes a `$match` condition: equality, comparisons, `$in`, a pattern as an anchored `$regex`, NULL as missing or null. The raw filter takes a query document, `{ qty: { $gt: 1 }, status: 'A' }`, checked before it is sent. A field whose name holds a `.` or starts with `$` is read through `$getField` and shown, but cannot be filtered, sorted or edited, since a query cannot name it. A JSON column filters with comparisons as well as patterns: a filter value is the BSON its Extended JSON names, and the server compares it with the field's values of the same type, numbers of every type with each other, so `price > 10` works on a field of `int` and `double`. A `number` column only compares, since a `$regex` never matches a number. A JSON cell is edited as JSON text, refused when it does not parse, and saved as the BSON its Extended JSON names. @@ -80,13 +80,13 @@ The editor runs mongosh statements. A statement ends at `;` or at the end of a l - `db.` or `db.getCollection("")`, with `db.getSiblingDB("")` in front, and the collection methods: `find`, `findOne`, `aggregate`, `countDocuments`, `estimatedDocumentCount`, `count`, `distinct`, `getIndexes`, `stats` and its sizes, the `insert`, `update`, `replace`, `delete` and `findOneAnd...` families, `findAndModify`, `remove`, `createIndex(es)`, `dropIndex(es)`, `hideIndex`, `unhideIndex`, `renameCollection` and `drop`. - A `find` cursor's `sort`, `limit`, `skip`, `projection`, `hint`, `maxTimeMS`, `batchSize`, `comment`, `collation`, `allowDiskUse`, `toArray`, `pretty`, `count`, `explain` and the like, and an `aggregate` cursor's `explain`, `toArray` and `pretty`; any other cursor method is refused rather than left out. - The database methods: `runCommand`, `adminCommand`, `aggregate`, `getCollectionNames`, `getCollectionInfos`, `createCollection`, `createView`, `dropDatabase`, `stats`, `serverStatus`, `version`, `currentOp`, `killOp`, the user and role helpers, and `rs.status()`, `rs.conf()`. -- Values as mongosh writes them: unquoted keys, single quotes, regular expressions, `ObjectId`, `ISODate`, `Date`, `NumberInt`, `NumberLong`, `NumberDecimal`, `Timestamp`, `UUID`, `BinData`, `MinKey`, `MaxKey`. A whole number in the int32 range is an `int`, any other number a `double`, as mongosh stores them. +- Values as mongosh writes them: unquoted keys, single quotes, regular expressions, `ObjectId`, `ISODate`, `Date`, `NumberInt`, `NumberLong`, `NumberDecimal`, `Double`, `Timestamp`, `UUID`, `BinData`, `MinKey`, `MaxKey`, `RegExp`, `BSONRegExp`. A whole number in the int32 range is an `int`, any other number a `double`, as mongosh stores them. Anything else, a JavaScript function, a variable, `forEach`, `bulkWrite`, is refused before it is sent with a `SyntaxError` at its place, or `Unsupported`. As mongosh checks, `updateOne` and `updateMany` take update operators or a pipeline and `replaceOne` a document with none, so a forgotten `$set` never replaces a document, and `dropIndex()` has to name its index. A write's `writeConcern`, `ordered`, `comment` and `let` options go with its command. A statement's documents show one per row, a column per field: the fields a projection names, in its order, or every field the documents hold, in the order they first show it. A write shows what mongosh prints for it (`insertedId`, `matchedCount`, `modifiedCount`). The run's row limit keeps 10,000 documents, and the cursor reads one more to say the result was cut; closing it frees the rest on the server. When a projection names the columns, the documents go to the grid a batch at a time, each batch read once the grid took the last, and the columns' types are settled when the last one is read. Otherwise the documents kept are all read before the first row is shown, so the columns hold every field they have. A tab's `use` changes only its own database. -A table export reads every field the documents hold, not only the tab's columns, so a field none of the first 1,000 documents holds is still written. Before the first row, the server tallies the fields the export's documents hold and the types each holds, the same census the columns come from but over every matching document, and no document crosses the wire for it. A field of nested or mixed values is written as Extended JSON. An export that matches no document names the collection's columns. An export of an editor result whose columns are not named by a projection finds them the same way for an `aggregate`, and reads a `find`'s documents twice: once for the fields they hold, then to write them under those columns. A document that gains a field between the two reads fails the export rather than losing the field. The MCP server's `browse_table` and `export_data` read the tab's columns, since they stop at a row limit and a deadline. +A table export reads every field the documents hold, not only the tab's columns, so a field none of the first 1,000 documents holds is still written. Before the first row, the server tallies the fields the export's documents hold and the types each holds, the same census the columns come from but over every matching document, and no document crosses the wire for it. The columns keep the tab's order, with the other fields after them, and the read projects exactly those fields, so a field another client adds while the export runs is left out. A field of nested or mixed values is written as Extended JSON, and a `number` field's values as numbers in CSV, XLSX and JSON, a fraction with the digits it was read with; a JSON column of any other engine is written as it is. Statements and Copy As write Extended JSON as the shell's constructors (`NumberLong("7")`, `ISODate(...)`, `ObjectId(...)`), a regular expression as `BSONRegExp(...)`, which keeps every option, and a whole double in the int32 range as `Double(2)`, which mongosh would otherwise store as an `int`, so mongosh reads them back as the same types. An export that matches no document names the collection's columns. An export of an editor result whose columns are not named by a projection finds them the same way for an `aggregate`, and reads a `find`'s documents twice: once for the fields they hold, then to write them under those columns. A document that gains a field between the two reads of a `find` fails the export rather than losing the field. The MCP server's `browse_table` and `export_data` read the tab's columns, since they stop at a row limit and a deadline. A history entry is withheld when a statement carries a password: `createUser`, `updateUser`, `changeUserPassword`, `auth`, `connect` and `Mongo`, by name or in brackets (`db['changeUserPassword'](...)`), a `pwd`, `password` or `passwordDigestor` key, and a connection string with a password. Text that does not close is withheld too. The editor highlights JavaScript and has no SQL completion, formatter or comment toggle. @@ -102,7 +102,7 @@ A call that times out or loses its connection checks the server with the probe c MongoDB has no read-only session, so a read-only connection rests on the app's own check (`ReadOnlyEnforcement::ClientGuard`): each statement is read before it is sent, and one that is not a read is refused with `ReadOnlyRefusal::ClientGuard`, in the editor, in a save and in an export. A statement reads only when `sql_syntax::mongo` lists its method, or its command's name, as a read, with every cursor method after it a read too; an `aggregate` with `$out` or `$merge` at any depth writes, and so does an `explain` of a write, and a `validate` with `repair` or `fixMultikey`, which change indexes and documents. A method it does not know counts as a write. For a guarantee, log in as a user with only the `read` role. -Safe Mode reads statements the same way: Confirm Writes asks before every statement that is not a read, and Off asks before every drop, as it does before SQL's `DROP`: `drop`, `dropDatabase`, `dropIndex`, `dropIndexes`, `dropUser`, `dropRole`, `dropAllUsersFromDatabase`, `dropAllRolesFromDatabase`, a `renameCollection` that drops its target, and the same commands through `runCommand`. It asks before what replaces a collection: an `aggregate` with `$out`, a `mapReduce` whose `out` names a collection or `replace`, `convertToCapped` and `applyOps`. It also asks before a `deleteMany`, `remove` or `updateMany` with no filter, `null` or `{}`, an `update` with `multi` and such a filter, and a `delete`, `update` or `bulkWrite` command through `runCommand` that reaches every document the same way. `aggregate()` takes only its own options, so none can stand in for the pipeline that was checked, and a key written twice is read as its last value, as the shell sends it. +Safe Mode reads statements the same way: Confirm Writes asks before every statement that is not a read, and Off asks before every drop, as it does before SQL's `DROP`: `drop`, `dropDatabase`, `dropIndex`, `dropIndexes`, `dropUser`, `dropRole`, `dropAllUsersFromDatabase`, `dropAllRolesFromDatabase`, a `renameCollection` that drops its target, and the same commands through `runCommand`. It asks before what replaces a collection: an `aggregate` with `$out` or with a `$merge` into the collection it reads, a `mapReduce` whose `out` names a collection or `replace`, `convertToCapped` and `applyOps`. It also asks before a `deleteMany`, `remove` or `updateMany` with no filter, `null` or `{}`, an `update` with `multi` and such a filter, and a `delete`, `update` or `bulkWrite` command through `runCommand` that reaches every document the same way. `aggregate()` takes only its own options, so none can stand in for the pipeline that was checked, and a key written twice is read as its last value, as the shell sends it. ## Trust diff --git a/linux/docs/testing.md b/linux/docs/testing.md index 6b81f84ac5..cc6a4e3cb4 100644 --- a/linux/docs/testing.md +++ b/linux/docs/testing.md @@ -325,6 +325,8 @@ meson and distro builds use cargo test, which works too: cargo test -p tablepro-driver-postgres -- --ignored --test-threads=1 ``` +The MongoDB seed-list and replica-set tests also need public DNS and host port 27017 free: they reach the MongoDB driver specification's records under `test.build.10gen.cc`, which point at `localhost:27017`. + ### Files from tablepro-cloud `core/crates/account` keeps copies of two tablepro-cloud files: `openapi/v2.openapi-3.0.json`, the `/v2` document `build.rs` generates the client from, and `tests/data/entitlement_vectors.json`, the vectors the entitlement checks run against. tablepro-cloud is private and CI cannot reach it, so they are copies, never links. From `linux/`, refresh both from a tablepro-cloud checkout: From 2a6ffda751a4208f1ffbb21b1e5973b436da47c6 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Thu, 12 Nov 2026 01:19:47 +0700 Subject: [PATCH 2/2] docs(linux): changelog for the MongoDB follow-up --- linux/changelog.d/4208.added.md | 1 + linux/changelog.d/4208.fixed.1.md | 1 + linux/changelog.d/4208.fixed.2.md | 1 + linux/changelog.d/4208.fixed.md | 1 + 4 files changed, 4 insertions(+) create mode 100644 linux/changelog.d/4208.added.md create mode 100644 linux/changelog.d/4208.fixed.1.md create mode 100644 linux/changelog.d/4208.fixed.2.md create mode 100644 linux/changelog.d/4208.fixed.md diff --git a/linux/changelog.d/4208.added.md b/linux/changelog.d/4208.added.md new file mode 100644 index 0000000000..66dcd1fb66 --- /dev/null +++ b/linux/changelog.d/4208.added.md @@ -0,0 +1 @@ +Set to Default on a saved MongoDB row removes the field. diff --git a/linux/changelog.d/4208.fixed.1.md b/linux/changelog.d/4208.fixed.1.md new file mode 100644 index 0000000000..483d8d803a --- /dev/null +++ b/linux/changelog.d/4208.fixed.1.md @@ -0,0 +1 @@ +Duplicating, pasting or importing MongoDB rows wrote `null` for fields the source documents did not have. diff --git a/linux/changelog.d/4208.fixed.2.md b/linux/changelog.d/4208.fixed.2.md new file mode 100644 index 0000000000..75dbca4d28 --- /dev/null +++ b/linux/changelog.d/4208.fixed.2.md @@ -0,0 +1 @@ +With Safe Mode off, a MongoDB `$merge` into the collection it reads ran without asking. diff --git a/linux/changelog.d/4208.fixed.md b/linux/changelog.d/4208.fixed.md new file mode 100644 index 0000000000..dcb26d417c --- /dev/null +++ b/linux/changelog.d/4208.fixed.md @@ -0,0 +1 @@ +MongoDB statements from Copy As or an export stored values such as `NumberLong` as nested documents in mongosh.