Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions core/crates/core/src/dialect/sql_dialect.rs
Original file line number Diff line number Diff line change
Expand Up @@ -285,4 +285,9 @@ pub trait SqlDialect: Send + Sync + std::fmt::Debug {
fn update_sets_default(&self) -> bool {
false
}

/// Whether a read gives NULL for a value the row does not hold, as a document's missing field.
fn reads_missing_as_null(&self) -> bool {
false
}
}
13 changes: 13 additions & 0 deletions core/crates/core/src/dialect/test_dialect.rs
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ pub struct TestDialect {
insert_limits: InsertBatchLimits,
insert_conflict: InsertConflict,
update_default: bool,
missing_reads_null: bool,
binds: Mutex<Vec<RecordedBind>>,
}

Expand Down Expand Up @@ -74,6 +75,7 @@ impl Default for TestDialect {
insert_limits: InsertBatchLimits::DEFAULT,
insert_conflict: InsertConflict::Refused,
update_default: true,
missing_reads_null: false,
binds: Mutex::new(Vec::new()),
}
}
Expand Down Expand Up @@ -139,6 +141,13 @@ impl TestDialect {
}
}

pub fn reading_missing_as_null(self) -> Self {
Self {
missing_reads_null: true,
..self
}
}

pub fn targets(&self) -> Vec<RecordedTarget> {
self.recorded().into_iter().map(|bind| bind.target).collect()
}
Expand Down Expand Up @@ -192,6 +201,10 @@ impl SqlDialect for TestDialect {
self.update_default
}

fn reads_missing_as_null(&self) -> bool {
self.missing_reads_null
}

fn quote_identifier(&self, name: &str) -> String {
format!("\"{}\"", name.replace('"', "\"\""))
}
Expand Down
18 changes: 17 additions & 1 deletion core/crates/core/src/document/document_filter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,30 @@ const JSON_OPERATORS: &[FilterOp] = &[
FilterOp::IsNotNull,
];

/// A pattern is a `$regex`, which never matches a number.
const NUMBER_OPERATORS: &[FilterOp] = &[
FilterOp::Eq,
FilterOp::NotEq,
FilterOp::Lt,
FilterOp::LtEq,
FilterOp::Gt,
FilterOp::GtEq,
FilterOp::Between,
FilterOp::In,
FilterOp::NotIn,
FilterOp::IsNull,
FilterOp::IsNotNull,
];

/// The operators a filter rule takes on a field, in the order the filter
/// strip lists them. A JSON column compares as well as matching text: a
/// value typed for it binds as the BSON its Extended JSON names, and the
/// server compares it with the field's values of that type, numbers of
/// every type with each other, so a field of ints and doubles keeps `<`
/// and Between.
/// and Between. A field of numbers only compares.
pub fn filter_operators(column: &ColumnInfo) -> &'static [FilterOp] {
match column.column_type.kind {
ColumnKind::Json { .. } if super::json_cells::holds_numbers(&column.column_type) => NUMBER_OPERATORS,
ColumnKind::Json { .. } => JSON_OPERATORS,
_ => operators_for(&column.column_type.kind),
}
Expand Down
38 changes: 16 additions & 22 deletions core/crates/core/src/document/document_write.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,7 @@ use super::document_read::collection;
use super::shell_params::ShellParams;

/// A new document of the cells set. A cell left to the server is left
/// out, which for `_id` lets the server make one. A NULL cell is left out
/// too: a page reads a missing field as NULL, and an explicit `null` is a
/// field that a sparse or partial index counts.
/// out, which for `_id` lets the server make one.
pub(crate) fn insert(
table: &TableRef,
columns: &[ColumnInfo],
Expand All @@ -28,7 +26,7 @@ pub(crate) fn insert(
let mut params = ShellParams::new();
let mut fields = Vec::new();
for (column, cell) in columns.iter().zip(cells) {
if let Some(value) = cell.value().filter(|value| !value.is_null()) {
if let Some(value) = cell.value() {
fields.push((column.name.clone(), params.bind(value.clone(), Some(column))?));
}
}
Expand All @@ -38,8 +36,7 @@ pub(crate) fn insert(
Ok(params.statement(&call))
}

/// Several new documents, all of the same columns, each leaving out its
/// NULL cells as `insert` does.
/// Several new documents, all of the same columns.
pub(crate) fn insert_many(
table: &TableRef,
columns: &[&ColumnInfo],
Expand All @@ -59,9 +56,7 @@ pub(crate) fn insert_many(
}
let mut fields = Vec::with_capacity(columns.len());
for (&column, value) in columns.iter().zip(row) {
if !value.is_null() {
fields.push((column.name.clone(), params.bind(value, Some(column))?));
}
fields.push((column.name.clone(), params.bind(value, Some(column))?));
}
documents.push(ShellValue::Object(fields));
}
Expand All @@ -72,9 +67,8 @@ pub(crate) fn insert_many(
}

/// The fields changed on one document, with `$set`. Only the fields named
/// change; one the document lacked is added. A field set to NULL is
/// removed with `$unset`, as a new document leaves it out, so a sparse or
/// partial index stops counting it. A key keeps `$set`.
/// change; one the document lacked is added. A field given back to the
/// server is removed with `$unset`, which a key cannot be.
pub(crate) fn update(
table: &TableRef,
columns: &[ColumnInfo],
Expand All @@ -99,19 +93,19 @@ pub(crate) fn update(
name: column.name.clone(),
});
}
let Some(value) = input.value() else {
return Err(BuildSqlError::ServerComputedValue {
name: column.name.clone(),
});
};
match value {
Value::Null if !names.contains(&column.name) => {
unset.push((column.name.clone(), ShellValue::string("")));
}
_ => {
match input {
CellInput::Value(value) => {
let read = read.iter().find(|(at, _)| at == index).map(|(_, value)| value);
set.push((column.name.clone(), params.bind_edit(value.clone(), column, read)?));
}
CellInput::Default if !names.contains(&column.name) => {
unset.push((column.name.clone(), ShellValue::string("")));
}
CellInput::Default | CellInput::Expression(_) => {
return Err(BuildSqlError::ServerComputedValue {
name: column.name.clone(),
});
}
}
}
let filter = key_filter(columns, identity, key, &mut params)?;
Expand Down
158 changes: 155 additions & 3 deletions core/crates/core/src/document/json_cells.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,13 @@
use crate::column::{ColumnInfo, ColumnKind};
use std::borrow::Cow;

use crate::column::{ColumnInfo, ColumnKind, ColumnType};
use crate::query_result::QueryResult;
use crate::value::{JsonText, Temporal, Value};
use crate::sql_syntax::mongo::{ShellValue, parse_value};
use crate::value::{JsonText, SqlDecimal, Temporal, Value};

/// Longer than any numeric wrapper: a longer cell is a document, read
/// without parsing it.
const WRAPPER_LIMIT: usize = 128;

/// A page's cells in a column the grid edits as JSON, written as
/// extended JSON. A field whose documents hold values of several types is
Expand All @@ -27,6 +34,63 @@ pub(crate) fn json_cells(mut result: QueryResult, columns: &[ColumnInfo]) -> Que
result
}

/// A cell of a `NUMBER_TYPE` column that holds one number, as the
/// number: a bare JSON number, or one Extended JSON wrapper of its digits,
/// `{"$numberLong": "7"}`. A row export writes it as a number column's
/// cell, which a sheet sums. Any other value, and any other column's,
/// comes back as it is.
pub(crate) fn json_number<'a>(value: &'a Value, column: &ColumnType) -> Cow<'a, Value> {
let number = match value {
Value::Json(json) if holds_numbers(column) => numeric_json(json.as_str().trim()),
_ => None,
};
number.map_or(Cow::Borrowed(value), Cow::Owned)
}

/// Whether the column is the JSON column of a field of numbers.
pub(crate) fn holds_numbers(column: &ColumnType) -> bool {
column.name == super::NUMBER_TYPE && matches!(column.kind, ColumnKind::Json { .. })
}

fn numeric_json(text: &str) -> Option<Value> {
match text.as_bytes().first()? {
b'-' | b'0'..=b'9' => plain_number(text),
b'{' if text.len() <= WRAPPER_LIMIT => wrapped_number(text),
_ => None,
}
}

/// A number's digits as they read. A double, which relaxed Extended JSON
/// writes in its shortest round-trip form, stays a double, so a sheet keeps
/// it a number; other digits stay exact as a decimal.
fn plain_number(text: &str) -> Option<Value> {
if let Ok(integer) = text.parse() {
return Some(Value::Int(integer));
}
let double = text.parse::<f64>().ok().filter(|number| number.is_finite());
match double {
Some(number) if text.contains(['e', 'E']) || format!("{number:?}") == text => Some(Value::Float64(number)),
_ => text.parse().ok().map(Value::Decimal),
}
}

fn wrapped_number(text: &str) -> Option<Value> {
let value = parse_value(text).ok()?;
let [(key, ShellValue::String(digits))] = value.as_object()? else {
return None;
};
match key.as_str() {
"$numberInt" => digits.parse::<i32>().ok().map(|number| Value::Int(number.into())),
"$numberLong" => digits.parse().ok().map(Value::Int),
"$numberDouble" => match digits.as_str() {
"NaN" | "Infinity" | "-Infinity" => digits.parse().ok().map(Value::Float64),
_ => plain_number(digits),
},
"$numberDecimal" => digits.parse::<SqlDecimal>().ok().map(Value::Decimal),
_ => None,
}
}

fn as_json(value: Value) -> Value {
let text = match &value {
Value::Null | Value::Json(_) | Value::Undecodable(_) => return value,
Expand Down Expand Up @@ -79,8 +143,9 @@ fn double(number: f64) -> String {
}
}

const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";

pub(super) fn base64(bytes: &[u8]) -> String {
const ALPHABET: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut out = String::with_capacity(bytes.len().div_ceil(3) * 4);
for chunk in bytes.chunks(3) {
let triple = chunk
Expand All @@ -99,6 +164,22 @@ pub(super) fn base64(bytes: &[u8]) -> String {
out
}

/// The bytes of standard base64, `None` for text that is not.
pub(super) fn base64_bytes(text: &str) -> Option<Vec<u8>> {
let mut bytes = Vec::with_capacity(text.len() / 4 * 3);
let (mut buffer, mut bits) = (0u32, 0u32);
for digit in text.trim_end_matches('=').bytes() {
let sextet = ALPHABET.iter().position(|&letter| letter == digit)?;
buffer = ((buffer << 6) | sextet as u32) & 0xffff;
bits += 6;
if bits >= 8 {
bits -= 8;
bytes.push((buffer >> bits) as u8);
}
}
Some(bytes)
}

#[cfg(test)]
mod tests {
use crate::column::ResultColumn;
Expand Down Expand Up @@ -127,6 +208,77 @@ mod tests {
}
}

#[test]
fn a_number_columns_cell_of_one_number_is_that_number_and_any_other_stays() {
let numbers_column = column("n", ColumnKind::Json { binary: false }).column_type;
let numbers_column = ColumnType {
name: crate::document::NUMBER_TYPE.to_owned(),
..numbers_column
};
let json = |text: &str| Value::Json(JsonText::parse(text.to_owned()).unwrap());
let decimal = |text: &str| Value::Decimal(text.parse().unwrap());
let numbers = [
("7", Value::Int(7)),
("-7", Value::Int(-7)),
("19.90", decimal("19.90")),
("-0.1", Value::Float64(-0.1)),
("9.99", Value::Float64(9.99)),
("0.30000000000000004", Value::Float64(0.300_000_000_000_000_04)),
("1e3", Value::Float64(1000.0)),
("2.5E-7", Value::Float64(2.5e-7)),
("123456789012345678901234", decimal("123456789012345678901234")),
(r#"{"$numberInt": "7"}"#, Value::Int(7)),
(r#"{"$numberLong": "3000000000"}"#, Value::Int(3_000_000_000)),
(r#" { "$numberDouble": "1.50" } "#, decimal("1.50")),
(r#"{"$numberDouble": "1e300"}"#, Value::Float64(1e300)),
(r#"{"$numberDouble": "-Infinity"}"#, Value::Float64(f64::NEG_INFINITY)),
(r#"{"$numberDecimal": "9.990"}"#, decimal("9.990")),
];
for (text, number) in numbers {
assert_eq!(json_number(&json(text), &numbers_column).into_owned(), number, "{text}");
}
assert!(matches!(
json_number(&json(r#"{"$numberDouble": "NaN"}"#), &numbers_column).into_owned(),
Value::Float64(number) if number.is_nan()
));

for kept in [
json(r#""7""#),
json("null"),
json("[1]"),
json(r#"{"$numberLong": 7}"#),
json(r#"{"$numberLong": "x"}"#),
json(r#"{"$numberInt": "3000000000"}"#),
json(r#"{"$numberLong": "1", "a": 1}"#),
json(r#"{"$oid": "5f1d7a0c9b1e8b2a3c4d5e6f"}"#),
Value::Text("7".to_owned()),
] {
assert!(
matches!(json_number(&kept, &numbers_column), Cow::Borrowed(_)),
"{kept:?}"
);
}
let jsonb = ColumnType {
name: "jsonb".to_owned(),
..numbers_column.clone()
};
let mixed = column("m", ColumnKind::Json { binary: false }).column_type;
let text_named_number = ColumnType {
kind: ColumnKind::Text {
length: TextLength::Unbounded,
},
..numbers_column
};
for other in [jsonb, mixed, text_named_number] {
for cell in [json("19.90"), json(r#"{"$numberLong": "7"}"#)] {
assert!(matches!(json_number(&cell, &other), Cow::Borrowed(_)), "{other:?}");
}
}
assert_eq!(base64_bytes("aGkh"), Some(b"hi!".to_vec()));
assert_eq!(base64_bytes("aGk="), Some(b"hi".to_vec()));
assert_eq!(base64_bytes("a*"), None);
}

#[test]
fn a_json_column_shows_each_value_as_json_and_the_others_stay() {
let columns = [
Expand Down
5 changes: 4 additions & 1 deletion core/crates/core/src/document/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ pub use document_filter::filter_operators;
pub(crate) use document_filter::{filter_document, typed_document};
pub(crate) use document_read::{count, export, page};
pub(crate) use document_write::{delete, insert, insert_many, key_probe, update};
pub(crate) use json_cells::json_cells;
pub(crate) use json_cells::{json_cells, json_number};
pub use shell_literal::shell_literal;
pub(crate) use shell_literal::{insert_document, insert_start};
pub(crate) use shell_params::ShellParams;
Expand All @@ -30,6 +30,9 @@ pub use shell_params::bind_param;
use crate::dialect::SqlDialect;
use crate::sql_syntax::SqlGrammar;

/// The type of a JSON column of numbers of more than one BSON type.
pub const NUMBER_TYPE: &str = "number";

/// Whether the dialect's statements are shell calls rather than SQL.
pub(crate) fn speaks_shell(dialect: &dyn SqlDialect) -> bool {
dialect.grammar() == SqlGrammar::Mongo
Expand Down
Loading
Loading