diff --git a/Cargo.lock b/Cargo.lock index 9155569008..85c996f59f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7888,6 +7888,7 @@ dependencies = [ "async-trait", "bytes", "chrono", + "deadpool", "deadpool-postgres", "fallible-iterator 0.2.0", "futures-util", diff --git a/core/crates/drivers/mysql/src/metadata.rs b/core/crates/drivers/mysql/src/metadata.rs index d2b5e970f6..fc376180ef 100644 --- a/core/crates/drivers/mysql/src/metadata.rs +++ b/core/crates/drivers/mysql/src/metadata.rs @@ -45,7 +45,8 @@ pub(crate) async fn catalog_rows( table: &'static str, binds: &[Option<&str>], ) -> Result, DriverError> { - let mut query = sqlx::query(sql).persistent(false); + // The text is fixed, so the connection keeps it prepared and a later read only executes it. + let mut query = sqlx::query(sql).persistent(true); for bind in binds { query = query.bind(bind.map(str::to_owned)); } diff --git a/core/crates/drivers/mysql/src/physical.rs b/core/crates/drivers/mysql/src/physical.rs index e527079793..ad84a15717 100644 --- a/core/crates/drivers/mysql/src/physical.rs +++ b/core/crates/drivers/mysql/src/physical.rs @@ -18,6 +18,9 @@ use crate::relay::Relay; use crate::session_statements::{EDITOR_SQL_MODE_STATEMENT, SessionLimits, for_session}; use crate::workspace_session::{WORKSPACE_SQL_MODE_STATEMENT, WORKSPACE_TIME_ZONE}; +// Room for every fixed catalog statement of either flavour. +const CACHED_STATEMENTS: usize = 32; + /// Which session a physical connection opens as. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum SessionKind { @@ -80,8 +83,8 @@ impl MysqlEndpoint { .no_engine_substitution(false) .set_names(false) .timezone(None::) - // No statement outlives the call that prepared it, so a pool - // connection never piles up server-side statement handles. + // No statement outlives the call that prepared it, so a + // session never piles up server-side statement handles. .statement_cache_capacity(0) // sqlx would log every statement, values and all. .disable_statement_logging(); @@ -147,7 +150,11 @@ impl MysqlEndpoint { }, }; Ok(match kind { - SessionKind::Workspace => options.timezone(Some(WORKSPACE_TIME_ZONE.to_owned())), + // The catalog's fixed statements stay prepared, so a later read + // skips the PREPARE; the bound caps the handles a pool member holds. + SessionKind::Workspace => options + .timezone(Some(WORKSPACE_TIME_ZONE.to_owned())) + .statement_cache_capacity(CACHED_STATEMENTS), SessionKind::Editor | SessionKind::Control => options, }) } diff --git a/core/crates/drivers/mysql/src/stream_read.rs b/core/crates/drivers/mysql/src/stream_read.rs index 946f66b94b..e1a6d373ba 100644 --- a/core/crates/drivers/mysql/src/stream_read.rs +++ b/core/crates/drivers/mysql/src/stream_read.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use futures_util::TryStreamExt; use sqlx::mysql::{MySqlConnection, MySqlDatabaseError}; -use sqlx::{AssertSqlSafe, Either, Executor as _, Row as _, SqlSafeStr as _, Statement as _}; +use sqlx::{AssertSqlSafe, Either, Executor as _, Row as _, Statement as _}; use tablepro_core::column::ResultColumn; use tablepro_core::sql_syntax::{SqlGrammar, StatementAccess, statement_access}; use tablepro_core::{ @@ -15,7 +15,7 @@ use crate::errors::map_error; use crate::flavour::MySqlFlavour; use crate::mysql_timestamp_mode::MySqlTimestampMode; use crate::mysql_wire_format::MySqlWireFormat; -use crate::workspace_call::{describe, prepared}; +use crate::workspace_call::{describe, prepare_uncached, prepared}; const BATCH_ROWS: usize = 256; @@ -106,10 +106,7 @@ async fn prepare_as_one( flavour: MySqlFlavour, sql: &str, ) -> Result, DriverError> { - let prepared = (&mut *connection) - .prepare(AssertSqlSafe(sql.to_owned()).into_sql_str()) - .await; - match prepared { + match prepare_uncached(connection, sql).await { Ok(statement) => Ok(statement .columns() .iter() diff --git a/core/crates/drivers/mysql/src/workspace_call.rs b/core/crates/drivers/mysql/src/workspace_call.rs index a798af81d6..29ca90248c 100644 --- a/core/crates/drivers/mysql/src/workspace_call.rs +++ b/core/crates/drivers/mysql/src/workspace_call.rs @@ -1,7 +1,7 @@ use futures_util::TryStreamExt; use futures_util::future::BoxFuture; -use sqlx::mysql::MySqlConnection; -use sqlx::{AssertSqlSafe, Executor as _, Row as _, SqlSafeStr as _, Statement as _}; +use sqlx::mysql::{MySqlConnection, MySqlStatement}; +use sqlx::{AssertSqlSafe, Connection as _, Executor as _, Row as _, SqlSafeStr as _, Statement as _}; use tablepro_core::column::ResultColumn; use tablepro_core::statement::Statement; use tablepro_core::{DriverError, QueryResult, RowLimit}; @@ -83,14 +83,13 @@ pub(crate) async fn query_bound( } /// The columns of a statement that returned no rows: the prepare -/// carries them, and with no cache the statement is closed right after. +/// carries them. pub(crate) async fn describe( connection: &mut MySqlConnection, flavour: MySqlFlavour, sql: &str, ) -> Result, DriverError> { - let prepared = (&mut *connection) - .prepare(AssertSqlSafe(sql.to_owned()).into_sql_str()) + let prepared = prepare_uncached(connection, sql) .await .map_err(|error| map_error(&error, flavour))?; Ok(prepared @@ -100,6 +99,20 @@ pub(crate) async fn describe( .collect()) } +// The cache keys a statement by its text alone and keeps the columns of its first PREPARE, and after a schema change +// the same text has other columns. sqlx can only empty the whole cache, so the catalog's statements go with it and are +// prepared again on their next read. +pub(crate) async fn prepare_uncached( + connection: &mut MySqlConnection, + sql: &str, +) -> Result { + let prepared = (&mut *connection) + .prepare(AssertSqlSafe(sql.to_owned()).into_sql_str()) + .await; + connection.clear_cached_statements().await?; + prepared +} + pub(crate) fn count_op(statement: Statement) -> ReadOp { read_op(move |connection, flavour| Box::pin(async move { count(connection, flavour, &statement).await })) } diff --git a/core/crates/drivers/mysql/tests/integration/main.rs b/core/crates/drivers/mysql/tests/integration/main.rs index dcd81aff31..583501623f 100644 --- a/core/crates/drivers/mysql/tests/integration/main.rs +++ b/core/crates/drivers/mysql/tests/integration/main.rs @@ -25,4 +25,5 @@ mod support; mod tls; mod user_read; mod value_corpus; +mod wire_round_trips; mod write_batch; diff --git a/core/crates/drivers/mysql/tests/integration/wire_round_trips.rs b/core/crates/drivers/mysql/tests/integration/wire_round_trips.rs new file mode 100644 index 0000000000..6a7cdea85f --- /dev/null +++ b/core/crates/drivers/mysql/tests/integration/wire_round_trips.rs @@ -0,0 +1,76 @@ +//! How many times a workspace read waits on the server, counted on the +//! wire. + +use std::sync::Arc; + +use tablepro_core::{RowLimit, TlsConfig, WorkspaceSettings}; +use tablepro_test_fixtures::MysqlFlavour; +use tablepro_test_support::RoundTripCounter; +use tablepro_test_support::conformance::test_liveness; + +use crate::support::{CALL, connect_with, direct, options_over, password, setup, start, statement, table, within}; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_catalog_read_on_a_warm_connection_skips_the_prepare() { + let fixture = start(MysqlFlavour::Mysql).await; + let liveness = test_liveness(); + let counter = Arc::new(RoundTripCounter::new(direct(fixture.host(), fixture.port(), &liveness))); + let connection = connect_with(options_over( + counter.clone(), + password(&fixture.password_credentials()), + fixture.database(), + TlsConfig::disabled(), + WorkspaceSettings::workspace(false), + liveness, + )) + .await; + setup( + connection.as_ref(), + &["CREATE TABLE t (id int PRIMARY KEY, name varchar(20))"], + ) + .await; + let columns = || async { + let before = counter.round_trips(); + connection + .fetch_columns(&table("t"), within(CALL)) + .await + .expect("the columns"); + counter.round_trips() - before + }; + columns().await; + + // The pool's ping before the read is the other round trip. + assert_eq!([columns().await, columns().await], [2, 2]); + connection.close().await.expect("a close"); +} + +/// The columns of an empty page come from a prepare, which must not +/// serve the columns the same text had before a schema change. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn an_empty_page_shows_the_columns_after_a_type_change() { + let fixture = start(MysqlFlavour::Mysql).await; + let connection = crate::support::connect(&fixture, false).await; + setup(connection.as_ref(), &["CREATE TABLE e (id int PRIMARY KEY, n int)"]).await; + let columns = || async { + connection + .query_bound(&statement("SELECT * FROM e"), RowLimit::EDITOR_DEFAULT, within(CALL)) + .await + .expect("an empty page") + .columns + .iter() + .map(|column| column.column_type.name.clone()) + .collect::>() + }; + let before = columns().await; + + setup(connection.as_ref(), &["ALTER TABLE e MODIFY n varchar(20)"]).await; + + assert_ne!( + columns().await, + before, + "the empty page kept the columns from before the change" + ); + connection.close().await.expect("a close"); +} diff --git a/core/crates/drivers/postgres/Cargo.toml b/core/crates/drivers/postgres/Cargo.toml index 5e770524a1..0215c3619d 100644 --- a/core/crates/drivers/postgres/Cargo.toml +++ b/core/crates/drivers/postgres/Cargo.toml @@ -11,6 +11,7 @@ tablepro-net.workspace = true async-trait.workspace = true bytes.workspace = true chrono.workspace = true +deadpool.workspace = true deadpool-postgres.workspace = true fallible-iterator.workspace = true futures-util.workspace = true diff --git a/core/crates/drivers/postgres/src/connection.rs b/core/crates/drivers/postgres/src/connection.rs index 009f52972d..10e8a4144d 100644 --- a/core/crates/drivers/postgres/src/connection.rs +++ b/core/crates/drivers/postgres/src/connection.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; use async_trait::async_trait; -use deadpool_postgres::{Pool, PoolError, TimeoutType, Timeouts}; +use deadpool_postgres::{PoolError, TimeoutType, Timeouts}; use tablepro_core::column::ColumnInfo; use tablepro_core::ddl::{DatabaseChoices, InboundForeignKey, OutboundForeignKey, TableDefinition}; use tablepro_core::dialect::SqlDialect; @@ -31,7 +31,7 @@ use crate::errors::{map_error, map_pool_error}; use crate::metadata; use crate::pg_call::PgCall; use crate::physical::PgEndpoint; -use crate::pool::{probe_timeouts, workspace_timeouts}; +use crate::pool::{PgObject, PgPool, probe_timeouts, workspace_timeouts}; use crate::standalone::Standalone; use crate::stream_read::stream; use crate::workspace_call::{ @@ -43,7 +43,7 @@ use crate::write_batch::{WritePlan, perform, precheck}; /// connections behind the sidebar, the grid and every save. pub(crate) struct PgConnection { endpoint: Arc, - pool: Pool, + pool: PgPool, calls: OwnedCalls, tracker: TaskTracker, settings: WorkspaceSettings, @@ -56,7 +56,7 @@ pub(crate) struct PgConnection { impl PgConnection { pub(crate) fn new( endpoint: Arc, - pool: Pool, + pool: PgPool, tracker: TaskTracker, settings: WorkspaceSettings, server: ServerInfo, @@ -229,6 +229,35 @@ impl PgConnection { } } +// A member that has not answered may sit on a dead route, so it leaves the pool instead of going back, also when the +// probe's deadline drops it mid-question. +struct Asking(Option); + +impl Asking { + async fn answer(mut self) -> Result<(), DriverError> { + let answered = match self.0.as_ref() { + Some(object) => object + .simple_query("SELECT 1") + .await + .map(|_| ()) + .map_err(|error| map_error(&error)), + None => Ok(()), + }; + if answered.is_ok() { + drop(self.0.take()); + } + answered + } +} + +impl Drop for Asking { + fn drop(&mut self) { + if let Some(object) = self.0.take() { + drop(PgObject::take(object)); + } + } +} + /// A failure that may mean the route is gone rather than one statement: /// a lost connection, a transport error, or a timeout the server never /// acknowledged. @@ -292,11 +321,12 @@ impl Connection for PgConnection { let probing = async { let quick: Timeouts = probe_timeouts(self.liveness()); match self.pool.timeout_get(&quick).await { - Ok(object) => object - .simple_query("SELECT 1") - .await - .map(|_| ()) - .map_err(|error| map_error(&error)), + Ok(object) => match Asking(Some(object)).answer().await { + // Only this member may have gone stale; a login of its own clears the route, and otherwise the + // member's answer stands. + Err(error) if suspects_link(&error) => self.probe_standalone(deadline).await.map_err(|_| error), + answered => answered, + }, // Every slot is busy, which is not the server being gone. Err(PoolError::Timeout(TimeoutType::Wait)) => self.probe_standalone(deadline).await, Err(error) => Err(map_pool_error(&error)), diff --git a/core/crates/drivers/postgres/src/editor/run.rs b/core/crates/drivers/postgres/src/editor/run.rs index 65b9ca4513..6d49dbdb8f 100644 --- a/core/crates/drivers/postgres/src/editor/run.rs +++ b/core/crates/drivers/postgres/src/editor/run.rs @@ -5,7 +5,7 @@ use futures_util::StreamExt; use tablepro_core::sql_syntax::script::{LexicalSettings, ScriptPlan, ScriptToken}; use tablepro_core::sql_syntax::{SourceTable, SqlGrammar}; use tablepro_core::{DriverError, ResultEvent, ResultSink, RowBatch, RowLimit, RunEnd, SinkFlow, Truncation, Value}; -use tokio_postgres::SimpleQueryMessage; +use tokio_postgres::{SimpleQueryMessage, SimpleQueryStream}; use tracing::debug; use super::cursor_flight::{ @@ -164,23 +164,23 @@ impl<'a> Run<'a> { } } - // A row source goes through a cursor, so a result larger than the limit stops on the server. The DECLARE and the - // FETCH go as separate messages. + // A row source goes through a cursor, so a result larger than the limit stops on the server. It costs two flights, + // each sent whole and answered in order: the open with the DECLARE, then the FETCH with the end of the cursor. async fn cursor( &mut self, text: &str, class: StatementClass, owner: CursorOwner, ) -> Result, Lost> { - let open = match owner { - CursorOwner::Driver { read_only: true } => "START TRANSACTION READ ONLY", - CursorOwner::Driver { read_only: false } => "BEGIN", - CursorOwner::Savepoint => SAVEPOINT, - }; - if let Some(error) = self.exchange(open, Role::Control).await? { + // With backslash escapes on, the statement the parser reads may not be the one the server ran. + let traced = traceable(text, owner) + .filter(|_| self.core.standard_conforming_strings() && !self.stopped() && self.watch.due().is_none()); + let opening = self.open_cursor(text, owner, traced).await?; + if let Some(error) = opening.opened { + self.core.discard_notices(); return Ok(Some(error)); } - let declared = self.read(&declare(text), Role::Wrapper).await?.map(correct_position); + let declared = opening.declared.map(correct_position); if declared .as_ref() .is_some_and(|error| needs_plain_statement(error) && !self.stopped()) @@ -195,35 +195,134 @@ impl<'a> Run<'a> { self.abandon_cursor(owner, false).await?; return Ok(Some(error)); } - // With backslash escapes on, the statement the parser reads may not be the one the server ran. - self.traced = match traceable(text, owner) { - Some(statement) if self.core.standard_conforming_strings() => self.trace(statement).await?, - _ => None, + let traced = match opening.trace { + Some(answers) => self.traced_table(answers).await?, + None => None, + }; + self.watch.settle().await; + if self.stopped() { + self.abandon_cursor(owner, true).await?; + return Ok(Some(DriverError::Cancelled)); + } + self.fetch_rows(class, owner, traced).await + } + + // A failed open fails the DECLARE before it plans anything, so no part of the statement runs outside the + // transaction or savepoint it needs. The origin lookup rides along, after the DECLARE that took the lock it reads. + async fn open_cursor( + &mut self, + text: &str, + owner: CursorOwner, + traced: Option, + ) -> Result { + let open = match owner { + CursorOwner::Driver { read_only: true } => "START TRANSACTION READ ONLY", + CursorOwner::Driver { read_only: false } => "BEGIN", + CursorOwner::Savepoint => SAVEPOINT, + }; + self.watch.settle().await; + let opening = self.queue(open).await?; + let declaring = self.queue(&declare(text)).await?; + let client = &self.core.client; + let mut flight = std::pin::pin!(async move { + let trace = async move { + match traced { + Some(statement) => Some(futures_util::join!( + client.batch_execute(result_origin::SAVEPOINT), + result_origin::lookup(client, statement), + client.batch_execute(result_origin::RELEASE_SAVEPOINT), + )), + None => None, + } + }; + futures_util::join!(answer(opening), answer(declaring), trace) + }); + let (opened, declared, trace) = loop { + tokio::select! { + biased; + answers = &mut flight => break answers, + interrupt = self.watch.triggered(), if self.watch.fired().is_none() => { + let running = Some((Arc::clone(&self.core.endpoint), self.core.cancel_token.clone())); + self.watch.fire(interrupt, running); + } + () = self.watch.gave_up(), if self.watch.fired().is_some() => return Err(Lost), + } + }; + Ok(Opening { + opened: opened?, + declared: declared?, + trace, + }) + } + + // A lookup that failed is rolled back to its own savepoint, which leaves the cursor's transaction and the user's + // rows as they were. + async fn traced_table(&mut self, (saved, found, released): TraceAnswers) -> Result, Lost> { + if saved.is_ok() && released.is_err() && !self.stopped() { + self.exchange(result_origin::ROLLBACK_TO_SAVEPOINT, Role::Control) + .await?; + } + Ok(match (found, released) { + (Ok(traced), Ok(())) => traced, + (Ok(_), Err(_)) => None, + (Err(error), _) => { + debug!(%error, "the origin lookup of a result failed"); + None + } + }) + } + + // COMMIT closes a cursor declared without HOLD and ends a block the FETCH aborted as a rollback, so it needs no + // answer before it goes. RELEASE leaves a cursor open, so the savepoint's cursor is closed first. The connection + // reads ahead into the notice queue, so a notice the COMMIT raises can reach the sink before the last rows. + async fn fetch_rows( + &mut self, + class: StatementClass, + owner: CursorOwner, + traced: Option, + ) -> Result, Lost> { + let fetching = self.queue(&fetch(self.limit)).await?; + let closing = match owner { + CursorOwner::Driver { .. } => None, + CursorOwner::Savepoint => Some(self.queue(CLOSE).await?), }; - let fetched = self - .exchange(&fetch(self.limit), Role::User { class, cursor: true }) - .await?; + let ending = match owner { + CursorOwner::Driver { .. } => self.queue("COMMIT").await?, + CursorOwner::Savepoint => self.queue(RELEASE_SAVEPOINT).await?, + }; + self.traced = traced; + let fetched = self.drain(fetching, Role::User { class, cursor: true }).await; self.traced = None; - if let Some(error) = fetched { - self.abandon_cursor(owner, false).await?; - return Ok(Some(error)); + let fetched = fetched?; + self.forward_notices().await; + if let Some(closing) = closing { + self.drain(closing, Role::Control).await?; } - self.exchange(CLOSE, Role::Control).await?; // Deferred triggers fire at the COMMIT, for the user's statement. - let closed = match owner { - CursorOwner::Driver { .. } => self.exchange("COMMIT", Role::Wrapper).await?, - CursorOwner::Savepoint => self.exchange(RELEASE_SAVEPOINT, Role::Wrapper).await?, - }; - Ok(closed) + let ended = self.drain(ending, Role::Wrapper).await?; + self.forward_notices().await; + match owner { + CursorOwner::Driver { .. } => { + // A cancel that reached the COMMIT before it ran leaves the block open and aborted. + if ended.is_some() { + self.exchange("ROLLBACK", Role::Control).await?; + } + self.core.tracker.set(KnownState::Idle); + } + CursorOwner::Savepoint if fetched.is_some() => self.core.tracker.set(KnownState::Failed), + CursorOwner::Savepoint => {} + } + Ok(fetched.or(ended)) } /// Undo the cursor's own transaction or savepoint after a failure. /// /// Inside the user's transaction a statement that really failed still - /// aborts it, as it would without the row limit; only a DECLARE the - /// cursor itself caused to fail is rolled back to the savepoint. - async fn abandon_cursor(&mut self, owner: CursorOwner, refused_declare: bool) -> Result<(), Lost> { - match (owner, refused_declare) { + /// aborts it, as it would without the row limit; a cursor whose + /// statement never ran, refused at its DECLARE or stopped before its + /// FETCH, is rolled back to the savepoint, which also closes it. + async fn abandon_cursor(&mut self, owner: CursorOwner, never_ran: bool) -> Result<(), Lost> { + match (owner, never_ran) { (CursorOwner::Driver { .. }, _) => { self.exchange("ROLLBACK", Role::Control).await?; self.core.tracker.set(KnownState::Idle); @@ -237,49 +336,6 @@ impl<'a> Run<'a> { Ok(()) } - // The lookup goes in one round trip between a savepoint of the driver's own and its release, so one that fails - // is rolled back to the savepoint and leaves the cursor's transaction and the user's rows as they were. - async fn trace(&mut self, statement: SourceTable) -> Result, Lost> { - if self.stopped() || self.watch.due().is_some() { - return Ok(None); - } - self.watch.settle().await; - let (saved, found, released) = { - let client = &self.core.client; - let mut flight = std::pin::pin!(async move { - futures_util::join!( - client.batch_execute(result_origin::SAVEPOINT), - result_origin::lookup(client, statement), - client.batch_execute(result_origin::RELEASE_SAVEPOINT), - ) - }); - loop { - tokio::select! { - biased; - answers = &mut flight => break answers, - interrupt = self.watch.triggered(), if self.watch.fired().is_none() => { - let running = Some((Arc::clone(&self.core.endpoint), self.core.cancel_token.clone())); - self.watch.fire(interrupt, running); - } - () = self.watch.gave_up(), if self.watch.fired().is_some() => return Err(Lost), - } - } - }; - self.core.discard_notices(); - if saved.is_ok() && released.is_err() && !self.stopped() { - self.exchange(result_origin::ROLLBACK_TO_SAVEPOINT, Role::Control) - .await?; - } - match (found, released) { - (Ok(traced), Ok(())) => Ok(traced), - (Ok(_), Err(_)) => Ok(None), - (Err(error), _) => { - debug!(%error, "the origin lookup of a result failed"); - Ok(None) - } - } - } - async fn track(&mut self, class: StatementClass, succeeded: bool) -> Result<(), Lost> { match self.core.tracker.after(class, succeeded) { Next::Known(state) => { @@ -331,10 +387,17 @@ impl<'a> Run<'a> { if self.stopped() && matches!(role, Role::User { .. }) { return Ok(Some(DriverError::Cancelled)); } - let stream = match self.core.client.simple_query_raw(sql).await { - Ok(stream) => stream, - Err(_) => return Err(Lost), - }; + let stream = self.queue(sql).await?; + self.drain(stream, role).await + } + + // tokio-postgres writes a message as soon as it is queued, and the server answers in order, so messages queued + // before any answer is read share one round trip. + async fn queue(&mut self, sql: &str) -> Result { + self.core.client.simple_query_raw(sql).await.map_err(|_| Lost) + } + + async fn drain(&mut self, stream: SimpleQueryStream, role: Role) -> Result, Lost> { let mut stream = std::pin::pin!(stream); let mut set = ResultSet::default(); let mut first_error = None; @@ -469,6 +532,34 @@ impl<'a> Run<'a> { } } +struct Opening { + opened: Option, + declared: Option, + trace: Option, +} + +type TraceAnswers = ( + Result<(), tokio_postgres::Error>, + Result, DriverError>, + Result<(), tokio_postgres::Error>, +); + +/// The first error of a driver message that returns no rows. +async fn answer(stream: SimpleQueryStream) -> Result, Lost> { + let mut stream = std::pin::pin!(stream); + let mut first_error = None; + while let Some(message) = stream.next().await { + match message { + Ok(_) => {} + Err(error) if error.as_db_error().is_none() => return Err(Lost), + Err(error) => { + first_error.get_or_insert_with(|| map_error(&error)); + } + } + } + Ok(first_error) +} + /// Why a set ended where it did. A cursor that stopped at the limit /// stopped the statement with it: what the statement does per row, a /// function it calls in its select list included, never happened for the diff --git a/core/crates/drivers/postgres/src/pg_call.rs b/core/crates/drivers/postgres/src/pg_call.rs index 525f329efa..777d0dd8bb 100644 --- a/core/crates/drivers/postgres/src/pg_call.rs +++ b/core/crates/drivers/postgres/src/pg_call.rs @@ -1,22 +1,23 @@ use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use deadpool_postgres::{Object, Pool, Timeouts}; +use deadpool_postgres::Timeouts; use tablepro_core::{CallResource, DriverError, StopRequest}; use tokio_postgres::Client; use crate::cancel::PgStop; use crate::errors::map_pool_error; +use crate::pool::{PgMember, PgObject, PgPool}; /// The pooled connection one workspace call runs on. /// /// It starts empty and is filled inside the call, so the call's deadline -/// and cancel cover the wait for a slot too. deadpool's own recycle only -/// runs `""`, which succeeds inside a half-finished transaction, so any -/// connection whose flight did not end with an acknowledged COMMIT or -/// ROLLBACK is taken out of the pool instead of returned to it. +/// and cancel cover the wait for a slot too. The pool's recycle never +/// asks whether a transaction is open, so any connection whose flight +/// did not end with an acknowledged COMMIT or ROLLBACK is taken out of +/// the pool instead of returned to it. pub(crate) struct PgCall { - object: Option, + object: Option, abandon: Arc, } @@ -32,7 +33,7 @@ impl PgCall { /// stopped. Once the slot is held, the stop can reach the server. pub(crate) async fn acquire( &mut self, - pool: &Pool, + pool: &PgPool, timeouts: &Timeouts, request: &StopRequest, stop: &PgStop, @@ -56,7 +57,7 @@ impl PgCall { pub(crate) fn client(&self) -> Result<&Client, DriverError> { self.object .as_deref() - .map(|wrapper| &**wrapper) + .map(PgMember::client) .ok_or_else(|| DriverError::Protocol("the call holds no connection".to_owned())) } @@ -68,7 +69,7 @@ impl PgCall { fn take(&mut self) { if let Some(object) = self.object.take() { - drop(Object::take(object)); + drop(PgObject::take(object)); } } } diff --git a/core/crates/drivers/postgres/src/pool.rs b/core/crates/drivers/postgres/src/pool.rs index 6ceeaabb4b..0f5501e2d9 100644 --- a/core/crates/drivers/postgres/src/pool.rs +++ b/core/crates/drivers/postgres/src/pool.rs @@ -1,11 +1,13 @@ use std::future::Future; use std::io; +use std::ops::Deref; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; -use std::time::Duration; +use std::time::{Duration, SystemTime}; -use deadpool_postgres::{Manager, ManagerConfig, Pool, RecyclingMethod, Runtime, Timeouts}; +use deadpool::managed::{self, Metrics, RecycleResult}; +use deadpool_postgres::{ClientWrapper, Manager, ManagerConfig, RecyclingMethod, Runtime, Timeouts}; use tablepro_core::{DriverError, LivenessPolicy}; use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; use tokio::task::JoinHandle; @@ -50,17 +52,80 @@ impl deadpool_postgres::Connect for PgConnect { } } -pub(crate) fn build_pool(endpoint: Arc, tracker: TaskTracker) -> Result { +pub(crate) type PgPool = managed::Pool; +pub(crate) type PgObject = managed::Object; + +// deadpool-postgres's Fast recycle hands a connection out again while its connection task runs: a server that closed +// it, or a route whose keepalive failed, has already ended that task. +pub(crate) struct PgManager { + fast: Manager, + ask_after_idle: Duration, +} + +// deadpool's own idle time is an Instant, which stops while the machine sleeps, and a sleep is when a route most +// often drops a connection, so the member also keeps the wall-clock time it was last handed out. +pub(crate) struct PgMember { + client: ClientWrapper, + handed_out: SystemTime, +} + +impl PgMember { + pub(crate) fn client(&self) -> &Client { + &self.client + } +} + +impl Deref for PgMember { + type Target = ClientWrapper; + + fn deref(&self) -> &ClientWrapper { + &self.client + } +} + +impl managed::Manager for PgManager { + type Type = PgMember; + type Error = tokio_postgres::Error; + + async fn create(&self) -> Result { + Ok(PgMember { + client: self.fast.create().await?, + handed_out: SystemTime::now(), + }) + } + + // A route can drop an idle connection without a word, and keepalive finds that out only after its idle time, so + // a connection idle that long answers the server first. + async fn recycle(&self, member: &mut PgMember, metrics: &Metrics) -> RecycleResult { + self.fast.recycle(&mut member.client, metrics).await?; + let asleep = member.handed_out.elapsed().unwrap_or_default(); + if metrics.last_used().max(asleep) >= self.ask_after_idle { + member.client.simple_query("").await?; + } + member.handed_out = SystemTime::now(); + Ok(()) + } + + fn detach(&self, member: &mut PgMember) { + self.fast.detach(&mut member.client); + } +} + +pub(crate) fn build_pool(endpoint: Arc, tracker: TaskTracker) -> Result { let liveness = *endpoint.liveness(); - let manager = Manager::from_connect( + let fast = Manager::from_connect( Config::new(), PgConnect { endpoint, tracker }, ManagerConfig { - recycling_method: RecyclingMethod::Verified, + recycling_method: RecyclingMethod::Fast, }, ); + let manager = PgManager { + fast, + ask_after_idle: liveness.tcp_keepalive_idle, + }; let timeouts = workspace_timeouts(&liveness); - Pool::builder(manager) + PgPool::builder(manager) .max_size(WORKSPACE_POOL_MAX_SIZE) .wait_timeout(timeouts.wait) .create_timeout(timeouts.create) diff --git a/core/crates/drivers/postgres/tests/integration/cancellation.rs b/core/crates/drivers/postgres/tests/integration/cancellation.rs index 3a185ddb6a..1bf905000e 100644 --- a/core/crates/drivers/postgres/tests/integration/cancellation.rs +++ b/core/crates/drivers/postgres/tests/integration/cancellation.rs @@ -15,8 +15,8 @@ use tokio_util::sync::CancellationToken; use crate::slow_route::{CANCEL_REQUEST, Hold, SlowRoute}; use crate::support::{ - CALL, connect, connect_with, direct, options_over, password, run, run_with, scalar, session, start, statement, - within, + CALL, connect, connect_with, direct, options_over, password, run, run_ok, run_with, scalar, session, start, + statement, within, }; /// How long a cancel request takes to reach the server on a slow route. @@ -171,3 +171,32 @@ async fn a_late_cancel_after_stop_never_stops_the_next_editor_run() { next.end ); } + +/// A stop that comes while the cursor's DECLARE is on its way, inside the +/// user's transaction, rolls the cursor back with its savepoint, so the +/// next row source in that transaction can declare it again. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_stop_before_the_fetch_in_a_transaction_leaves_no_cursor_behind() { + let fixture = start().await; + let connection = connect_over( + &fixture, + vec![Hold::new(b"DECLARE tablepro_cursor", Duration::from_millis(600))], + ) + .await; + let editor = session(connection.as_ref()).await; + run_ok(editor.as_ref(), "BEGIN").await; + let cancel = CancellationToken::new(); + let stopper = cancel.clone(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(200)).await; + stopper.cancel(); + }); + + let (stopped, _) = run_with(editor.as_ref(), "SELECT 1", RunOptions::new(cancel)).await; + let (next, _) = run(editor.as_ref(), "SELECT 2").await; + + assert_eq!(stopped.end, RunEnd::Cancelled); + assert_eq!(next.end, RunEnd::Completed, "the next select ended {:?}", next.end); + run_ok(editor.as_ref(), "ROLLBACK").await; +} diff --git a/core/crates/drivers/postgres/tests/integration/main.rs b/core/crates/drivers/postgres/tests/integration/main.rs index 6499d3f86c..57f303971a 100644 --- a/core/crates/drivers/postgres/tests/integration/main.rs +++ b/core/crates/drivers/postgres/tests/integration/main.rs @@ -29,4 +29,5 @@ mod slow_route; mod support; mod tls; mod value_corpus; +mod wire_round_trips; mod write_batch; diff --git a/core/crates/drivers/postgres/tests/integration/wire_round_trips.rs b/core/crates/drivers/postgres/tests/integration/wire_round_trips.rs new file mode 100644 index 0000000000..30922db4a9 --- /dev/null +++ b/core/crates/drivers/postgres/tests/integration/wire_round_trips.rs @@ -0,0 +1,128 @@ +//! How many times a statement waits on the server, counted on the wire. + +use std::sync::Arc; +use std::time::Duration; + +use tablepro_core::{Connection, EditorSession, LivenessPolicy, TlsConfig, WorkspaceSettings}; +use tablepro_test_fixtures::PostgresFixture; +use tablepro_test_support::conformance::test_liveness; +use tablepro_test_support::{RoundTripCounter, StreamTally}; + +use crate::support::{connect_with, direct, options_over, password, rows, run_ok, session, setup, start}; + +struct Wire { + counter: Arc, + connection: Arc, +} + +async fn wire(read_only: bool, liveness: LivenessPolicy) -> (PostgresFixture, Wire) { + let fixture = start().await; + let counter = Arc::new(RoundTripCounter::new(direct(fixture.host(), fixture.port(), &liveness))); + let connection = connect_with(options_over( + counter.clone(), + password(&fixture.password_credentials()), + fixture.database(), + TlsConfig::disabled(), + WorkspaceSettings::workspace(read_only), + liveness, + )) + .await; + (fixture, Wire { counter, connection }) +} + +impl Wire { + async fn editor(&self) -> (Arc, StreamTally) { + let before = self.counter.streams().len(); + let session = session(self.connection.as_ref()).await; + let streams = self.counter.streams(); + assert_eq!(streams.len(), before + 1, "the session opened one connection"); + let tally = streams.last().cloned().expect("the session's connection"); + (session, tally) + } + + /// Round trips of one workspace read, over every pooled connection. + async fn read(&self, sql: &str) -> usize { + let before = self.counter.round_trips(); + rows(self.connection.as_ref(), sql).await; + self.counter.round_trips() - before + } +} + +/// Round trips of one script on the session's own connection. +async fn run(session: &dyn EditorSession, tally: &StreamTally, sql: &str) -> usize { + let before = tally.round_trips(); + run_ok(session, sql).await; + tally.round_trips() - before +} + +const TABLE: &[&str] = &[ + "CREATE TABLE t (id int PRIMARY KEY, name text)", + "INSERT INTO t SELECT n, 'row ' || n FROM generate_series(1, 50) AS n", +]; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn an_editor_select_costs_two_round_trips() { + let (_fixture, wire) = wire(false, test_liveness()).await; + setup(wire.connection.as_ref(), TABLE).await; + let (session, tally) = wire.editor().await; + + let traced = run(session.as_ref(), &tally, "SELECT * FROM t").await; + let untraced = run(session.as_ref(), &tally, "SELECT count(*) FROM t").await; + run_ok(session.as_ref(), "BEGIN").await; + let in_transaction = run(session.as_ref(), &tally, "SELECT * FROM t").await; + assert_eq!([traced, untraced, in_transaction], [2, 2, 2]); + run_ok(session.as_ref(), "ROLLBACK").await; + session.close().await; + wire.connection.close().await.expect("a close"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_read_only_editor_select_costs_two_round_trips() { + let (_fixture, wire) = wire(true, test_liveness()).await; + let (session, tally) = wire.editor().await; + + let traced = run(session.as_ref(), &tally, "SELECT * FROM pg_catalog.pg_am").await; + let untraced = run(session.as_ref(), &tally, "SELECT n FROM generate_series(1, 10) AS n").await; + assert_eq!([traced, untraced], [2, 2]); + session.close().await; + wire.connection.close().await.expect("a close"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_workspace_read_on_a_pooled_connection_costs_one_round_trip() { + let (_fixture, wire) = wire(false, test_liveness()).await; + setup(wire.connection.as_ref(), TABLE).await; + wire.read("SELECT * FROM t").await; + + let reads = [ + wire.read("SELECT * FROM t").await, + wire.read("SELECT * FROM t WHERE id > 10").await, + ]; + assert_eq!(reads, [1, 1]); + wire.connection.close().await.expect("a close"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore = "requires docker"] +async fn a_pooled_connection_idle_past_the_keepalive_idle_time_answers_first() { + // Whole seconds: Linux refuses a keepalive idle time under one. + let idle = Duration::from_secs(1); + let liveness = LivenessPolicy { + tcp_keepalive_idle: idle, + ..test_liveness() + }; + let (_fixture, wire) = wire(false, liveness).await; + setup(wire.connection.as_ref(), TABLE).await; + wire.read("SELECT * FROM t").await; + + tokio::time::sleep(idle + Duration::from_millis(200)).await; + assert_eq!( + wire.read("SELECT * FROM t").await, + 2, + "the recycle check, then the read" + ); + wire.connection.close().await.expect("a close"); +} diff --git a/core/crates/ffi/src/runtime.rs b/core/crates/ffi/src/runtime.rs index da0142bc64..3b22ad5a6c 100644 --- a/core/crates/ffi/src/runtime.rs +++ b/core/crates/ffi/src/runtime.rs @@ -1,7 +1,7 @@ use std::future::Future; use std::sync::OnceLock; -use tablepro_session::runtime::{TaskFailure, TaskResult, Tasks}; +use tablepro_session::runtime::{TaskFailure, TaskResult, Tasks, worker_count}; use tokio::runtime::{Builder, Runtime}; use crate::error::CoreError; @@ -12,6 +12,7 @@ static RUNTIME: OnceLock> = OnceLock::new(); pub(crate) fn tasks() -> Result { let built = RUNTIME.get_or_init(|| { Builder::new_multi_thread() + .worker_threads(worker_count()) .enable_all() .thread_name("tablepro-core") .build() @@ -50,3 +51,19 @@ impl Drop for AbortOnDrop { self.0.abort(); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_runtime_keeps_the_worker_count_of_a_database_client() { + tasks().expect("the runtime"); + let workers = RUNTIME + .get() + .and_then(|built| built.as_ref().ok()) + .map(|runtime| runtime.metrics().num_workers()); + + assert_eq!(workers, Some(worker_count())); + } +} diff --git a/core/crates/session/src/runtime/mod.rs b/core/crates/session/src/runtime/mod.rs index 20db131b5f..593ef9af0d 100644 --- a/core/crates/session/src/runtime/mod.rs +++ b/core/crates/session/src/runtime/mod.rs @@ -6,6 +6,7 @@ mod task_panic; mod task_result; mod tasks; mod watch_forwarder; +mod worker_count; pub use catch_panic::catch_panic; pub use latest_wins_writer::LatestWinsWriter; @@ -15,3 +16,4 @@ pub use task_panic::TaskPanic; pub use task_result::TaskResult; pub use tasks::Tasks; pub use watch_forwarder::WatchForwarder; +pub use worker_count::worker_count; diff --git a/core/crates/session/src/runtime/worker_count.rs b/core/crates/session/src/runtime/worker_count.rs new file mode 100644 index 0000000000..e01fac9968 --- /dev/null +++ b/core/crates/session/src/runtime/worker_count.rs @@ -0,0 +1,22 @@ +// A database client is not compute-bound: its work is waiting on sockets and files. More workers than this only adds +// context switching, and fewer than two lets one blocking call stall the rest. +pub fn worker_count() -> usize { + std::thread::available_parallelism().map_or(2, |count| clamped(count.get())) +} + +fn clamped(available: usize) -> usize { + available.clamp(2, 4) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn worker_count_clamps() { + assert_eq!(clamped(1), 2); + assert_eq!(clamped(3), 3); + assert_eq!(clamped(16), 4); + assert!((2..=4).contains(&worker_count())); + } +} diff --git a/core/crates/test-support/src/lib.rs b/core/crates/test-support/src/lib.rs index 8ebd8f1c0a..bf87b7d1e6 100644 --- a/core/crates/test-support/src/lib.rs +++ b/core/crates/test-support/src/lib.rs @@ -26,6 +26,7 @@ mod fake_secret_vault; mod fault; pub mod history_cases; mod replies; +mod round_trip_counter; mod snippet_text; mod stream_end; @@ -45,6 +46,7 @@ pub use fake_run::{FakeRun, RunWait}; pub use fake_secret_vault::{Call, FakeSecretVault}; pub use fault::FaultTransport; pub use replies::Replies; +pub use round_trip_counter::{RoundTripCounter, StreamTally}; pub use snippet_text::expand_snippet; pub use stream_end::StreamEnd; diff --git a/core/crates/test-support/src/round_trip_counter.rs b/core/crates/test-support/src/round_trip_counter.rs new file mode 100644 index 0000000000..c6ee28bff7 --- /dev/null +++ b/core/crates/test-support/src/round_trip_counter.rs @@ -0,0 +1,128 @@ +use std::io; +use std::pin::Pin; +use std::sync::{Arc, Mutex, PoisonError}; +use std::task::{Context, Poll}; + +use async_trait::async_trait; +use tablepro_core::{NetworkEndpoint, Transport, TransportError, TransportKind, TransportRoute, TransportStream}; +use tokio::io::{AsyncRead, AsyncWrite, ReadBuf}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Io { + Wrote, + Read, +} + +/// A transport that counts the round trips on each stream it opens: a +/// burst of writes after the client read something is one more wait on +/// the server. Its route is in-process, so an engine whose client dials +/// the route itself goes through `open()` as well. +#[derive(Debug)] +pub struct RoundTripCounter { + inner: Arc, + streams: Mutex>, +} + +/// The round trips one stream has made so far. +#[derive(Debug, Clone, Default)] +pub struct StreamTally(Arc>>); + +impl StreamTally { + pub fn round_trips(&self) -> usize { + let log = self.0.lock().unwrap_or_else(PoisonError::into_inner); + log.iter().filter(|io| **io == Io::Wrote).count() + } + + fn note(&self, io: Io) { + let mut log = self.0.lock().unwrap_or_else(PoisonError::into_inner); + if log.last() != Some(&io) { + log.push(io); + } + } +} + +impl RoundTripCounter { + pub fn new(inner: Arc) -> Self { + Self { + inner, + streams: Mutex::new(Vec::new()), + } + } + + /// Every stream opened so far, oldest first. + pub fn streams(&self) -> Vec { + self.streams.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + /// The round trips of every stream together. + pub fn round_trips(&self) -> usize { + self.streams().iter().map(StreamTally::round_trips).sum() + } +} + +#[async_trait] +impl Transport for RoundTripCounter { + fn service_endpoint(&self) -> &NetworkEndpoint { + self.inner.service_endpoint() + } + + fn kind(&self) -> TransportKind { + self.inner.kind() + } + + fn route(&self) -> TransportRoute { + TransportRoute::InProcess + } + + async fn open(&self) -> Result, TransportError> { + let inner = self.inner.open().await?; + let tally = StreamTally::default(); + self.streams + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(tally.clone()); + Ok(Box::new(Counted { inner, tally })) + } + + async fn reroute(&self, endpoint: NetworkEndpoint) -> Result, TransportError> { + self.inner.reroute(endpoint).await + } + + async fn explain_closed_stream(&self) -> Option { + None + } +} + +struct Counted { + inner: Box, + tally: StreamTally, +} + +impl AsyncRead for Counted { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let before = buf.filled().len(); + let read = Pin::new(&mut self.inner).poll_read(cx, buf); + if matches!(read, Poll::Ready(Ok(()))) && buf.filled().len() > before { + self.tally.note(Io::Read); + } + read + } +} + +impl AsyncWrite for Counted { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { + let written = Pin::new(&mut self.inner).poll_write(cx, buf); + if matches!(written, Poll::Ready(Ok(count)) if count > 0) { + self.tally.note(Io::Wrote); + } + written + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_flush(cx) + } + + fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_shutdown(cx) + } +} diff --git a/linux/changelog.d/4224.changed.md b/linux/changelog.d/4224.changed.md new file mode 100644 index 0000000000..17bc04dc47 --- /dev/null +++ b/linux/changelog.d/4224.changed.md @@ -0,0 +1 @@ +SQL editor queries on PostgreSQL and table pages wait on the server fewer times, which shows on slow links. diff --git a/linux/changelog.d/4224.fixed.1.md b/linux/changelog.d/4224.fixed.1.md new file mode 100644 index 0000000000..7cb69007c2 --- /dev/null +++ b/linux/changelog.d/4224.fixed.1.md @@ -0,0 +1 @@ +Scrolling a table with highlight rules was slow when its rows had many columns. diff --git a/linux/changelog.d/4224.fixed.md b/linux/changelog.d/4224.fixed.md new file mode 100644 index 0000000000..ad43e3f4b8 --- /dev/null +++ b/linux/changelog.d/4224.fixed.md @@ -0,0 +1 @@ +Deleting thousands of pasted rows at once froze the table tab. diff --git a/linux/crates/app/src/runtime/app_runtime.rs b/linux/crates/app/src/runtime/app_runtime.rs index 02b6a5c2da..f72af14115 100644 --- a/linux/crates/app/src/runtime/app_runtime.rs +++ b/linux/crates/app/src/runtime/app_runtime.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use tablepro_session::runtime::Tasks; +use tablepro_session::runtime::{Tasks, worker_count}; /// How long a shutdown waits for in-flight work before giving up. Long /// enough for a durable write to finish, short enough that quitting @@ -37,29 +37,10 @@ impl AppRuntime { } } -/// A database client is not compute-bound: its work is waiting on -/// sockets and files. More workers than this only adds context -/// switching, and fewer than two lets one blocking call stall the rest. -fn worker_count() -> usize { - std::thread::available_parallelism() - .map(|count| count.get().clamp(2, 4)) - .unwrap_or(2) -} - #[cfg(test)] mod tests { use super::*; - #[test] - fn worker_count_clamps() { - let clamp = |available: usize| available.clamp(2, 4); - - assert_eq!(clamp(1), 2); - assert_eq!(clamp(3), 3); - assert_eq!(clamp(16), 4); - assert!((2..=4).contains(&worker_count())); - } - #[test] #[expect( clippy::disallowed_methods, diff --git a/linux/crates/app/src/services/change_tracker.rs b/linux/crates/app/src/services/change_tracker.rs index 28b3d49fde..53fd4a3856 100644 --- a/linux/crates/app/src/services/change_tracker.rs +++ b/linux/crates/app/src/services/change_tracker.rs @@ -2,7 +2,9 @@ //! so an edit survives sort, filter and paging; the tracker is reached //! only through `TabTrackerHandle`. +use std::borrow::Borrow; use std::collections::{HashMap, HashSet, VecDeque}; +use std::hash::{Hash, Hasher}; use std::sync::Arc; use indexmap::IndexMap; @@ -49,6 +51,56 @@ impl RowKey { } } +/// A cell as the tracker's maps file it, `(RowKey, column)`. A draw looks +/// cells up by a borrowed row key, which finds the same entry without +/// copying the key. +trait CellKey { + fn row_key(&self) -> &RowKey; + fn col(&self) -> usize; +} + +impl CellKey for (RowKey, usize) { + fn row_key(&self) -> &RowKey { + &self.0 + } + + fn col(&self) -> usize { + self.1 + } +} + +impl CellKey for (&RowKey, usize) { + fn row_key(&self) -> &RowKey { + self.0 + } + + fn col(&self) -> usize { + self.1 + } +} + +impl<'a> Borrow for (RowKey, usize) { + fn borrow(&self) -> &(dyn CellKey + 'a) { + self + } +} + +// Field by field, as the tuple hashes, so both forms find one entry. +impl<'a> Hash for dyn CellKey + 'a { + fn hash(&self, state: &mut H) { + self.row_key().hash(state); + self.col().hash(state); + } +} + +impl<'a> PartialEq for dyn CellKey + 'a { + fn eq(&self, other: &Self) -> bool { + self.row_key() == other.row_key() && self.col() == other.col() + } +} + +impl<'a> Eq for dyn CellKey + 'a {} + /// A key value that hashes and compares. /// /// `ValueIdentity` wraps the value itself rather than a mirror of it: @@ -160,9 +212,20 @@ pub struct TabChangeTracker { /// A save that may have landed. Saving again could apply it twice, /// so nothing saves until the tab is reloaded or discarded. outcome_unknown: Option, + /// Moves on every change the handle makes, so what a draw worked out + /// from the tracker can tell it is out of date. + version: u64, } impl TabChangeTracker { + pub(crate) fn version(&self) -> u64 { + self.version + } + + pub(crate) fn note_change(&mut self) { + self.version = self.version.wrapping_add(1); + } + pub fn pending_count(&self) -> usize { self.drafts.len() + self.deletes.len() + self.updates.len() + self.set_aside.len() } @@ -224,7 +287,7 @@ impl TabChangeTracker { /// Whether the running save writes `col` of `row_key`, or inserts or /// deletes the row. pub fn cell_is_saving(&self, row_key: &RowKey, col: usize) -> bool { - self.is_being_written(&(row_key.clone(), col)) || self.row_in_flight(row_key) + self.saving.is_some() && (self.is_being_written(row_key, col) || self.row_in_flight(row_key)) } fn draft_in_flight(&self, draft_id: u64) -> bool { @@ -276,9 +339,8 @@ impl TabChangeTracker { value: CellInput, steps: &mut Vec, ) -> TrackerDelta { - let key = (row_key.clone(), col); - let before = self.updates.get(&key).cloned(); - let in_flight = self.is_being_written(&key); + let before = self.updates.get(&(&row_key, col) as &dyn CellKey).cloned(); + let in_flight = self.is_being_written(&row_key, col); let original = before.as_ref().map_or(Some(original), |edit| edit.original.clone()); let edit = CellEdit { original, @@ -331,15 +393,16 @@ impl TabChangeTracker { drafts: Vec, undo: &mut Vec, ) -> TrackerDelta { - let mut changed = Vec::new(); - let mut removed_drafts = Vec::new(); - for draft_id in drafts { - if !self.draft_in_flight(draft_id) && self.drop_draft(draft_id) { - removed_drafts.push(draft_id); - undo.retain_mut(|op| op.forget_draft(draft_id)); - } - changed.push(RowKey::Draft(draft_id)); - } + let mut removed = HashSet::new(); + let removed_drafts: Vec = drafts + .iter() + .copied() + .filter(|draft_id| { + self.drafts.contains_key(draft_id) && !self.draft_in_flight(*draft_id) && removed.insert(*draft_id) + }) + .collect(); + self.drop_drafts(&removed, undo); + let mut changed: Vec = drafts.into_iter().map(RowKey::Draft).collect(); let mut steps = Vec::new(); for (row_key, original) in persisted { changed.push(row_key.clone()); @@ -363,19 +426,20 @@ impl TabChangeTracker { } } - // The draft's undo and redo entries go with it, so Ctrl+Z cannot - // bring it back half-edited. - fn drop_draft(&mut self, draft_id: u64) -> bool { - if self.drafts.shift_remove(&draft_id).is_none() { - return false; + // A draft's undo and redo entries go with it, so Ctrl+Z cannot bring + // it back half-edited. One pass over each list for the whole set: a + // Delete can take every row of the largest paste. + fn drop_drafts(&mut self, removed: &HashSet, undo: &mut Vec) { + if removed.is_empty() { + return; } - self.updates - .retain(|(k, _), _| !matches!(k, RowKey::Draft(id) if *id == draft_id)); - self.deletes - .retain(|k, _| !matches!(k, RowKey::Draft(id) if *id == draft_id)); - self.undo.retain_mut(|op| op.forget_draft(draft_id)); - self.redo.retain_mut(|op| op.forget_draft(draft_id)); - true + let gone = |row_key: &RowKey| matches!(row_key, RowKey::Draft(draft_id) if removed.contains(draft_id)); + self.drafts.retain(|draft_id, _| !removed.contains(draft_id)); + self.updates.retain(|(row_key, _), _| !gone(row_key)); + self.deletes.retain(|row_key, _| !gone(row_key)); + self.undo.retain_mut(|op| op.forget_drafts(removed)); + self.redo.retain_mut(|op| op.forget_drafts(removed)); + undo.retain_mut(|op| op.forget_drafts(removed)); } fn track_draft_cell_edit( @@ -405,19 +469,20 @@ impl TabChangeTracker { Some((draft.cells.get(col)?.clone(), draft.touched)) } - fn is_being_written(&self, cell: &(RowKey, usize)) -> bool { - self.saving.as_ref().is_some_and(|save| save.written.contains_key(cell)) + fn is_being_written(&self, row_key: &RowKey, col: usize) -> bool { + self.saving + .as_ref() + .is_some_and(|save| save.written.contains_key(&(row_key, col) as &dyn CellKey)) } fn put_cell_edit(&mut self, row_key: &RowKey, col: usize, edit: Option) -> TrackerDelta { - let key = (row_key.clone(), col); // An edit equal to the fetched value is kept only while a save // writes the cell. Undo and redo wait for the save, so they find // such an edit no longer changing anything. - let edit = edit.filter(|edit| !edit.is_no_change() || self.is_being_written(&key)); + let edit = edit.filter(|edit| !edit.is_no_change() || self.is_being_written(row_key, col)); match edit { - Some(edit) => self.updates.insert(key, edit), - None => self.updates.remove(&key), + Some(edit) => self.updates.insert((row_key.clone(), col), edit), + None => self.updates.remove(&(row_key, col) as &dyn CellKey), }; self.delta(vec![row_key.clone()]) } @@ -601,7 +666,7 @@ impl TabChangeTracker { pub fn cell_marks(&self, row_key: &RowKey, col: usize) -> CellMarks { CellMarks { - row_deleted: self.deletes.contains_key(row_key), + row_deleted: !self.deletes.is_empty() && self.deletes.contains_key(row_key), state: self.cell_state(row_key, col), } } @@ -610,18 +675,20 @@ impl TabChangeTracker { if matches!(row_key, RowKey::Draft(_)) { return CellState::InsertDraft; } - if self.updates.contains_key(&(row_key.clone(), col)) { + if !self.updates.is_empty() && self.updates.contains_key(&(row_key, col) as &dyn CellKey) { return CellState::Modified; } CellState::Clean } - fn pending_input(&self, row_key: &RowKey, col: usize) -> Option { + /// The cell's pending input, `None` when it shows what was read. + pub fn pending_input(&self, row_key: &RowKey, col: usize) -> Option { match row_key { RowKey::Draft(draft_id) => self.draft_input(*draft_id, col), + RowKey::Persisted(_) if self.updates.is_empty() => None, RowKey::Persisted(_) => self .updates - .get(&(row_key.clone(), col)) + .get(&(row_key, col) as &dyn CellKey) .map(|edit| edit.pending.clone()), } } @@ -633,12 +700,10 @@ impl TabChangeTracker { if self.updates.is_empty() { return Vec::new(); } - let mut cell = (row_key.clone(), 0); (0..width) .filter_map(|col| { - cell.1 = col; self.updates - .get(&cell) + .get(&(row_key, col) as &dyn CellKey) .map(|edit| (col, edit.pending.value().cloned().unwrap_or(Value::Null))) }) .collect() @@ -1283,6 +1348,29 @@ mod tests { assert_eq!(marks(&draft, 0), (false, CellState::InsertDraft)); } + #[test] + fn a_cell_is_found_by_a_borrowed_key_of_many_parts() { + let mut t = TabChangeTracker::default(); + let key = rk(&[Value::Int(1), Value::Text("north".into())]); + t.apply(TrackerEdit::Cell { + row_key: key.clone(), + col: 2, + original: Value::Text("a".into()), + value: CellInput::Value(Value::Text("b".into())), + }); + let other = rk(&[Value::Int(1), Value::Text("south".into())]); + + assert_eq!(t.cell_marks(&key, 2).state, CellState::Modified); + assert_eq!( + t.pending_input(&key, 2), + Some(CellInput::Value(Value::Text("b".into()))) + ); + assert_eq!(t.cell_marks(&key, 1).state, CellState::Clean); + assert_eq!(t.cell_marks(&other, 2).state, CellState::Clean); + assert_eq!(t.pending_input(&other, 2), None); + assert_eq!(t.pending_values(&key, 3), vec![(2, Value::Text("b".into()))]); + } + #[test] fn delete_marks_row_pending() { let mut t = TabChangeTracker::default(); @@ -1915,6 +2003,62 @@ mod tests { assert!(elapsed < std::time::Duration::from_secs(2), "took {elapsed:?}"); } + #[test] + fn deleting_the_largest_paste_takes_one_pass_and_its_undo_entry() { + let mut t = TabChangeTracker::default(); + let rows = (0..tablepro_core::import::MAX_PASTED_ROWS) + .map(|row| vec![CellInput::Value(Value::Int(row as i64))]) + .collect(); + let drafts = t.apply(TrackerEdit::InsertDrafts { rows }).inserted_drafts; + // Select All reads the drafts as the grid shows them, newest on top. + let selected: Vec = drafts.iter().rev().copied().collect(); + + let started = std::time::Instant::now(); + let deleted = drop_drafts(&mut t, selected.clone()); + let elapsed = started.elapsed(); + + assert_eq!(deleted.removed_drafts, selected); + assert_eq!(deleted.changed.len(), tablepro_core::import::MAX_PASTED_ROWS); + assert_eq!(deleted.pending_count, 0); + assert!(t.drafts().is_empty()); + assert!(!t.can_undo(), "the paste's undo entry outlived every row it added"); + assert!(!t.can_redo()); + assert!(elapsed < std::time::Duration::from_secs(2), "took {elapsed:?}"); + } + + #[test] + fn a_delete_drops_its_drafts_once_and_keeps_the_rest_of_their_undo_steps() { + let mut t = TabChangeTracker::default(); + let inserted = t.apply(TrackerEdit::InsertDrafts { + rows: (1..=4).map(|id| vec![CellInput::Value(Value::Int(id))]).collect(), + }); + let [first, second, third, fourth] = inserted.inserted_drafts[..] else { + panic!("four drafts: {:?}", inserted.inserted_drafts); + }; + let _ = t.apply(TrackerEdit::DraftCell { + draft_id: second, + col: 0, + input: CellInput::Value(Value::Int(20)), + }); + + let deleted = drop_drafts(&mut t, vec![third, second, third, 99_999]); + + assert_eq!( + deleted.removed_drafts, + vec![third, second], + "each draft is reported once, in the order asked" + ); + assert_eq!( + deleted.changed, + [third, second, 99_999].map(RowKey::Draft), + "every row asked for draws again" + ); + assert_eq!(t.drafts().len(), 2); + let undone = t.undo().expect("the rest of the insert to undo"); + assert_eq!(undone.removed_drafts, vec![first, fourth]); + assert!(!t.can_undo(), "the edit to a deleted draft stayed in the history"); + } + #[test] fn one_delete_over_many_rows_is_one_undo_step() { let mut t = TabChangeTracker::default(); diff --git a/linux/crates/app/src/services/grid/highlight.rs b/linux/crates/app/src/services/grid/highlight.rs index 506b4bebbd..43db77aacf 100644 --- a/linux/crates/app/src/services/grid/highlight.rs +++ b/linux/crates/app/src/services/grid/highlight.rs @@ -126,6 +126,52 @@ struct Compiled { column: usize, kind: ColumnKind, rule: HighlightRule, + sensitive: bool, + operands: Operands, +} + +impl Compiled { + fn new(column: usize, definition: &GridColumn, rule: &HighlightRule) -> Self { + // ILIKE ignores case whatever the rule asks. + let sensitive = rule.case_sensitive && rule.op != FilterOp::Ilike; + let operand = |typed: &String| Operand::new(typed, sensitive); + let operands = match &rule.value { + None => Operands::Absent, + Some(FilterValue::Single(typed)) => Operands::Single(operand(typed)), + Some(FilterValue::Pair(low, high)) => Operands::Pair(operand(low), operand(high)), + Some(FilterValue::List(items)) => Operands::List(items.iter().map(operand).collect()), + }; + Self { + column, + kind: definition.column_type.kind.clone(), + rule: rule.clone(), + sensitive, + operands, + } + } +} + +/// A rule's operand, read once as the rules compile rather than for +/// every cell they are matched against. +struct Operand { + folded: String, + number: Option, +} + +impl Operand { + fn new(typed: &str, sensitive: bool) -> Self { + Self { + folded: fold(typed, sensitive).into_owned(), + number: typed.trim().parse().ok(), + } + } +} + +enum Operands { + Absent, + Single(Operand), + Pair(Operand, Operand), + List(Vec), } #[derive(Default)] @@ -145,11 +191,7 @@ impl HighlightRules { .enumerate() .filter(|(_, column)| column.name == rule.column) .nth(rule.column_occurrence) - .map(|(column, definition)| Compiled { - column, - kind: definition.column_type.kind.clone(), - rule: rule.clone(), - }) + .map(|(column, definition)| Compiled::new(column, definition, rule)) }) .collect(); let mut columns: Vec = compiled.iter().map(|rule| rule.column).collect(); @@ -172,7 +214,7 @@ impl HighlightRules { let Some(value) = cells.get(compiled.column) else { continue; }; - if !matches(&compiled.rule, &compiled.kind, value) { + if !matches(compiled, value) { continue; } match compiled.rule.target { @@ -187,73 +229,67 @@ impl HighlightRules { } } -fn matches(rule: &HighlightRule, kind: &ColumnKind, value: &HighlightValue<'_>) -> bool { +fn matches(compiled: &Compiled, value: &HighlightValue<'_>) -> bool { let text = match value { HighlightValue::Unavailable => return false, - HighlightValue::Null => return rule.op == FilterOp::IsNull, - HighlightValue::NonText => return rule.op == FilterOp::IsNotNull, + HighlightValue::Null => return compiled.rule.op == FilterOp::IsNull, + HighlightValue::NonText => return compiled.rule.op == FilterOp::IsNotNull, HighlightValue::Text(text) => text.as_ref(), }; - let sensitive = rule.case_sensitive; - let same = |a: &str, b: &str| compare(a, b, sensitive, kind.is_numeric()) == std::cmp::Ordering::Equal; - let single = || match &rule.value { - Some(FilterValue::Single(operand)) => Some(operand.as_str()), + let sensitive = compiled.sensitive; + let numeric = compiled.kind.is_numeric(); + let same = |operand: &Operand| compare(text, operand, sensitive, numeric).is_eq(); + let order = |operand: &Operand| compare(text, operand, sensitive, true); + let single = || match &compiled.operands { + Operands::Single(operand) => Some(operand), + _ => None, + }; + let list = || match &compiled.operands { + Operands::List(items) => Some(items), _ => None, }; - match rule.op { + match compiled.rule.op { FilterOp::IsNull => false, FilterOp::IsNotNull => true, - FilterOp::Eq => single().is_some_and(|operand| same(text, operand)), - FilterOp::NotEq => single().is_some_and(|operand| !same(text, operand)), - FilterOp::Lt => single().is_some_and(|operand| compare(text, operand, sensitive, true).is_lt()), - FilterOp::LtEq => single().is_some_and(|operand| compare(text, operand, sensitive, true).is_le()), - FilterOp::Gt => single().is_some_and(|operand| compare(text, operand, sensitive, true).is_gt()), - FilterOp::GtEq => single().is_some_and(|operand| compare(text, operand, sensitive, true).is_ge()), - FilterOp::Contains => single().is_some_and(|operand| fold(text, sensitive).contains(&fold(operand, sensitive))), - FilterOp::StartsWith => { - single().is_some_and(|operand| fold(text, sensitive).starts_with(&fold(operand, sensitive))) - } - FilterOp::EndsWith => { - single().is_some_and(|operand| fold(text, sensitive).ends_with(&fold(operand, sensitive))) - } - FilterOp::Like => single().is_some_and(|pattern| like(text, pattern, sensitive)), - FilterOp::NotLike => single().is_some_and(|pattern| !like(text, pattern, sensitive)), - FilterOp::Ilike => single().is_some_and(|pattern| like(text, pattern, false)), - FilterOp::In => match &rule.value { - Some(FilterValue::List(items)) => items.iter().any(|item| same(text, item)), - _ => false, - }, - FilterOp::NotIn => match &rule.value { - Some(FilterValue::List(items)) => !items.is_empty() && !items.iter().any(|item| same(text, item)), - _ => false, - }, - FilterOp::Between => match &rule.value { - Some(FilterValue::Pair(low, high)) => { - compare(text, low, sensitive, true).is_ge() && compare(text, high, sensitive, true).is_le() - } + FilterOp::Eq => single().is_some_and(same), + FilterOp::NotEq => single().is_some_and(|operand| !same(operand)), + FilterOp::Lt => single().is_some_and(|operand| order(operand).is_lt()), + FilterOp::LtEq => single().is_some_and(|operand| order(operand).is_le()), + FilterOp::Gt => single().is_some_and(|operand| order(operand).is_gt()), + FilterOp::GtEq => single().is_some_and(|operand| order(operand).is_ge()), + FilterOp::Contains => single().is_some_and(|operand| fold(text, sensitive).contains(&operand.folded)), + FilterOp::StartsWith => single().is_some_and(|operand| fold(text, sensitive).starts_with(&operand.folded)), + FilterOp::EndsWith => single().is_some_and(|operand| fold(text, sensitive).ends_with(&operand.folded)), + FilterOp::Like | FilterOp::Ilike => single().is_some_and(|pattern| like(text, &pattern.folded, sensitive)), + FilterOp::NotLike => single().is_some_and(|pattern| !like(text, &pattern.folded, sensitive)), + FilterOp::In => list().is_some_and(|items| items.iter().any(same)), + FilterOp::NotIn => list().is_some_and(|items| !items.is_empty() && !items.iter().any(same)), + FilterOp::Between => match &compiled.operands { + Operands::Pair(low, high) => order(low).is_ge() && order(high).is_le(), _ => false, }, } } -fn fold(text: &str, sensitive: bool) -> String { +fn fold(text: &str, sensitive: bool) -> Cow<'_, str> { if sensitive { - text.to_owned() + Cow::Borrowed(text) } else { - text.to_lowercase() + Cow::Owned(text.to_lowercase()) } } -fn compare(a: &str, b: &str, sensitive: bool, numeric: bool) -> std::cmp::Ordering { - if numeric && let (Ok(x), Ok(y)) = (a.trim().parse::(), b.trim().parse::()) { - return x.cmp_numeric(&y); +fn compare(text: &str, operand: &Operand, sensitive: bool, numeric: bool) -> std::cmp::Ordering { + if numeric && let (Ok(x), Some(y)) = (text.trim().parse::(), operand.number.as_ref()) { + return x.cmp_numeric(y); } - fold(a, sensitive).cmp(&fold(b, sensitive)) + Ord::cmp(&*fold(text, sensitive), operand.folded.as_str()) } +/// `pattern` comes folded as the rule compiled it; only `text` folds here. fn like(text: &str, pattern: &str, sensitive: bool) -> bool { let text = fold(text, sensitive); - let pattern: Vec = fold(pattern, sensitive).chars().collect(); + let pattern: Vec = pattern.chars().collect(); // Each prefix is visited once per character, so repeated wildcards cannot branch exponentially. let mut reachable = vec![false; pattern.len() + 1]; reachable[0] = true; @@ -336,6 +372,18 @@ mod tests { assert!(!highlights(rule(FilterOp::NotLike, single("e%")), Some("error"))); } + #[test] + fn ilike_ignores_case_even_in_a_case_sensitive_rule() { + let mut ilike = rule(FilterOp::Ilike, single("ER%")); + ilike.case_sensitive = true; + let mut like = rule(FilterOp::Like, single("ER%")); + like.case_sensitive = true; + + assert!(highlights(ilike, Some("error"))); + assert!(!highlights(like.clone(), Some("error"))); + assert!(highlights(like, Some("ERROR"))); + } + #[test] fn lists_and_ranges() { let list = Some(FilterValue::List(vec!["a".to_owned(), "b".to_owned()])); diff --git a/linux/crates/app/src/services/tab_tracker_handle.rs b/linux/crates/app/src/services/tab_tracker_handle.rs index efcdddf388..6c3b76dbc4 100644 --- a/linux/crates/app/src/services/tab_tracker_handle.rs +++ b/linux/crates/app/src/services/tab_tracker_handle.rs @@ -1,4 +1,4 @@ -use std::cell::RefCell; +use std::cell::{RefCell, RefMut}; use std::rc::Rc; use tablepro_core::dml::BuildSqlError; @@ -19,10 +19,26 @@ use crate::workspace::{PendingState, SaveOutcome, SaveRejection, SaveSnapshot}; pub struct TabTrackerHandle(Rc>); impl TabTrackerHandle { + // Every change goes through here, so the version moves with each one. + fn write(&self) -> RefMut<'_, TabChangeTracker> { + let mut tracker = self.0.borrow_mut(); + tracker.note_change(); + tracker + } + + /// Moves on every change, for a cache of what the tracker held. + pub fn version(&self) -> u64 { + self.0.borrow().version() + } + pub fn cell_overlay(&self, row_key: &RowKey, col: usize) -> CellOverlay { self.0.borrow().cell_overlay(row_key, col) } + pub fn pending_input(&self, row_key: &RowKey, col: usize) -> Option { + self.0.borrow().pending_input(row_key, col) + } + pub fn cell_marks(&self, row_key: &RowKey, col: usize) -> CellMarks { self.0.borrow().cell_marks(row_key, col) } @@ -37,12 +53,12 @@ impl TabTrackerHandle { #[must_use = "the grid shows stale rows until the delta is applied"] pub fn apply(&self, edit: TrackerEdit) -> TrackerDelta { - self.0.borrow_mut().apply(edit) + self.write().apply(edit) } #[must_use = "the grid shows stale rows until the delta is applied"] pub fn apply_all(&self, edits: Vec) -> TrackerDelta { - self.0.borrow_mut().apply_all(edits) + self.write().apply_all(edits) } pub fn cell_is_saving(&self, row_key: &RowKey, col: usize) -> bool { @@ -56,12 +72,12 @@ impl TabTrackerHandle { #[must_use = "the grid shows stale rows until the delta is applied"] pub fn undo(&self) -> Option { - self.0.borrow_mut().undo() + self.write().undo() } #[must_use = "the grid shows stale rows until the delta is applied"] pub fn redo(&self) -> Option { - self.0.borrow_mut().redo() + self.write().redo() } pub fn pending_count(&self) -> usize { @@ -102,12 +118,12 @@ impl TabTrackerHandle { #[must_use = "the grid shows stale rows until the delta is applied"] pub fn discard(&self) -> TrackerDelta { - self.0.borrow_mut().discard() + self.write().discard() } #[must_use = "the grid shows stale rows until the delta is applied"] pub fn mark_error_row(&self, row_key: RowKey) -> TrackerDelta { - self.0.borrow_mut().mark_error_row(row_key) + self.write().mark_error_row(row_key) } pub fn is_error_row(&self, row_key: &RowKey) -> bool { @@ -118,7 +134,7 @@ impl TabTrackerHandle { /// against. #[must_use = "the grid shows stale rows until the delta is applied"] pub(crate) fn set_target(&self, target: SaveTarget) -> TrackerDelta { - self.0.borrow_mut().set_target(target) + self.write().set_target(target) } pub(crate) fn pending_state(&self) -> PendingState { @@ -130,12 +146,12 @@ impl TabTrackerHandle { } pub(crate) fn begin_save(&self) -> Result, SaveRejection> { - self.0.borrow_mut().begin_save() + self.write().begin_save() } #[must_use = "the grid shows stale rows until the delta is applied"] pub(crate) fn settle_save(&self, outcome: &SaveOutcome, snapshot: SaveSnapshot) -> TrackerDelta { - self.0.borrow_mut().settle_save(outcome, snapshot) + self.write().settle_save(outcome, snapshot) } } @@ -362,6 +378,25 @@ mod tests { assert_eq!(tracker.cell_overlay(&row(1), 1).shown(fetched), Value::Text("b".into())); } + #[test] + fn every_change_moves_the_version_and_a_read_does_not() { + let tracker = TabTrackerHandle::default(); + let start = tracker.version(); + let _ = tracker.cell_marks(&row(1), 1); + assert_eq!(tracker.version(), start); + + edit_cell(&tracker, 1, 2); + let edited = tracker.version(); + let _ = tracker.discard(); + + assert_ne!(edited, start); + assert_ne!( + tracker.version(), + edited, + "a discard whose delta nobody applies still moves it" + ); + } + #[test] fn clones_share_one_tracker() { let tracker = TabTrackerHandle::default(); diff --git a/linux/crates/app/src/services/undo_op.rs b/linux/crates/app/src/services/undo_op.rs index e32a877f77..945aa62b84 100644 --- a/linux/crates/app/src/services/undo_op.rs +++ b/linux/crates/app/src/services/undo_op.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use tablepro_core::edit::CellInput; use super::change_tracker::{CellEdit, DeleteMark, DraftRow, RowKey}; @@ -120,23 +122,23 @@ impl UndoOp { } } - /// Drops what the step says about a draft that is gone, and returns + /// Drops what the step says about drafts that are gone, and returns /// whether anything is left of the step. - pub fn forget_draft(&mut self, draft_id: u64) -> bool { - let gone = RowKey::Draft(draft_id); + pub fn forget_drafts(&mut self, gone: &HashSet) -> bool { + let is_gone = |row_key: &RowKey| matches!(row_key, RowKey::Draft(draft_id) if gone.contains(draft_id)); match self { UndoOp::Group { steps } => { - steps.retain_mut(|step| step.forget_draft(draft_id)); + steps.retain_mut(|step| step.forget_drafts(gone)); !steps.is_empty() } UndoOp::Insert { drafts } => { - drafts.retain(|draft| draft.draft_id != draft_id); + drafts.retain(|draft| !gone.contains(&draft.draft_id)); !drafts.is_empty() } - UndoOp::DraftCell { draft_id: id, .. } => *id != draft_id, - UndoOp::CellEdit { row_key, .. } => *row_key != gone, + UndoOp::DraftCell { draft_id, .. } => !gone.contains(&*draft_id), + UndoOp::CellEdit { row_key, .. } => !is_gone(row_key), UndoOp::Delete { steps } => { - steps.retain(|step| step.row_key != gone); + steps.retain(|step| !is_gone(&step.row_key)); !steps.is_empty() } } diff --git a/linux/crates/app/src/ui/data_grid/grid_row.rs b/linux/crates/app/src/ui/data_grid/grid_row.rs index 6cba43087b..aeb55cd906 100644 --- a/linux/crates/app/src/ui/data_grid/grid_row.rs +++ b/linux/crates/app/src/ui/data_grid/grid_row.rs @@ -6,6 +6,7 @@ use gtk4::prelude::*; use gtk4::subclass::prelude::*; use tablepro_core::Value; +use super::highlight::RowTint; use crate::services::change_tracker::RowKey; use crate::services::grid::GridRowId; @@ -64,6 +65,10 @@ mod imp { /// Moves only when the slot is given another row, so anything that /// captured the row (a menu, an editor) can tell it is gone. pub generation: Cell, + /// The row's highlight, worked out once for all of its cells. + pub(super) tint: RefCell>, + #[cfg(test)] + pub(super) tints_worked_out: Cell, } impl Default for GridRow { @@ -72,6 +77,9 @@ mod imp { data: RefCell::new(RowData::result(Arc::from([]))), revision: Cell::new(0), generation: Cell::new(0), + tint: RefCell::new(None), + #[cfg(test)] + tints_worked_out: Cell::new(0), } } } @@ -112,6 +120,12 @@ impl GridRow { self.imp().data.borrow().id.tracker_key() } + /// Calls `read` with the row's id, lent rather than copied, since a + /// draw reads it for every cell. + pub fn with_id(&self, read: impl FnOnce(&GridRowId) -> R) -> R { + read(&self.imp().data.borrow().id) + } + pub fn draft_id(&self) -> Option { self.imp().data.borrow().id.draft_id() } @@ -154,6 +168,21 @@ impl GridRow { self.notify_revision(); } + pub(super) fn tint(&self) -> &RefCell> { + &self.imp().tint + } + + #[cfg(test)] + pub(super) fn count_tint(&self) { + let count = &self.imp().tints_worked_out; + count.set(count.get() + 1); + } + + #[cfg(test)] + pub(super) fn tints_worked_out(&self) -> u32 { + self.imp().tints_worked_out.get() + } + /// A hold on the row as it is now, which stops resolving once the /// slot is given another row. pub fn pin(&self) -> PinnedRow { diff --git a/linux/crates/app/src/ui/data_grid/highlight.rs b/linux/crates/app/src/ui/data_grid/highlight.rs index 5d7cc5542d..fbcd1d004c 100644 --- a/linux/crates/app/src/ui/data_grid/highlight.rs +++ b/linux/crates/app/src/ui/data_grid/highlight.rs @@ -10,6 +10,36 @@ use tablepro_linux_platform::settings::DateFormat; use super::{DataGrid, GridRow}; use crate::services::grid::display_text::whole_text; use crate::services::grid::{HighlightColor, HighlightRule, HighlightRules, HighlightValue}; +use crate::services::tab_tracker_handle::TabTrackerHandle; + +/// A row's highlight, worked out once and drawn by each of its cells. +pub(super) struct RowTint { + basis: TintBasis, + colours: Vec>, + /// What `RowHighlight::color_of` gives a column past the grid's. + beyond: Option, +} + +impl RowTint { + fn colour(&self, column: usize) -> Option { + self.colours.get(column).copied().unwrap_or(self.beyond) + } +} + +/// What a row's highlight was worked out from. Each part moves when what +/// it reads changes, a tracker change no row was redrawn for included. +struct TintBasis { + revision: u32, + // Held, not only compared by address, so no later compile can reuse it. + rules: Rc, + tracker: Option, +} + +impl TintBasis { + fn holds_for(&self, other: &Self) -> bool { + self.revision == other.revision && Rc::ptr_eq(&self.rules, &other.rules) && self.tracker == other.tracker + } +} impl DataGrid { pub fn set_highlight_rules(&self, rules: Vec) { @@ -44,12 +74,31 @@ impl DataGrid { if rules.is_empty() { return None; } + let basis = TintBasis { + revision: row.revision(), + rules, + tracker: self + .config() + .and_then(|config| config.tracker.as_ref().map(TabTrackerHandle::version)), + }; + if let Some(tint) = row.tint().borrow().as_ref().filter(|tint| tint.basis.holds_for(&basis)) { + return tint.colour(column); + } + let tint = self.work_out_tint(row, basis); + let colour = tint.colour(column); + row.tint().replace(Some(tint)); + colour + } + + fn work_out_tint(&self, row: &GridRow, basis: TintBasis) -> RowTint { + #[cfg(test)] + row.count_tint(); let columns = self.grid_columns(); let mut values = vec![HighlightValue::Unavailable; columns.len()]; let tracked = self.tracked(); let mut scratch = String::new(); let stored = row.cells(); - for read in rules.columns() { + for read in basis.rules.columns() { let Some(definition) = columns.get(read) else { continue; }; @@ -71,7 +120,12 @@ impl DataGrid { } }); } - rules.highlight(&values).color_of(column) + let found = basis.rules.highlight(&values); + RowTint { + colours: (0..columns.len()).map(|column| found.color_of(column)).collect(), + beyond: found.row, + basis, + } } } @@ -82,7 +136,81 @@ mod tests { use super::*; use crate::services::tracker_edit::TrackerEdit; - use crate::ui::data_grid::test_grid::{Grid, key, two_text_columns}; + use crate::test_support::drain_main_context; + use crate::ui::data_grid::GridCell; + use crate::ui::data_grid::test_grid::{Grid, key, two_text_columns, two_text_values}; + + fn name_is(value: &str) -> HighlightRule { + let mut rule = HighlightRule::new("name"); + rule.value = Some(FilterValue::Single(value.into())); + rule + } + + fn tinted(cell: &GridCell) -> bool { + cell.has_css_class(HighlightColor::Yellow.css_class()) + } + + fn rename_first(grid: &Grid, from: &str, to: &str) { + let delta = grid.tracker.apply(TrackerEdit::Cell { + row_key: key(1), + col: 1, + original: Value::Text(from.into()), + value: CellInput::Value(Value::Text(to.into())), + }); + grid.grid.apply_delta(&delta); + } + + #[gtk4::test] + fn a_row_works_out_its_highlight_once_for_all_of_its_cells() { + let grid = Grid::open(two_text_columns(), two_text_values()); + grid.grid.set_highlight_rules(vec![name_is("A")]); + drain_main_context(); + let row = grid.grid.store().row(0).expect("a row"); + let cells = [grid.cell_of(1, 0), grid.cell_of(1, 1)]; + let drawn: Vec = cells.iter().map(GridCell::render_count).collect(); + let worked_out = row.tints_worked_out(); + + row.bump_revision(); + + for (cell, before) in cells.iter().zip(drawn) { + assert_eq!(cell.render_count(), before + 1, "every cell of the row drew again"); + assert!(tinted(cell), "a cell of the row lost its colour"); + } + assert_eq!(row.tints_worked_out(), worked_out + 1, "each cell ran the rules again"); + } + + #[gtk4::test] + fn an_edit_to_the_rule_column_recolours_the_other_cells_of_its_row() { + let grid = Grid::open(two_text_columns(), two_text_values()); + grid.grid.set_highlight_rules(vec![name_is("A")]); + drain_main_context(); + let id_cell = grid.cell_of(1, 0); + assert!(tinted(&id_cell)); + + rename_first(&grid, "A", "Z"); + assert!( + !tinted(&id_cell), + "the id kept the colour of a name the row no longer has" + ); + + grid.grid.undo(); + assert!(tinted(&id_cell), "undo left the id without its colour"); + } + + #[gtk4::test] + fn a_tracker_change_no_row_was_redrawn_for_reaches_the_next_draw() { + let grid = Grid::open(two_text_columns(), two_text_values()); + grid.grid.set_highlight_rules(vec![name_is("Z")]); + rename_first(&grid, "A", "Z"); + let id_cell = grid.cell_of(1, 0); + assert!(tinted(&id_cell)); + + // As a close prompt's Discard leaves the grid until the next page lands. + let _ = grid.tracker.discard(); + id_cell.render(); + + assert!(!tinted(&id_cell), "the draw kept the colour of the discarded edit"); + } #[gtk4::test] fn highlights_read_past_the_display_cut_and_follow_pending_values() { diff --git a/linux/crates/app/src/ui/data_grid/tracked_values.rs b/linux/crates/app/src/ui/data_grid/tracked_values.rs index 4ad39aec60..cd82d84e03 100644 --- a/linux/crates/app/src/ui/data_grid/tracked_values.rs +++ b/linux/crates/app/src/ui/data_grid/tracked_values.rs @@ -143,8 +143,15 @@ impl TrackedValues { !self.read_only && self.tracker.is_some() } - fn tracked_key(&self, row: &GridRow) -> Option<(&TabTrackerHandle, RowKey)> { - Some((self.tracker.as_ref()?, row.tracker_key()?)) + /// Calls `read` with the tracker and the row's key, lent from the row + /// rather than copied: every draw reads it several times per cell. + fn with_key(&self, row: &GridRow, read: impl FnOnce(&TabTrackerHandle, &RowKey) -> R) -> Option { + let tracker = self.tracker.as_ref()?; + row.with_id(|id| match id { + GridRowId::Persisted(key) => Some(read(tracker, key)), + GridRowId::Draft(draft_id) => Some(read(tracker, &RowKey::Draft(*draft_id))), + GridRowId::Result | GridRowId::Unkeyed => None, + }) } /// Why the cell cannot change, `None` when it can. @@ -155,33 +162,32 @@ impl TrackedValues { /// `lock` for a cell whose mark the caller already read. pub fn lock_marked(&self, column: &GridColumn, row: &GridRow, row_deleted: bool) -> Option { - if let (Some(refusal), GridRowId::Result) = (self.result_lock, row.id()) { - return Some(CellLock::Result(refusal)); - } - if let Err(lock) = cell_editability(column, &row.id(), &self.identity, self.read_only, row_deleted) { - return Some(lock); + let locked = row.with_id(|id| { + if let (Some(refusal), GridRowId::Result) = (self.result_lock, id) { + return Some(CellLock::Result(refusal)); + } + cell_editability(column, id, &self.identity, self.read_only, row_deleted).err() + }); + if locked.is_some() { + return locked; } - self.tracked_key(row) - .is_some_and(|(tracker, key)| tracker.cell_is_saving(&key, column.index)) - .then_some(CellLock::Saving) + let saving = self.with_key(row, |tracker, key| tracker.cell_is_saving(key, column.index)); + (saving == Some(true)).then_some(CellLock::Saving) } pub fn row_state(&self, row: &GridRow) -> RowPendingState { - self.tracked_key(row) - .map_or(RowPendingState::Unchanged, |(tracker, key)| { - RowPendingState::new(tracker.row_state(&key), tracker.is_error_row(&key)) - }) + self.with_key(row, |tracker, key| { + RowPendingState::new(tracker.row_state(key), tracker.is_error_row(key)) + }) + .unwrap_or(RowPendingState::Unchanged) } pub fn mark(&self, row: &GridRow, column: usize) -> CellMark { - match self.tracked_key(row) { - Some((tracker, key)) => { - let marks = tracker.cell_marks(&key, column); - CellMark { - state: marks.state, - row_deleted: marks.row_deleted, - } - } + match self.with_key(row, |tracker, key| tracker.cell_marks(key, column)) { + Some(marks) => CellMark { + state: marks.state, + row_deleted: marks.row_deleted, + }, None => CellMark { state: CellState::Clean, row_deleted: false, @@ -194,8 +200,8 @@ impl TrackedValues { /// as NULL. pub fn with_value(&self, row: &GridRow, column: usize, f: impl FnOnce(&Value, Option) -> R) -> R { let pending = self - .tracked_key(row) - .and_then(|(tracker, key)| tracker.cell_overlay(&key, column).pending); + .with_key(row, |tracker, key| tracker.pending_input(key, column)) + .flatten(); match pending { Some(CellInput::Value(value)) => f(&value, None), Some(CellInput::Default) => f(&Value::Null, Some(ServerValue::Default)), @@ -220,13 +226,12 @@ impl TrackedValues { /// The row as the grid shows it. A draft's cell left to the server /// reads as NULL. pub fn snapshot(&self, row: &GridRow) -> RowSnapshot { - if let (GridRowId::Draft(draft_id), Some(tracker)) = (row.id(), self.tracker.as_ref()) { + if let (Some(draft_id), Some(tracker)) = (row.draft_id(), self.tracker.as_ref()) { return RowSnapshot::new(tracker.draft_values(draft_id).unwrap_or_default().into(), Vec::new()); } let cells = row.cells().unwrap_or_else(|| Arc::from([])); let overrides = self - .tracked_key(row) - .map(|(tracker, key)| tracker.pending_values(&key, cells.len())) + .with_key(row, |tracker, key| tracker.pending_values(key, cells.len())) .unwrap_or_default(); RowSnapshot::new(cells, overrides) } @@ -236,8 +241,8 @@ impl TrackedValues { pub fn column_snapshot(&self, row: &GridRow, column: usize) -> RowSnapshot { let cells = row.cells().unwrap_or_else(|| Arc::from([])); let pending = self - .tracked_key(row) - .and_then(|(tracker, key)| tracker.cell_overlay(&key, column).pending) + .with_key(row, |tracker, key| tracker.pending_input(key, column)) + .flatten() .map(|input| (column, input.value().cloned().unwrap_or(Value::Null))); RowSnapshot::new(cells, pending.into_iter().collect()) } @@ -245,7 +250,7 @@ impl TrackedValues { /// Like `snapshot`, except a cell left to the server stays `Default`, /// a draft's or a saved row's pending one. pub fn inputs(&self, row: &GridRow) -> Vec { - if let (GridRowId::Draft(draft_id), Some(tracker)) = (row.id(), self.tracker.as_ref()) { + if let (Some(draft_id), Some(tracker)) = (row.draft_id(), self.tracker.as_ref()) { return tracker.draft_inputs(draft_id).unwrap_or_default(); } let width = row.cells().map_or(0, |cells| cells.len()); @@ -255,10 +260,10 @@ impl TrackedValues { /// Whether the cell shows the value as it was read, with no edit over /// it. pub fn shows_read_value(&self, row: &GridRow, column: usize) -> bool { - !matches!(row.id(), GridRowId::Draft(_)) + row.draft_id().is_none() && self - .tracked_key(row) - .is_none_or(|(tracker, key)| tracker.cell_overlay(&key, column).pending.is_none()) + .with_key(row, |tracker, key| tracker.pending_input(key, column).is_none()) + .unwrap_or(true) } }