diff --git a/Cargo.lock b/Cargo.lock index f7a8637eb..123cae836 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -524,9 +524,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "calamine" -version = "0.36.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3" +checksum = "c9ca341b94bc4b29e08970b1a03914fad2dfe177e2ecb67fcd6de9046004aafb" dependencies = [ "atoi_simd", "byteorder", @@ -534,9 +534,9 @@ dependencies = [ "encoding_rs", "fast-float2", "log", - "quick-xml 0.41.0", + "quick-xml 0.42.0", "serde", - "zip 8.6.0", + "zip 9.0.5", ] [[package]] @@ -3423,6 +3423,7 @@ version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b" dependencies = [ + "encoding_rs", "memchr", ] @@ -6293,14 +6294,15 @@ dependencies = [ [[package]] name = "zip" -version = "8.6.0" +version = "9.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" dependencies = [ "crc32fast", "flate2", "indexmap", "memchr", + "nix", "typed-path", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index 5b65de568..fc8457022 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -66,7 +66,7 @@ text-splitter = { version = "0.32", features = ["tiktoken-rs"] } tiktoken-rs = "0.12" # 句子边界按 Unicode 标准（UAX #29）切，不自己写标点规则 pdf-extract = "0.12" -calamine = "0.36" +calamine = "0.37" pgvector = { version = "0.4", features = ["sqlx"] } # `gzip` 不是可选项：一次嵌入回 64×1024 个浮点数，写成 JSON 文本每个数字占 # 21 字节而信息量只有 4 字节，一次响应 1.33 MB。实测同一条链路上不压缩 62 秒、 diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index c88fc1b7c..2d09f0936 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -82,7 +82,7 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**，不是最大的编号（守卫 `schema_version_policy_compares_against_current` // 按个数比）：编号有空缺时两者不同——0071 由一个开放 PR 占着，0072 先落，个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 102; +const CURRENT_SCHEMA_VERSION: u32 = 103; fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 1702f84b4..9b39ff747 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -860,6 +860,9 @@ pub struct EntityFact { /// 终点是时间线推出来的（`facts.end_derived`）：后一条事实开始时引擎把它关上，原文没说 /// 这一天（#970） pub end_derived: bool, + /// `valid_to` 仍是存的边界；true 说原文讲的是在它之前结束的那个时段（`facts.end_after_period`， + /// #994）：「2024 年离开」存 2025-01-01，原文说的是 2024 + pub end_after_period: bool, /// 区间是人改过的（#970）：`corrected_ends` 有值。与 `corrected` 不同，那一位是任何一次 /// 改写：自动接续、审阅裁决、人手改 pub time_corrected: bool, diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs index 686b59b3c..a80d11ce5 100644 --- a/crates/utopia-ingest/src/parsers.rs +++ b/crates/utopia-ingest/src/parsers.rs @@ -7,6 +7,11 @@ use std::io::{Cursor, Read, Write}; use std::process::Command; const SPREADSHEET_ROW_LIMIT: usize = 2_000; +// Each retained cell passes through a 40-byte GridRow entry and two 48-byte renderer +// Cells, besides source values, strings and Vec spare capacity (~370 B/cell in a +// full dense import). Bound actual records/segments, not their empty column spans: +// a 2,000 x 150 sheet uses 300,000 entries and fits without losing any rows. +const XLSX_SEGMENT_LIMIT: usize = 1_000_000; const CSV_RECORD_LIMIT: usize = 10_000; /// 文本解码：chardetng 探测编码（覆盖 GBK/GB18030/BIG5 等中文常见编码）。 @@ -773,53 +778,200 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<(String, Vec { + let Ok(sheet) = xlsx_sheet(book, &sheet_name, XLSX_SEGMENT_LIMIT) else { + continue; + }; + sheet + } + _ => { + let Ok(range) = workbook.worksheet_range(&sheet_name) else { + continue; + }; + let merges = match &mut workbook { + calamine::Sheets::Xls(book) => book + .merge_cells_by_sheet_name(&sheet_name) + .unwrap_or_default(), + _ => Vec::new(), + }; + let (rows, headers) = spreadsheet_grid(&range, &merges); + let text = crate::table::render_grid_with_headers(&rows, headers) + .map(|text| format!("{text}\n")) + .unwrap_or_else(|| { + let mut text = String::new(); + for row in &rows { + let cells: Vec<&str> = row.iter().map(|(s, _, _)| s.as_str()).collect(); + text.push_str(&cells.join("\t")); + text.push('\n'); + } + text + }); + ( + text, + range.height().min(SPREADSHEET_ROW_LIMIT), + range.height(), + ) + } }; - if range.is_empty() { + if rows_read == 0 { continue; } - // xlsx/xls 把 merge 单独给出；xlsb/ods 的 calamine reader 没有这一步，按无 merge 读。 - let merges = match &mut workbook { - calamine::Sheets::Xlsx(book) => book - .merge_cells_by_sheet_name(&sheet_name) - .unwrap_or_default(), - calamine::Sheets::Xls(book) => book - .merge_cells_by_sheet_name(&sheet_name) - .unwrap_or_default(), - _ => Vec::new(), - }; out.push_str(&format!("\n# Sheet: {sheet_name}\n\n")); - let rows_total = range.height(); - if rows_total > SPREADSHEET_ROW_LIMIT { - let rows_read = SPREADSHEET_ROW_LIMIT; + if rows_total > rows_read { warnings.push(crate::ParseWarning { kind: crate::ParseWarning::SPREADSHEET_ROWS_TRUNCATED, detail: serde_json::json!({ - "sheet": sheet_name.clone(), + "sheet": sheet_name, "rows_read": rows_read, "rows_total": rows_total, "rows_omitted": rows_total - rows_read, }), }); } - // 每张表按网格渲染成带列头的 Markdown 表；一格字都没有的表退回制表符分隔 - let (rows, headers) = spreadsheet_grid(&range, &merges); - match crate::table::render_grid_with_headers(&rows, headers) { - Some(md) => { - out.push_str(&md); - out.push('\n'); + out.push_str(&text); + } + Ok((out, warnings)) +} + +fn xlsx_sheet( + book: &mut calamine::Xlsx>>, + sheet: &str, + budget: usize, +) -> anyhow::Result<(String, usize, usize)> { + let mut start_row = u32::MAX; + let mut end_row = 0; + let mut row_counts = vec![0usize; SPREADSHEET_ROW_LIMIT]; + let mut reader = book.worksheet_cells_reader(sheet)?; + // Keep only the first 2,000 physical rows' counts, even if XML cells arrive + // out of order. The full row bounds survive so truncation remains honest. + while let Some(cell) = reader.next_cell()? { + if cell.get_value() == &calamine::DataRef::Empty { + continue; + } + let (row, col) = cell.get_position(); + anyhow::ensure!( + row < 1_048_576 && col < 16_384, + "Cell exceeds the Excel worksheet limits" + ); + if row < start_row { + let shift = (start_row - row).min(SPREADSHEET_ROW_LIMIT as u32) as usize; + row_counts.rotate_right(shift); + row_counts[..shift].fill(0); + start_row = row; + } + end_row = end_row.max(row); + let offset = (row - start_row) as usize; + if let Some(count) = row_counts.get_mut(offset) { + // Repeated coordinates overwrite values below, so even malformed + // XML cannot retain more than one physical row's worth of cells. + *count = (*count + 1).min(16_384); + } + } + drop(reader); + if start_row == u32::MAX { + return Ok((String::new(), 0, 0)); + } + let rows_total = (end_row - start_row + 1) as usize; + let mut height = rows_total.min(SPREADSHEET_ROW_LIMIT); + let mut count = 0; + for (row, &cells) in row_counts[..height].iter().enumerate() { + if cells > budget - count { + height = row; + break; + } + count += cells; + } + // Counts bound this allocation before any source values are retained. A + // distant note adds one entry, not thousands of empty Data/String objects. + let mut values = vec![std::collections::BTreeMap::::new(); height]; + let mut reader = book.worksheet_cells_reader(sheet)?; + while let Some(cell) = reader.next_cell()? { + let (row, col) = cell.get_position(); + let offset = row.saturating_sub(start_row) as usize; + if row >= start_row && offset < height && cell.get_value() != &calamine::DataRef::Empty { + let text = spreadsheet_cell_text(Some(&calamine::Data::from(cell.get_value().clone()))); + values[offset].insert(col, text); + } + } + drop(reader); + let merges = book.merge_cells_by_sheet_name(sheet).unwrap_or_default(); + loop { + let start_col = values + .iter() + .filter_map(|row| row.keys().next()) + .min() + .copied() + .unwrap(); + let end_col = values + .iter() + .filter_map(|row| row.keys().next_back()) + .max() + .copied() + .unwrap(); + let width = (end_col - start_col + 1) as usize; + let text_at = |row: usize, col: usize| { + values[row] + .get(&(start_col + col as u32)) + .cloned() + .unwrap_or_default() + }; + let regions = spreadsheet_regions( + (start_row, start_col), + (start_row + height as u32 - 1, end_col), + &merges, + text_at, + ); + let mut owner = vec![usize::MAX; width]; + let mut rows = Vec::with_capacity(height); + let mut segments = 0; + for row in 0..height { + let cells = spreadsheet_row( + row, + width, + ®ions, + &mut owner, + |col| text_at(row, col), + true, + ); + if cells.len() > budget - segments { + break; } + segments += cells.len(); + rows.push(cells); + } + if rows.len() != height { + height = rows.len(); + values.truncate(height); + // Dropped rows cannot leave a distant column in the kept window's + // geometry. Reproject merges too, retaining their physical spans. + continue; + } + let headers = spreadsheet_headers(®ions, &rows); + let text = match crate::table::render_grid_with_headers(&rows, headers) { + Some(text) => format!("{text}\n"), None => { - for line in &rows { - let cells: Vec<&str> = line.iter().map(|(s, _, _)| s.as_str()).collect(); - out.push_str(&cells.join("\t")); - out.push('\n'); + // Preserve the old TSV fallback exactly, but materialize only + // one dense row at a time rather than the whole rectangle. + let mut text = String::new(); + for row in 0..height { + let cells = spreadsheet_row( + row, + width, + ®ions, + &mut owner, + |col| text_at(row, col), + false, + ); + let cells: Vec<&str> = cells.iter().map(|(s, _, _)| s.as_str()).collect(); + text.push_str(&cells.join("\t")); + text.push('\n'); } + text } - } + }; + return Ok((text, height, rows_total)); } - Ok((out, warnings)) } #[derive(Clone)] @@ -843,9 +995,36 @@ fn spreadsheet_grid( if height == 0 || width == 0 { return (Vec::new(), None); } - let end_row = start_row + height as u32 - 1; - let end_col = start_col + width as u32 - 1; + let text_at = |row, col| spreadsheet_cell_text(range.get((row, col))); + let regions = spreadsheet_regions( + (start_row, start_col), + (start_row + height as u32 - 1, start_col + width as u32 - 1), + merges, + text_at, + ); + let mut owner = vec![usize::MAX; width]; + let rows: Vec<_> = (0..height) + .map(|row| { + spreadsheet_row( + row, + width, + ®ions, + &mut owner, + |col| text_at(row, col), + false, + ) + }) + .collect(); + let headers = spreadsheet_headers(®ions, &rows); + (rows, headers) +} +fn spreadsheet_regions( + (start_row, start_col): (u32, u32), + (end_row, end_col): (u32, u32), + merges: &[calamine::Dimensions], + text_at: impl Fn(usize, usize) -> String, +) -> Vec { let mut regions = Vec::new(); for merge in merges { if merge.start.0 > merge.end.0 @@ -868,30 +1047,40 @@ fn spreadsheet_grid( regions.push(GridMerge { start, end, - text: spreadsheet_cell_text(range.get(start)), + text: text_at(start.0, start.1), }); } + regions +} - let mut owner = vec![usize::MAX; height * width]; +fn spreadsheet_row( + row: usize, + width: usize, + regions: &[GridMerge], + owner: &mut [usize], + text_at: impl Fn(usize) -> String, + compact_empty: bool, +) -> GridRow { + // One owner row keeps the old last-merge-wins projection and anchor jumps, + // without allocating height * width indices. XLSX bounds width at XFD. + owner.fill(usize::MAX); for (index, region) in regions.iter().enumerate() { - for row in region.start.0..=region.end.0 { - for col in region.start.1..=region.end.1 { - owner[row * width + col] = index; - } + if region.start.0 <= row && row <= region.end.0 { + owner[region.start.1..=region.end.1].fill(index); } } - - let mut rows: Vec = Vec::with_capacity(height); - for row in 0..height { - let mut cells = GridRow::new(); - let mut col = 0usize; - while col < width { - let index = owner[row * width + col]; - if index == usize::MAX { - cells.push((spreadsheet_cell_text(range.get((row, col))), 1, 0)); - col += 1; - continue; + let mut cells: GridRow = Vec::new(); + let mut col = 0; + while col < width { + let index = owner[col]; + if index == usize::MAX { + let text = text_at(col); + match (compact_empty && text.is_empty(), cells.last_mut()) { + (true, Some((previous, span, _))) if previous.is_empty() => *span += 1, + _ => cells.push((text, 1, 0)), } + col += 1; + } else { let region = ®ions[index]; if col == region.start.1 { cells.push((region.text.clone(), region.end.1 - region.start.1 + 1, 0)); @@ -900,11 +1089,8 @@ fn spreadsheet_grid( col += 1; } } - rows.push(cells); } - - let headers = spreadsheet_headers(®ions, &rows); - (rows, headers) + cells } fn spreadsheet_cell_text(cell: Option<&calamine::Data>) -> String { @@ -1205,6 +1391,95 @@ fn pptx_xml_to_text(xml: &str) -> anyhow::Result { mod tests { use super::*; + fn budget_book(sheet_data: &str, merges: &str) -> calamine::Xlsx>> { + let worksheet = format!( + r#"{sheet_data}{merges}"# + ); + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + for (name, xml) in [ + ( + "[Content_Types].xml", + r#""#, + ), + ( + "_rels/.rels", + r#""#, + ), + ( + "xl/workbook.xml", + r#""#, + ), + ( + "xl/_rels/workbook.xml.rels", + r#""#, + ), + ("xl/worksheets/sheet1.xml", &worksheet), + ] { + zip.start_file(name, zip::write::SimpleFileOptions::default()) + .unwrap(); + zip.write_all(xml.as_bytes()).unwrap(); + } + calamine::open_workbook_from_rs(Cursor::new(zip.finish().unwrap().into_inner())).unwrap() + } + + #[test] + fn a_segment_budget_keeps_only_complete_rows() { + // Four source cells fit; two rows of three emitted segments do not. + let rows = r#"leftright12"#; + let (_, rows_read, rows_total) = + xlsx_sheet(&mut budget_book(rows, ""), "Budget", 5).unwrap(); + assert_eq!((rows_read, rows_total), (1, 2)); + } + + #[test] + fn dropping_a_row_reprojects_the_retained_title_and_merged_span() { + let title = r#"Title"#; + let rest = r#" note"#; + let merges = r#""#; + // Four source cells fit, but the first two rows use 3 + 4 segments. + let (text, rows_read, rows_total) = xlsx_sheet( + &mut budget_book(&format!("{title}{rest}"), merges), + "Budget", + 5, + ) + .unwrap(); + let (expected, _, _) = xlsx_sheet(&mut budget_book(title, merges), "Budget", 5).unwrap(); + assert_eq!((rows_read, rows_total), (1, 3)); + assert!(text.contains("Title")); + assert_eq!(text, expected); + } + + #[test] + fn empty_runs_keep_the_physical_cell_and_merge_coordinates() { + let regions = [GridMerge { + start: (0, 2), + end: (0, 4), + text: "Merged".into(), + }]; + let row = spreadsheet_row( + 0, + 10, + ®ions, + &mut [usize::MAX; 10], + |col| match col { + 0 => "Item".into(), + 7 => "3".into(), + _ => String::new(), + }, + true, + ); + let expected = [ + ("Item", 1), + ("", 1), + ("Merged", 3), + ("", 2), + ("3", 1), + ("", 2), + ] + .map(|(text, span)| (text.to_owned(), span, 0)); + assert_eq!(row, expected); + } + const DOC: &str = r#" Segment results SegmentRevenue diff --git a/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md b/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md new file mode 100644 index 000000000..16a30bcb7 --- /dev/null +++ b/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md @@ -0,0 +1,23 @@ +The MIT License (MIT) + +Copyright (c) 2016 Johann Tuffe + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls b/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls new file mode 100644 index 000000000..6c6decfab Binary files /dev/null and b/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls differ diff --git a/crates/utopia-ingest/tests/spreadsheet_dates.rs b/crates/utopia-ingest/tests/spreadsheet_dates.rs index ec7cd2ae2..ef23c5766 100644 --- a/crates/utopia-ingest/tests/spreadsheet_dates.rs +++ b/crates/utopia-ingest/tests/spreadsheet_dates.rs @@ -11,7 +11,7 @@ use std::io::{Cursor, Write}; // 3 date and time (22), 4 time of day (20), 5 elapsed time (46), 6 "#,##0.00" (4). const STYLES: &str = r#""#; -fn workbook(rows: &[Vec], date1904: bool) -> Vec { +fn workbook(rows: &[Vec], date1904: bool, styles: &str) -> Vec { let sheet_data: String = rows .iter() .enumerate() @@ -28,7 +28,7 @@ fn workbook(rows: &[Vec], date1904: bool) -> Vec { ("_rels/.rels", r#""#.to_string()), ("xl/workbook.xml", format!(r#"{workbook_pr}"#)), ("xl/_rels/workbook.xml.rels", r#""#.to_string()), - ("xl/styles.xml", STYLES.to_string()), + ("xl/styles.xml", styles.to_string()), ("xl/worksheets/sheet1.xml", format!(r#"{sheet_data}"#)), ] { zip.start_file(part, zip::write::SimpleFileOptions::default()).unwrap(); @@ -38,7 +38,7 @@ fn workbook(rows: &[Vec], date1904: bool) -> Vec { } fn read(rows: &[Vec], date1904: bool) -> String { - utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904)) + utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904, STYLES)) .unwrap() .text } @@ -141,3 +141,94 @@ fn a_workbook_counting_from_1904_reads_the_same_date() { "{text}" ); } + +fn read_format(format_id: u16, value: &str, date1904: bool, format_code: Option<&str>) -> String { + let custom = format_code.map_or_else(String::new, |code| { + let code = quick_xml::escape::escape(code); + format!( + r#""# + ) + }); + let styles = format!( + r#"{custom}"# + ); + let rows = [ + header(1), + vec![ + words("A2", "Value"), + number("B2", 0, value), + words("C2", "ok"), + ], + ]; + utopia_ingest::parse("ledger.xlsx", &workbook(&rows, date1904, &styles)) + .unwrap() + .text +} + +#[test] +fn east_asian_builtin_dates_need_no_custom_format_definition() { + for format_id in [27, 31, 50, 58] { + for (date1904, serial) in [(false, "45306"), (true, "43844")] { + let text = read_format(format_id, serial, date1904, None); + assert!( + text.contains("| Value | 2024-01-15 | ok |"), + "format {format_id}: {text}" + ); + } + } +} + +#[test] +fn east_asian_fractional_clock_times_and_explicit_formats_are_preserved() { + // calamine does not retain clock-only metadata; a serial >= 1 still gains a date. + for format_id in [32, 33] { + let text = read_format(format_id, "0.5", false, None); + assert!( + text.contains("| Value | 12:00 | ok |"), + "format {format_id}: {text}" + ); + } + for (format_id, code, serial, expected) in [ + (27, "0.00", "45306", "45306"), + (32, "yyyy-mm-dd", "45306", "2024-01-15"), + (27, "[h]:mm:ss", "1.5104166666666667", "36:15:00"), + ] { + let text = read_format(format_id, serial, false, Some(code)); + assert!( + text.contains(&format!("| Value | {expected} | ok |")), + "{text}" + ); + } +} + +#[test] +fn east_asian_builtin_formats_are_normalized_in_xls() { + // Fixture from calamine PR #734, commit d93c5a463399a5554d287ebda0ed1d2a43377ec2, + // tests/date_east_asian.xls (MIT; license alongside). Rows 0..19 are IDs 27..36/50..58; + // each row stores 45306, 0.5, 45306.123456789 with no explicit format definition. + let text = utopia_ingest::parse( + "ledger.xls", + include_bytes!("fixtures/calamine/date_east_asian.xls"), + ) + .unwrap() + .text; + let rows: Vec<_> = text + .lines() + .filter(|line| line.starts_with("| ") && !line.starts_with("| ---")) + .collect(); + assert_eq!(rows.len(), 28, "{text}"); + for row in [0, 4, 10, 18] { + // formats 27, 31, 50, 58 + assert!(rows[row].starts_with("| 2024-01-15 |"), "row {row}: {text}"); + } + for row in [5, 6] { + // formats 32, 33 + assert_eq!( + rows[row].split('|').nth(2).unwrap().trim(), + "12:00", + "{text}" + ); + } + assert!(rows[19].starts_with("| 45306 | 0.5 |"), "{text}"); // General + assert!(rows[24].starts_with("| 2024-01-15 |"), "{text}"); // Chinese custom date +} diff --git a/crates/utopia-ingest/tests/spreadsheet_truncation.rs b/crates/utopia-ingest/tests/spreadsheet_truncation.rs index e4f8b7cde..d4df2fd24 100644 --- a/crates/utopia-ingest/tests/spreadsheet_truncation.rs +++ b/crates/utopia-ingest/tests/spreadsheet_truncation.rs @@ -4,7 +4,7 @@ use std::fmt::Write as _; use std::io::{Cursor, Write}; -fn workbook(data_rows: usize) -> Vec { +fn ledger_rows(data_rows: usize) -> String { let mut sheet_data = String::from( r#"idname"#, ); @@ -14,17 +14,69 @@ fn workbook(data_rows: usize) -> Vec { r#"{id}customer-{id}"# )); } - let worksheet = format!( - r#"{sheet_data}"# + sheet_data +} + +fn workbook(data_rows: usize) -> Vec { + workbook_xml(&ledger_rows(data_rows)) +} + +fn workbook_xml(sheet_data: &str) -> Vec { + workbook_sheets(&[("Ledger", sheet_data)]) +} + +fn workbook_sheets(sheets: &[(&str, &str)]) -> Vec { + workbook_sheets_with_chart(sheets, false, "") +} + +fn workbook_sheets_with_chart( + sheets: &[(&str, &str)], + chart: bool, + worksheet_suffix: &str, +) -> Vec { + let mut content_types = String::from( + r#""#, ); + let mut sheet_entries = String::new(); + let mut sheet_rels = String::new(); + let mut parts = Vec::new(); + for (i, (name, sheet_data)) in sheets.iter().enumerate() { + let id = i + 1; + write!(content_types, r#""#).unwrap(); + write!( + sheet_entries, + r#""# + ) + .unwrap(); + write!(sheet_rels, r#""#).unwrap(); + parts.push((format!("xl/worksheets/sheet{id}.xml"), format!(r#"{sheet_data}{worksheet_suffix}"#))); + } + if chart { + let id = sheets.len() + 1; + write!(content_types, r#""#).unwrap(); + write!( + sheet_entries, + r#""# + ) + .unwrap(); + write!(sheet_rels, r#""#).unwrap(); + parts.extend([ + (format!("xl/chartsheets/sheet{id}.xml"), r#""#.into()), + (format!("xl/chartsheets/_rels/sheet{id}.xml.rels"), r#""#.into()), + ("xl/drawings/drawing1.xml".into(), r#""#.into()), + ("xl/drawings/_rels/drawing1.xml.rels".into(), r#""#.into()), + ("xl/charts/chart1.xml".into(), r#"Series 1ItemGeneral1"#.into()), + ]); + } + content_types.push_str(""); + parts.extend([ + ("[Content_Types].xml".into(), content_types), + ("_rels/.rels".into(), r#""#.into()), + ("xl/workbook.xml".into(), format!(r#"{sheet_entries}"#)), + ("xl/_rels/workbook.xml.rels".into(), format!(r#"{sheet_rels}"#)), + ]); let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); - for (part, xml) in [ - ("[Content_Types].xml", r#""#.to_string()), - ("_rels/.rels", r#""#.to_string()), - ("xl/workbook.xml", r#""#.to_string()), - ("xl/_rels/workbook.xml.rels", r#""#.to_string()), - ("xl/worksheets/sheet1.xml", worksheet), - ] { + for (part, xml) in parts { zip.start_file(part, zip::write::SimpleFileOptions::default()) .unwrap(); zip.write_all(xml.as_bytes()).unwrap(); @@ -32,6 +84,23 @@ fn workbook(data_rows: usize) -> Vec { zip.finish().unwrap().into_inner() } +fn words(cell: &str, text: &str) -> String { + format!(r#"{text}"#) +} + +fn column(mut col: usize) -> String { + let mut letters = Vec::new(); + loop { + letters.push(b'A' + (col % 26) as u8); + if col < 26 { + break; + } + col = col / 26 - 1; + } + letters.reverse(); + String::from_utf8(letters).unwrap() +} + fn csv(records: usize) -> String { let mut csv = String::from("id,name\n"); for id in 1..=records { @@ -78,6 +147,126 @@ fn a_spreadsheet_at_the_row_cap_has_no_warning() { assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); } +#[test] +fn a_distant_cell_past_the_row_window_does_not_shrink_the_retained_ledger() { + let mut rows = ledger_rows(1_999); + rows.push_str(&format!( + r#"{}"#, + words("P1000000", "omitted-distant-note") + )); + let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + assert!(parsed.text.contains("| 1999 | customer-1999 |")); + assert!(!parsed.text.contains("omitted-distant-note")); + assert_eq!(parsed.warnings.len(), 1); + assert_eq!(parsed.warnings[0].kind, "spreadsheet.rows_truncated"); + assert_eq!( + parsed.warnings[0].detail, + serde_json::json!({ + "sheet": "Ledger", "rows_read": 2000, + "rows_total": 1000000, "rows_omitted": 998000, + }) + ); +} + +#[test] +fn a_header_note_beyond_empty_columns_does_not_truncate_a_ledger() { + let rows = ledger_rows(1_500); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + assert!(expected.text.contains("| 1500 | customer-1500 |")); + for reference in ["AZZ1", "XFD1"] { + let with_note = rows.replacen( + "", + &format!("{}", words(reference, "header-only-note")), + 1, + ); + let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&with_note)).unwrap(); + assert!( + parsed.text == expected.text, + "note at {reference} changed the ledger" + ); + assert!( + parsed.warnings.is_empty(), + "{reference}: {:?}", + parsed.warnings + ); + } +} + +#[test] +fn an_ordinary_dense_sheet_retains_two_thousand_rows_and_one_hundred_fifty_columns() { + let columns: Vec<_> = (0..150).map(column).collect(); + let mut rows = String::from(r#""#); + for (col, name) in columns.iter().enumerate() { + rows.push_str(&words(&format!("{name}1"), &format!("column-{col}"))); + } + rows.push_str(""); + for row in 2..=2_000 { + write!( + rows, + r#"{}"#, + words(&format!("A{row}"), &format!("customer-{}", row - 1)) + ) + .unwrap(); + for (col, name) in columns.iter().enumerate().skip(1) { + write!( + rows, + r#"{}"#, + row * 1_000 + col + ) + .unwrap(); + } + rows.push_str(""); + } + let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); + let last = parsed + .text + .lines() + .find(|line| line.starts_with("| customer-1999 |")) + .expect("the final row is retained"); + assert!(last.ends_with("| 2000149 |"), "the last column is retained"); + assert_eq!(last.split('|').count(), 152, "all 150 columns are present"); +} + +#[test] +fn a_broken_sheet_does_not_prevent_reading_the_other_sheets() { + let good = ledger_rows(1); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&good)).unwrap(); + let broken = r#"not-a-number"#; + let bytes = workbook_sheets(&[("Ledger", &good), ("Broken", broken)]); + let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap(); + assert_eq!(parsed.text, expected.text); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); +} + +#[test] +fn unreadable_merges_do_not_prevent_reading_the_worksheet_cells() { + use calamine::Reader as _; + + let rows = ledger_rows(1); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + let merges = r#""#; + let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], false, merges); + let mut book: calamine::Xlsx<_> = + calamine::open_workbook_from_rs(Cursor::new(bytes.clone())).unwrap(); + assert_eq!(book.worksheet_range("Ledger").unwrap().height(), 2); + assert!(book.merge_cells_by_sheet_name("Ledger").is_err()); + + let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap(); + assert_eq!(parsed.text, expected.text); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); +} + +#[test] +fn a_chartsheet_does_not_prevent_reading_the_worksheets() { + let rows = ledger_rows(1); + let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap(); + let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], true, ""); + let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap(); + assert_eq!(parsed.text, expected.text); + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); +} + #[test] fn a_csv_past_the_record_cap_reports_the_records_it_left_out() { let parsed = utopia_ingest::parse("customers.csv", csv(10_000).as_bytes()).unwrap(); diff --git a/crates/utopia-server/src/api/chat_graph_citation_tests.rs b/crates/utopia-server/src/api/chat_graph_citation_tests.rs index f5c23b284..8b09de749 100644 --- a/crates/utopia-server/src/api/chat_graph_citation_tests.rs +++ b/crates/utopia-server/src/api/chat_graph_citation_tests.rs @@ -696,6 +696,7 @@ async fn a_fact_line_says_when_a_date_is_not_the_passages() -> anyhow::Result<() to_precision: Some("day"), attested_at: None, from_grade: None, + end_after_period: false, }, ) .await? @@ -820,6 +821,7 @@ async fn a_fact_line_says_when_a_date_is_not_the_passages() -> anyhow::Result<() to_precision: Some("day"), attested_at: None, from_grade: None, + end_after_period: false, }, ) .await? diff --git a/crates/utopia-server/src/api/graph_routes.rs b/crates/utopia-server/src/api/graph_routes.rs index e7adb3040..8eb3e165f 100644 --- a/crates/utopia-server/src/api/graph_routes.rs +++ b/crates/utopia-server/src/api/graph_routes.rs @@ -11,6 +11,10 @@ use crate::auth::{AuthUser, SessionOrToken}; use crate::error::ApiResult; use crate::state::AppState; +#[cfg(test)] +#[path = "graph_routes_time_tests.rs"] +mod time_tests; + pub(super) async fn require_kb( state: &AppState, user: &utopia_core::models::User, @@ -274,7 +278,8 @@ pub async fn update_entity( /// 一条事实的新有效区间。**整体替换，不是部分更新**：区间的两端互相定义， /// 「把结束端清空」与「这次不动结束端」必须能分辨，而缺字段与 null 在 JSON 里 -/// 分不开。界面提交的是表单里的四个值，不是差异。 +/// 分不开。界面提交的是表单里的四个值，不是差异。`end_after_period` 是例外：缺省即 +/// 保留，见那一格。 #[derive(Deserialize)] pub struct FactTimePatch { #[serde(default)] @@ -290,6 +295,13 @@ pub struct FactTimePatch { /// 这里升格成字段 #[serde(default)] pub note: Option, + /// 终点是不是原文所说时段之后的那一刻（`facts.end_after_period`，#994）。**唯一一个 + /// 缺省即保留的字段**：读出来什么就发回什么的客户端（不认识这个字段的也一样）不该 + /// 改掉它。不给时，终点（截到精度之后的值与精度）没变、起点还在，就留着存的标记；清掉 + /// 起点的保留时刻、丢掉标记（没有起点的行第一刀不标）；别的终点照给的收下。路由从不 + /// 换算：存的、收的都是边界本身，按时段显示与换算是界面的事 + #[serde(default)] + pub end_after_period: Option, } // 梯子只有一张（0024）：与数据库的 CHECK、抽取端、显示端同一列 @@ -302,9 +314,21 @@ struct FactInterval { valid_from_precision: Option, valid_to: Option>, valid_to_precision: Option, + end_after_period: bool, + /// 谓词声明成状态：标记只能落在这样的行上 + declared_state: bool, +} + +/// 显式要标记，却不是一个能带它的终点 +fn bad_end_after_period() -> AppError { + AppError::invalid( + "bad_end_after_period", + "An end that names a period needs a state property, a start and an end date.", + ) } -/// 复刻 `facts` 表的两条 CHECK，外加一条数据库没有的：起点不能晚于终点。 +/// 复刻 `facts` 表的 CHECK：两端的精度，标记要两端带得了它（`period_mark_fits`，属性是不是 +/// 状态读到行之后再查）；外加一条数据库没有的：截到各自精度之后，起点不能晚于终点。 /// /// 在这里挡住是为了让错误说人话。交给数据库挡会得到一句约束名，而这三态 /// （仍在持续 / 结束了不知哪天 / 某日结束）本来就是账本里最容易记混的地方。 @@ -331,8 +355,19 @@ fn check_interval(p: &FactTimePatch) -> Result<(), AppError> { )) } } + // 显式要标记：两端得带得了它。属性是不是状态要读到行才知道，这里先当它是，只看两端 + if p.end_after_period == Some(true) + && !utopia_store::graph::period_mark_fits(true, p.valid_from, p.valid_to, false) + { + return Err(bad_end_after_period()); + } + // 比截到各自精度之后的两端：存下的就是截过的。比原值时，小时精度的 08:30 终点与分钟精度 + // 的 08:10 起点放得过去，截完成了 08:10 → 08:00，存下一段倒着的区间 + let truncate = utopia_store::graph::truncate_to; if let (Some(from), Some(to)) = (p.valid_from, p.valid_to) { - if from > to { + if truncate(from, p.valid_from_precision.as_deref()) + > truncate(to, p.valid_to_precision.as_deref()) + { return Err(AppError::invalid( "interval_inverted", "The start of the interval is after its end.", @@ -361,9 +396,12 @@ pub async fn update_fact_time( // 归属与状态校验：本 KB 的、未作废的事实才可改。派生事实不在此列—— // 它的区间来自前提，改它等于改一个算出来的值，下一轮推理就会覆盖 let before: Option = sqlx::query_as( - "SELECT valid_from, valid_from_precision, valid_to, valid_to_precision - FROM facts - WHERE id = $1 AND kb_id = $2 AND invalidated_at IS NULL AND derived_by_rule IS NULL", + "SELECT f.valid_from, f.valid_from_precision, f.valid_to, f.valid_to_precision, + f.end_after_period, COALESCE(r.temporal = 'state', false) AS declared_state + FROM facts f + LEFT JOIN relation_types r ON r.id = f.predicate_id + WHERE f.id = $1 AND f.kb_id = $2 AND f.invalidated_at IS NULL + AND f.derived_by_rule IS NULL", ) .bind(fact_id) .bind(kb_id) @@ -373,12 +411,39 @@ pub async fn update_fact_time( let Some(before) = before else { return Err(AppError::NotFound.into()); }; + // 存下的是截到精度之后的值：比较、判标记都按它 + let truncate = utopia_store::graph::truncate_to; + let from = req + .valid_from + .map(|t| truncate(t, req.valid_from_precision.as_deref())); + let to = req + .valid_to + .map(|t| truncate(t, req.valid_to_precision.as_deref())); + // 读到了行，属性是不是状态也知道了：显式要的标记按整条规矩再判一遍 + if req.end_after_period == Some(true) + && !utopia_store::graph::period_mark_fits(before.declared_state, from, to, false) + { + return Err(bad_end_after_period().into()); + } + // 不给标记：发回存的终点（时刻与精度）、这次的两端还带得了它，就是没动它，留着；清掉起点 + // 的带不了，丢掉（时刻照旧）。属性这里不再判：存的标记是按当时的属性写下的，原样发回的 + // 请求不改写这一行 + let end_after_period = match req.end_after_period { + Some(mark) => mark, + None => { + before.end_after_period + && to == before.valid_to + && req.valid_to_precision == before.valid_to_precision + && utopia_store::graph::period_mark_fits(true, from, to, false) + } + }; // 一字未改就不写一条修正进账本：History 上多一行「改过」而区间没动， // 读者会去找那个不存在的差异 - if before.valid_from == req.valid_from + if before.valid_from == from && before.valid_from_precision == req.valid_from_precision - && before.valid_to == req.valid_to + && before.valid_to == to && before.valid_to_precision == req.valid_to_precision + && before.end_after_period == end_after_period { return Ok(Json(json!({ "ok": true, "unchanged": true }))); } @@ -394,6 +459,7 @@ pub async fn update_fact_time( // 修正行从被替代的行继承锚点（correct_interval 的 SQL 里）：改区间是重述 // 同一份证据，不是更新的证据 attested_at: None, + end_after_period, }; let Some(corrected) = utopia_store::temporal::correct_interval(&state.pool, fact_id, validity).await? @@ -418,12 +484,14 @@ pub async fn update_fact_time( "valid_from_precision": before.valid_from_precision, "valid_to": before.valid_to.map(|t| t.to_rfc3339()), "valid_to_precision": before.valid_to_precision, + "end_after_period": before.end_after_period, }); d["to"] = json!({ "valid_from": req.valid_from.map(|t| t.to_rfc3339()), "valid_from_precision": req.valid_from_precision, "valid_to": req.valid_to.map(|t| t.to_rfc3339()), "valid_to_precision": req.valid_to_precision, + "end_after_period": end_after_period, }); if let Some(note) = req.note.as_deref().map(str::trim).filter(|s| !s.is_empty()) { d["note"] = json!(note); diff --git a/crates/utopia-server/src/api/graph_routes_time_tests.rs b/crates/utopia-server/src/api/graph_routes_time_tests.rs new file mode 100644 index 000000000..2e26e35c3 --- /dev/null +++ b/crates/utopia-server/src/api/graph_routes_time_tests.rs @@ -0,0 +1,270 @@ +//! 改区间的路由收发的是存的边界（#994）：`end_after_period` 是唯一一个缺省即保留的字段。 +//! +//! 实体详情读得到标记；不带它的请求，终点（时刻与精度）没动、起点还在就留着，改了终点或清掉 +//! 起点就丢掉（时刻照旧）；显式的值照收，只改标记也算改了终点；显式 true 得是状态属性、有起点、 +//! 终点有日期；倒置按截到精度之后的两端判。每一步的结果收进一张表一起比，dev 上哪几步不同一次 +//! 看全。没有 `UTOPIA_DATABASE_URL` 时跳过。 +use axum::body::{to_bytes, Body}; +use axum::http::{Request, StatusCode}; +use axum::Router; +use chrono::{DateTime, Utc}; +use serde_json::{json, Value}; +use std::sync::Arc; +use tower::ServiceExt; +use uuid::Uuid; + +const MARCH: &str = "2024-03-01T00:00:00Z"; +const FEB: &str = "2024-02-01T00:00:00Z"; +const END: &str = "2025-01-01T00:00:00Z"; +const LATER: &str = "2026-01-01T00:00:00Z"; + +async fn send( + app: &Router, + token: &str, + method: &str, + uri: String, + body: &Value, +) -> anyhow::Result<(StatusCode, Value)> { + let response = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(uri) + .header("content-type", "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from(body.to_string()))?, + ) + .await?; + let status = response.status(); + let bytes = to_bytes(response.into_body(), usize::MAX).await?; + Ok((status, serde_json::from_slice(&bytes)?)) +} + +/// 一个请求体：起点日精度、终点年精度，空串是没有；标记给了才带 +fn span(from: &str, to: &str, mark: Option) -> Value { + let from = Some(from).filter(|t| !t.is_empty()); + let to = Some(to).filter(|t| !t.is_empty()); + let mut body = json!({ + "valid_from": from, "valid_from_precision": from.map(|_| "day"), + "valid_to": to, "valid_to_precision": to.map(|_| "year"), + }); + if let Some(mark) = mark { + body["end_after_period"] = json!(mark); + } + body +} + +#[tokio::test] +async fn a_correction_keeps_the_mark_unless_the_request_changes_it() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = sqlx::PgPool::connect(&url).await?; + utopia_store::db::migrate(&pool).await?; + let [org, ws, kb, editor, works_for, lin, meridian, marked, loose] = + std::array::from_fn(|_| Uuid::now_v7()); + // Only locally generated UUIDs and the constants above are interpolated into fixture SQL. + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','fact-time-route'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','fact-time-route'); + INSERT INTO users(id,org_id,email,display_name,password_hash) + VALUES ('{editor}','{org}','{editor}@time.test','editor','unused'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','times'); + INSERT INTO kb_members(kb_id,user_id,role) VALUES ('{kb}','{editor}','editor'); + INSERT INTO relation_types(id,kb_id,key,label,temporal) + VALUES ('{works_for}','{kb}','works_for','works for','state'); + INSERT INTO entities(id,kb_id,canonical_name) + VALUES ('{lin}','{kb}','Lin Zhao'), ('{meridian}','{kb}','Meridian Systems'); + -- 「left in 2024」关上的一行，记着说的是 2024；同样的区间、没有谓词的一行 + INSERT INTO facts(id,kb_id,subject_id,predicate_id,object_id,layer,confidence, + valid_from,valid_from_precision,valid_to,valid_to_precision, + end_after_period) VALUES + ('{marked}','{kb}','{lin}','{works_for}','{meridian}','typed',0.9, + '{MARCH}','day','{END}','year',TRUE), + ('{loose}','{kb}','{lin}',NULL,'{meridian}','typed',0.9, + '{MARCH}','day','{END}','year',FALSE);" + )) + .execute(&pool) + .await?; + let result = async { + let dir = tempfile::tempdir()?; + let config = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open(&dir.path().join("search"))?); + let state = crate::state::AppState::new(pool.clone(), &config, search, "test-only".into()); + let token = crate::auth::issue_token(&state, editor)?; + let app = crate::api::router(state, &config); + + let detail = format!("/api/v1/kbs/{kb}/entities/{lin}"); + let (_, detail) = send(&app, &token, "GET", detail, &Value::Null).await?; + let mut read = detail["facts"].as_array().into_iter().flatten(); + let read = read.find(|x| x["id"] == json!(marked)); + let mut seen = vec![("entity detail", json!(read.map(|x| &x["end_after_period"])))]; + let mut wanted = vec![("entity detail", json!(true))]; + // 每一步：改哪一行（None 是上一步写下的那行）、请求体、要的结果——写下的那一行（标记、 + // 人改过哪一端、终点），或拒绝的代码。不带标记的：只改起点留着，改了终点的时刻或精度、 + // 清掉起点都丢掉；显式的照收，只改标记把终点那一端记成人改的；显式 true 缺起点、缺终点、 + // 不是状态的都不收 + let bad = json!([422, "bad_end_after_period"]); + let steps = [ + ("start only", None, span(FEB, END, None), json!([true, "start", END])), + ("mark false", None, span(FEB, END, Some(false)), json!([false, "both", END])), + ("mark true", None, span(FEB, END, Some(true)), json!([true, "both", END])), + ("new end", None, span(FEB, LATER, None), json!([false, "both", LATER])), + ("mark true again", None, span(FEB, END, Some(true)), json!([true, "both", END])), + ( + "precision only", + None, + json!({"valid_from": FEB, "valid_from_precision": "day", + "valid_to": END, "valid_to_precision": "month"}), + json!([false, "both", END]), + ), + ("marked once more", None, span(FEB, END, Some(true)), json!([true, "both", END])), + ("start cleared", None, span("", END, None), json!([false, "both", END])), + ("true, no start", None, span("", END, Some(true)), bad.clone()), + ("true, no end", None, span(FEB, "", Some(true)), bad.clone()), + ("true, no state", Some(loose), span(MARCH, END, Some(true)), bad), + // 小时精度的 08:30 是 08:00，早于分钟精度的起点 08:10 + ( + "inverted once truncated", + Some(loose), + json!({"valid_from": "2026-06-01T08:10:00Z", "valid_from_precision": "minute", + "valid_to": "2026-06-01T08:30:00Z", "valid_to_precision": "hour"}), + json!([422, "interval_inverted"]), + ), + ]; + let mut fact = marked; + for (label, target, body, want) in steps { + let uri = format!("/api/v1/kbs/{kb}/facts/{}", target.unwrap_or(fact)); + let (status, reply) = send(&app, &token, "PATCH", uri, &body).await?; + let got = match reply["fact_id"].as_str() { + Some(id) => { + fact = id.parse()?; + let row: (bool, Option, Option>) = sqlx::query_as( + "SELECT end_after_period, corrected_ends, valid_to FROM facts WHERE id = $1", + ) + .bind(fact) + .fetch_one(&pool) + .await?; + json!(row) + } + None => json!([status.as_u16(), reply["code"]]), + }; + seen.push((label, got)); + wanted.push((label, want)); + } + assert_eq!(seen, wanted); + anyhow::Ok(()) + } + .await; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + result +} + +#[tokio::test] +async fn review_closes_refuse_inverted_intervals_without_settling_the_item() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = sqlx::PgPool::connect(&url).await?; + let [org, ws, kb, editor, property, lin, meridian, orion, old, new, conflict, violation] = + std::array::from_fn(|_| Uuid::now_v7()); + // Only locally generated UUIDs and fixed timestamps enter this fixture SQL. + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','review-close-time'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','review-close-time'); + INSERT INTO users(id,org_id,email,display_name,password_hash) + VALUES ('{editor}','{org}','{editor}@time.test','editor','unused'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','times'); + INSERT INTO kb_members(kb_id,user_id,role) VALUES ('{kb}','{editor}','editor'); + INSERT INTO relation_types(id,kb_id,key,label,temporal) + VALUES ('{property}','{kb}','works_for','works for','state'); + INSERT INTO entities(id,kb_id,canonical_name) VALUES + ('{lin}','{kb}','Lin Zhao'), ('{meridian}','{kb}','Meridian'), ('{orion}','{kb}','Orion'); + INSERT INTO facts(id,kb_id,subject_id,predicate_id,object_id,confidence, + valid_from,valid_from_precision) VALUES + ('{old}','{kb}','{lin}','{property}','{meridian}',0.9,'{MARCH}','day'), + ('{new}','{kb}','{lin}','{property}','{orion}',0.9,NULL,NULL); + INSERT INTO fact_conflicts(id,kb_id,old_fact_id,new_fact_id,reason) + VALUES ('{conflict}','{kb}','{old}','{new}','no_time'); + INSERT INTO axiom_violations(id,kb_id,kind,left_fact,right_fact) + VALUES ('{violation}','{kb}','derived_contradiction','{old}','{new}');" + )) + .execute(&pool) + .await?; + let result = async { + let dir = tempfile::tempdir()?; + let config = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open(&dir.path().join("search"))?); + let state = crate::state::AppState::new(pool.clone(), &config, search, "test-only".into()); + let token = crate::auth::issue_token(&state, editor)?; + let app = crate::api::router(state, &config); + let prefix = format!("/api/v1/kbs/{kb}"); + // The first date is later as written, but earlier at year precision. + // The existing equal-state refusal must reach all three cards as well. + for (at, precision, code) in [ + ("2024-10-01T00:00:00Z", "year", "interval_inverted"), + (MARCH, "month", "empty_state_span"), + ] { + for (path, body) in [ + ( + format!("facts/{old}/close"), + json!({"valid_to": at, "valid_to_precision": precision}), + ), + ( + format!("conflicts/{conflict}"), + json!({"action": "close", "close_at": at, "close_at_precision": precision}), + ), + ( + format!("review/violations/{violation}"), + json!({"resolution": "fact_closed", "close_at": at, "close_at_precision": precision}), + ), + ] { + let (status, reply) = send(&app, &token, "POST", format!("{prefix}/{path}"), &body).await?; + assert_eq!((status, reply["code"].as_str()), (StatusCode::UNPROCESSABLE_ENTITY, Some(code)), "{path}: {reply}"); + } + } + // A conflict with no explicit date closes at the new row's start; it must + // get the same store guard, even though no date came through the route. + sqlx::query("UPDATE facts SET valid_from = '2024-01-01', valid_from_precision = 'year' WHERE id = $1") + .bind(new).execute(&pool).await?; + let (status, reply) = send(&app, &token, "POST", format!("{prefix}/conflicts/{conflict}"), &json!({"action": "close"})).await?; + assert_eq!((status, reply["code"].as_str()), (StatusCode::UNPROCESSABLE_ENTITY, Some("interval_inverted"))); + let unchanged: bool = sqlx::query_scalar( + "SELECT (SELECT count(*) = 2 FROM facts WHERE kb_id = $1) + AND EXISTS (SELECT 1 FROM facts WHERE id = $2 AND invalidated_at IS NULL AND valid_to IS NULL) + AND EXISTS (SELECT 1 FROM fact_conflicts WHERE id = $3 AND status = 'open' AND resolution IS NULL) + AND EXISTS (SELECT 1 FROM axiom_violations WHERE id = $4 AND status = 'open' AND resolution IS NULL)", + ).bind(kb).bind(old).bind(conflict).bind(violation).fetch_one(&pool).await?; + assert!(unchanged, "refused closes write no replacement and leave both queues open"); + + // A later date can still close the refused row, with the endpoint stored + // at its supplied precision, without translating it to a period's end. + let (status, _) = send(&app, &token, "POST", format!("{prefix}/facts/{old}/close"), + &json!({"valid_to": "2024-09-19T00:00:00Z", "valid_to_precision": "month"})).await?; + assert_eq!(status, StatusCode::OK); + let closed: (DateTime, String, bool) = sqlx::query_as( + "SELECT valid_to, valid_to_precision, end_after_period FROM facts WHERE supersedes = $1", + ).bind(old).fetch_one(&pool).await?; + assert_eq!(closed, ("2024-09-01T00:00:00Z".parse()?, "month".into(), false)); + anyhow::Ok(()) + }.await; + sqlx::query("DELETE FROM knowledge_bases WHERE id = $1") + .bind(kb) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + result +} diff --git a/crates/utopia-server/src/api/oidc_exchange_tests.rs b/crates/utopia-server/src/api/oidc_exchange_tests.rs index 817c68dfa..d2b0cf35d 100644 --- a/crates/utopia-server/src/api/oidc_exchange_tests.rs +++ b/crates/utopia-server/src/api/oidc_exchange_tests.rs @@ -211,6 +211,24 @@ async fn exchange_maps_only_linked_active_people_and_spends_each_token_once() -> let (status, body) = exchange(sign_email(&format!("sub-{ann}"), &format!("{ann}@EXAMPLE.test"), true)?).await?; anyhow::ensure!(status == StatusCode::OK && user_of(&body)? == ann, "existing by email: {status} {body}"); + let duplicate = Uuid::now_v7(); + sqlx::query("INSERT INTO users(id,org_id,email,password_hash,display_name) VALUES($1,$2,$3,'unused','Duplicate Ann')") + .bind(duplicate) + .bind(org) + .bind(format!("{ann}@EXAMPLE.TEST")) + .execute(&pool) + .await?; + let (status, body) = exchange(sign_email("ambiguous-ann", &format!("{ann}@example.test"), true)?).await?; + anyhow::ensure!( + status == StatusCode::UNPROCESSABLE_ENTITY + && body["code"] == "oidc_unavailable" + && body["error"] == "The identity could not be mapped to an account", + "ambiguous email: {status} {body}" + ); + let linked: i64 = sqlx::query_scalar("SELECT count(*) FROM oidc_identities WHERE issuer = $1 AND (subject = 'ambiguous-ann' OR user_id = $2)") + .bind(&issuer).bind(duplicate).fetch_one(&pool).await?; + anyhow::ensure!(linked == 0); + let bea_sub = format!("sub-{}", Uuid::now_v7()); let (status, body) = exchange(sign_email(&bea_sub, &bea_email, true)?).await?; anyhow::ensure!(status == StatusCode::OK, "opened: {status} {body}"); diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rs index 12ea8eb7a..08b216cf9 100644 --- a/crates/utopia-server/src/api/tools.rs +++ b/crates/utopia-server/src/api/tools.rs @@ -1533,6 +1533,7 @@ mod tests { stale: false, corrected: false, end_derived: false, + end_after_period: false, time_corrected: false, corrected_ends: None, closed_by_id: None, diff --git a/crates/utopia-server/src/api/tools_graph.rs b/crates/utopia-server/src/api/tools_graph.rs index a207c6986..eb27a1214 100644 --- a/crates/utopia-server/src/api/tools_graph.rs +++ b/crates/utopia-server/src/api/tools_graph.rs @@ -1505,6 +1505,7 @@ mod tests { stale: false, corrected: false, end_derived: false, + end_after_period: false, time_corrected: false, corrected_ends: None, closed_by_id: None, diff --git a/crates/utopia-server/src/phrase_alignment.rs b/crates/utopia-server/src/phrase_alignment.rs index 378324d11..ba83160db 100644 --- a/crates/utopia-server/src/phrase_alignment.rs +++ b/crates/utopia-server/src/phrase_alignment.rs @@ -229,24 +229,6 @@ fn undecided( .collect() } -/// 只有键的签名：计数与例句为空，够 `settled` / `declared` 判 -fn keyed( - (phrase, subject_type_id, object_type_id, object_is_value): &SignatureKey, -) -> PhraseSignature { - PhraseSignature { - phrase: phrase.clone(), - subject_type_id: *subject_type_id, - subject_type_key: None, - object_type_id: *object_type_id, - object_type_key: None, - object_is_value: *object_is_value, - count: 0, - examples: Vec::new(), - quotes: Vec::new(), - example_statement_ids: Vec::new(), - } -} - /// 推送来的文档抽完（0054）：它的陈述落在的签名里，短语正好是一条声明了的关系的，由本体 /// 直接绑上，记成本体判的（#1242）。不问模型，没配对话模型也照判；别的签名照旧留给对齐器。 /// 判定照旧是签名上的一行（0044 决定 3），人看得见、改得了；人的判定不动（`decide_on` @@ -289,12 +271,6 @@ pub(crate) async fn bind_declared( let Some(p) = declared(s, &props, &closure) else { continue; }; - if existing - .get(&s.key()) - .is_some_and(|b| settled(b, s, &props, &closure)) - { - continue; - } if phrase_bindings::decide( pool, kb_id, @@ -320,6 +296,25 @@ pub(crate) async fn bind_declared( /// 签名的键：短语 + 两端的类 + 宾语是不是字面值（与 `PhraseSignature::key` 同形） type SignatureKey = (String, Option, Option, bool); + +/// 只有键的签名：`settled` 只看键上的这几格，不必为它把库里的签名数一遍 +fn keyed( + (phrase, subject_type_id, object_type_id, object_is_value): &SignatureKey, +) -> PhraseSignature { + PhraseSignature { + phrase: phrase.clone(), + subject_type_id: *subject_type_id, + subject_type_key: None, + object_type_id: *object_type_id, + object_type_key: None, + object_is_value: *object_is_value, + count: 0, + examples: Vec::new(), + quotes: Vec::new(), + example_statement_ids: Vec::new(), + } +} + /// 每条签名此刻的候选与指纹 type Considered<'a> = HashMap, String)>; diff --git a/crates/utopia-server/src/phrase_alignment_declared_tests.rs b/crates/utopia-server/src/phrase_alignment_declared_tests.rs index 6440f47fa..8f750d442 100644 --- a/crates/utopia-server/src/phrase_alignment_declared_tests.rs +++ b/crates/utopia-server/src/phrase_alignment_declared_tests.rs @@ -86,8 +86,8 @@ impl Rig { })) } - /// 推一篇文档并抽取：一条陈述，写成 [主语, 它的类别词, 短语, 宾语, 它的类别词] - async fn push(&self, external_id: &str, said: [&str; 5]) -> anyhow::Result<()> { + /// 推一篇文档并抽取：一条陈述，写成 [主语, 它的类别词, 短语, 宾语, 它的类别词]。返回文档 + async fn push(&self, external_id: &str, said: [&str; 5]) -> anyhow::Result { let [subject, subject_kind, phrase, object, object_kind] = said; let payload: Value = json!({ "external_id": external_id, @@ -118,7 +118,8 @@ impl Rig { doc, utopia_core::models::Proposer::default(), ) - .await + .await?; + Ok(doc) } async fn bindings(&self) -> anyhow::Result> { @@ -274,6 +275,34 @@ async fn only_a_phrase_naming_one_relation_that_fits_both_ends_binds() -> anyhow rig.cleanup().await } +/// 签名都已判过的推送不数库里的签名（#1296）。一条连接的池上 sqlx 缓存它跑过的每条语句， +/// 没跑的不在 `pg_prepared_statements` 里 +#[tokio::test] +async fn a_push_whose_signatures_are_settled_skips_the_base_signatures() -> anyhow::Result<()> { + let Some(rig) = Rig::new().await? else { + return Ok(()); + }; + rig.push("obs-1", ["S-1", "statement", "validated_by", "T-1", "test"]) + .await?; + let second = rig + .push("obs-2", ["S-2", "statement", "validated_by", "T-2", "test"]) + .await?; + let one = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect_with((*rig.pool.connect_options()).clone()) + .await?; + assert_eq!(bind_declared(&one, rig.kb, second).await?, 0); + let grouped: Vec = sqlx::query_scalar( + "SELECT statement FROM pg_prepared_statements + WHERE statement LIKE '%WITH live AS%' AND statement NOT LIKE '%pg_prepared_statements%'", + ) + .fetch_all(&one) + .await?; + assert_eq!(grouped, Vec::::new()); + one.close().await; + rig.cleanup().await +} + #[tokio::test] async fn a_person_decision_is_not_overwritten_by_the_ontology() -> anyhow::Result<()> { let Some(rig) = Rig::new().await? else { diff --git a/crates/utopia-store/src/accounts.rs b/crates/utopia-store/src/accounts.rs index b665ed6c2..49ad7add1 100644 --- a/crates/utopia-store/src/accounts.rs +++ b/crates/utopia-store/src/accounts.rs @@ -292,6 +292,7 @@ pub async fn deactivate_user(pool: &PgPool, target: Uuid, actor: Uuid) -> AppRes return Err(AppError::Validation("不能停用自己的账号".into())); } let mut tx = pool.begin().await?; + lock_account_organization(&mut tx, target).await?; let victim: Option<(bool, Uuid, Option>)> = sqlx::query_as("SELECT is_admin, org_id, deactivated_at FROM users WHERE id = $1") .bind(target) @@ -360,23 +361,64 @@ pub async fn set_system_admin( if target == actor && !value { return Err(AppError::Validation("不能收回自己的系统管理员".into())); } + let mut tx = pool.begin().await?; + let org_id = lock_account_organization(&mut tx, target).await?; + let is_admin: bool = + sqlx::query_scalar("SELECT is_admin FROM users WHERE id = $1 AND deactivated_at IS NULL") + .bind(target) + .fetch_optional(&mut *tx) + .await? + .ok_or(AppError::NotFound)?; + if is_admin && !value { + let others: i64 = sqlx::query_scalar( + "SELECT count(*) FROM users + WHERE org_id = $1 AND is_admin AND deactivated_at IS NULL AND id <> $2", + ) + .bind(org_id) + .bind(target) + .fetch_one(&mut *tx) + .await?; + if others == 0 { + return Err(AppError::Validation( + "这是组织里最后一个管理员，收回权限之后没人能管成员".into(), + )); + } + } let res = sqlx::query("UPDATE users SET is_admin = $2 WHERE id = $1 AND deactivated_at IS NULL") .bind(target) .bind(value) - .execute(pool) + .execute(&mut *tx) .await?; if res.rows_affected() == 0 { return Err(AppError::NotFound); } + tx.commit().await?; Ok(()) } +async fn lock_account_organization( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + target: Uuid, +) -> AppResult { + // 撤权与停用共用组织锁；拿锁后再读账号、数管理员，不能信任请求鉴权时的快照。 + sqlx::query_scalar( + "SELECT id FROM organizations + WHERE id = (SELECT org_id FROM users WHERE id = $1) FOR NO KEY UPDATE", + ) + .bind(target) + .fetch_optional(&mut **tx) + .await? + .ok_or(AppError::NotFound) +} + /// 单点登录允许自动开户时，用身份提供方核实过的 email 找账号；找不到就按注册的 /// 规则开一个（默认工作区的 viewer；部署里还没人时就是首个管理员），并把这个 /// subject 绑上。开户与绑定同一个事务：绑定撞车（`taken` / `already_linked`）时 /// 不留下一个没人能登进去的账号。 /// +/// 大小写不敏感的邮箱匹配必须只有一个活动账号；多个匹配返回冲突，不开户也不绑定。 +/// /// 这个 email 只剩一个停用的账号时返回 `None`：停用的人不能换个新号回来—— /// email 的唯一索引只管启用中的账号，挡不住这一步，得在这里查。 /// @@ -389,13 +431,20 @@ pub async fn account_for_sso( display_name: &str, ) -> AppResult> { let mut tx = pool.begin().await?; - let active: Option = sqlx::query_as( - "SELECT * FROM users WHERE lower(email) = lower($1) AND deactivated_at IS NULL", + // 本次只拒绝快照中已有的歧义；阻止并发开户产生重复邮箱需要另行迁移并处理存量重复。 + let mut active: Vec = sqlx::query_as( + "SELECT * FROM users WHERE lower(email) = lower($1) AND deactivated_at IS NULL LIMIT 2", ) .bind(email) - .fetch_optional(&mut *tx) + .fetch_all(&mut *tx) .await?; - let user = match active { + if active.len() > 1 { + return Err(AppError::CodedConflict { + code: "unlinked", + message: "Multiple active accounts match the SSO email".into(), + }); + } + let user = match active.pop() { Some(u) => u, None => { let deactivated: bool = sqlx::query_scalar( diff --git a/crates/utopia-store/src/errata.rs b/crates/utopia-store/src/errata.rs index 48d19de72..4677a5766 100644 --- a/crates/utopia-store/src/errata.rs +++ b/crates/utopia-store/src/errata.rs @@ -622,7 +622,7 @@ async fn apply( let fact = fact_id.ok_or_else(|| AppError::Validation("no fact to retract".into()))?; validity_row = sqlx::query_as( "SELECT valid_from, valid_from_precision, valid_from_grade, valid_to, valid_to_precision, - attested_from FROM facts WHERE id = $1", + end_after_period, attested_from FROM facts WHERE id = $1", ) .bind(fact) .fetch_optional(pool) @@ -652,6 +652,8 @@ async fn apply( to: v.valid_to, to_precision: v.valid_to_precision.as_deref(), attested_at: v.attested_from.or(doc_time), + // 新行接旧行的终点，也接它说的是哪个时段（#994）；改到事件、恒常属性上的，写入口丢掉 + end_after_period: v.end_after_period, }; let object = match (target.object_id, &target.object_value) { (Some(o), _) => FactObject::Entity(o), @@ -724,6 +726,7 @@ struct ValidityRow { valid_from_grade: Option, valid_to: Option>, valid_to_precision: Option, + end_after_period: bool, attested_from: Option>, } diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs index 6ccd29788..6d981c008 100644 --- a/crates/utopia-store/src/graph.rs +++ b/crates/utopia-store/src/graph.rs @@ -329,6 +329,10 @@ pub struct Validity<'a> { /// `C` 有时间词但锚不到。`None` = 没经过时间解析（人写的、规则算的），按 A 算。 /// 时态引擎读它决定这一行能不能关上前任 pub from_grade: Option<&'a str>, + /// 终点是原文所说时段之后的那一刻（`facts.end_after_period`，#994）：「2024 年离开」关上 + /// 2024-03-01 起的那段，存 2025-01-01，原文说的是 2024。只看值分不出它和「到 2025 年」。 + /// 写入口归一（`drop_period_mark_unless`），只留在带得了它的行上（[`period_mark_fits`]） + pub end_after_period: bool, } /// `valid_to_precision` 表示「结束了，但不知道是哪天」。 @@ -447,6 +451,7 @@ impl<'a> Validity<'a> { to_precision: None, attested_at: None, from_grade: None, + end_after_period: false, } } @@ -516,6 +521,15 @@ impl<'a> Validity<'a> { self } + /// 「终点说的是它之前那个时段」带不了的行（[`period_mark_fits`]），标记在这里丢掉：两个 + /// 写入口（`insert_fact_on`、`correct_interval`）在 [`Validity::under`] 之后各过一遍。 + /// 事件归一成一刻、恒常抹掉日期、没有谓词的行不过 `under`，都落在这里。进写入口的终点是 + /// 写明的，不是时间线画的 + pub(crate) fn drop_period_mark_unless(mut self, declared_state: bool) -> Self { + self.end_after_period &= period_mark_fits(declared_state, self.from, self.to, false); + self + } + /// 这条断言是否已经不再成立——**两种结束都算**。 /// /// 判据写在这里而不是散在各处的 `valid_to.is_some()`：那种写法会把 @@ -525,6 +539,23 @@ impl<'a> Validity<'a> { } } +/// 一个终点带不带得了「说的是它之前那个时段」的标记（`end_after_period`，#994）：属性声明成 +/// 状态、有起点、终点有日期、不是时间线画的。 +/// +/// 与约束 `facts_end_after_period_needs_a_stated_end` 同一句话（属性是不是状态约束看不到， +/// 由这里查）。Rust 这边判这件事的都调它：写入口的归一（`Validity::drop_period_mark_unless`）、 +/// `temporal::rewrite_end_tx` 落标记前、`temporal::mark_end_after_period` 补标记前、改区间的 +/// 路由。没有起点的行第一刀不标：终点只在起点落在那个时段里时才被挪到时段的尽头。第二刀让 +/// 时间解析自己写这种终点时，起点那一条在这里和约束里一起去掉 +pub fn period_mark_fits( + declared_state: bool, + from: Option>, + to: Option>, + end_derived: bool, +) -> bool { + declared_state && from.is_some() && to.is_some() && !end_derived +} + /// 一段起止同值的状态（#966）：写入、改区间、关上都拒它，说的是同一句话 pub fn empty_state_span() -> AppError { AppError::invalid( @@ -581,7 +612,8 @@ pub(crate) async fn insert_fact_on( validity.truncated().under(temporal)? } else { validity.truncated() - }; + } + .drop_period_mark_unless(declared_state); let same_sql = match object { FactObject::Entity(_) => { "SELECT id, valid_from, valid_to, valid_to_precision FROM facts @@ -639,9 +671,14 @@ pub(crate) async fn insert_fact_on( && vtp.as_deref() == Some(ENDED_UNKNOWN) && validity.from.is_none_or(|f| Some(f) == *vf) }) { - if let Some(stated) = - crate::temporal::state_derived_end(&mut *conn, *ended, None, validity.attested_at) - .await? + if let Some(stated) = crate::temporal::state_derived_end( + &mut *conn, + *ended, + None, + false, + validity.attested_at, + ) + .await? { return Ok((stated, true)); } @@ -663,27 +700,43 @@ pub(crate) async fn insert_fact_on( // 照旧关在时段的开头（0053 修订 2026-09-27）。同一天开始又结束的也是这样：关在 // 那一天的尽头，不关在它自己的起点——那是一段不成立的状态（#966） let period_end = declared_state.then(|| bucket_end(to, Some(precision))); + // 关在哪一刻，和终点是不是因此挪到了时段的尽头：挪了的，行上记下终点说的是它之前 + // 那个时段（`end_after_period`，#994），读者才印得出原文说的 2024 let closes_at = |from: Option>| match (from, period_end) { - (Some(f), Some(end)) if to <= f && f < end => end, - _ => to, + (Some(f), Some(end)) if to <= f && f < end => (end, true), + _ => (to, false), }; // 已经关在那一刻的：同一件事，复用那一行（引擎推的终点改成写明的，同上） if let Some((ended, from, _, _)) = same .iter() - .find(|(_, vf, vt, _)| *vt == Some(closes_at(*vf))) + .find(|(_, vf, vt, _)| *vt == Some(closes_at(*vf).0)) { + let (at, moved) = closes_at(*from); if let Some(stated) = crate::temporal::state_derived_end( &mut *conn, *ended, - Some((closes_at(*from), precision)), + Some((at, precision)), + moved, validity.attested_at, ) .await? { return Ok((stated, true)); } - attest_earlier(&mut *conn, *ended, validity.attested_at, true).await?; - return Ok((*ended, false)); + // 写明的终点被另一种说法再说一遍：那一行是「到 2025 年」关上的，这次是「2024 年 + // 离开」，同一个终点。标上它，两句谁先到都一样。上面只比了时刻：人改过的终点、 + // 别的精度写明的终点不改，由 `mark_end_after_period` 锁住那一行再查。 + // 标上了也是并进那一行，不报新行：报成新行，物化会给它记上这句的 + // `from_statement_id`、规则会记上 `implied`，人关上的行就成了算出来的，这句的 + // 陈述一没，它跟着被作废 + let marked = if moved { + crate::temporal::mark_end_after_period(&mut *conn, *ended, precision).await? + } else { + None + }; + let row = marked.unwrap_or(*ended); + attest_earlier(&mut *conn, row, validity.attested_at, true).await?; + return Ok((row, false)); } let open = same .iter() @@ -697,13 +750,9 @@ pub(crate) async fn insert_fact_on( }) .max_by_key(|(_, vf, _, _)| *vf); if let Some((open, from, _, _)) = open { - if let Some(closed) = crate::temporal::close_superseded( - &mut *conn, - *open, - closes_at(*from), - precision, - ) - .await? + let (at, moved) = closes_at(*from); + if let Some(closed) = + crate::temporal::close_stated(&mut *conn, *open, at, precision, moved).await? { return Ok((closed, true)); } @@ -713,13 +762,16 @@ pub(crate) async fn insert_fact_on( // 精确重复：同 valid_from → 复用。同起点、**这次带了终点、那行还开着** → 关上它 // （「自 2020-01-10 起任董事」之后读到「2020-01-10 至 2024-04-30 任董事」） if let Some((existing, _, vt, vtp)) = same.iter().find(|(_, vf, _, _)| *vf == validity.from) { + // 这次带来的终点若说的是它之前那个时段（勘误改出来的行接旧行的标记），关上时一起带上； + // 复用的那行原样留着它自己的终点和标记 if temporal == Temporal::State && vt.is_none() && vtp.is_none() { if let Some(to) = validity.to { - if let Some(closed) = crate::temporal::close_superseded( + if let Some(closed) = crate::temporal::close_stated( &mut *conn, *existing, to, validity.to_precision.unwrap_or("day"), + validity.end_after_period, ) .await? { @@ -736,6 +788,7 @@ pub(crate) async fn insert_fact_on( &mut *conn, *existing, stated_to, + validity.end_after_period, validity.attested_at, ) .await? @@ -789,22 +842,22 @@ pub(crate) async fn insert_fact_on( if vf.is_some() { return None; } - let end = match (validity.to, *vt) { + let (end, moved) = match (validity.to, *vt) { (None, Some(t)) if declared_state && t <= from => { let end = bucket_end(t, Some(vtp.as_deref().unwrap_or("day"))); if from >= end { return None; } - Some(end) + (Some(end), true) } (Some(to), Some(t)) if to != t => return None, - (_, vt) => vt, + (_, vt) => (vt, false), }; - Some((*id, end, vtp.clone())) + Some((*id, end, vtp.clone(), moved)) }), None => None, }; - if let Some((_, Some(vt), vtp)) = &refine_target { + if let Some((_, Some(vt), vtp, moved)) = &refine_target { if validity.to.is_none() { validity.to = Some(*vt); validity.to_precision = vtp.as_deref().map(|p| match p { @@ -816,9 +869,11 @@ pub(crate) async fn insert_fact_on( "second" => "second", _ => ENDED_UNKNOWN, }); + // 沿用来的终点挪到了时段的尽头：与收尾那条路同一个标记，两种陈述顺序写下同一行 + validity.end_after_period = *moved; } } - let refine_target = refine_target.map(|(id, _, _)| id); + let refine_target = refine_target.map(|(id, _, _, _)| id); let id = Uuid::now_v7(); let insert_sql = match object { @@ -827,17 +882,17 @@ pub(crate) async fn insert_fact_on( "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, - attested_from, attested_to, valid_from_grade) + attested_from, attested_to, valid_from_grade, end_after_period) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, COALESCE($11, now()), - CASE WHEN $9::text = 'unknown' THEN COALESCE($11, now()) END, $12)" + CASE WHEN $9::text = 'unknown' THEN COALESCE($11, now()) END, $12, $13)" } FactObject::Value(_) => { "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, - attested_from, attested_to, valid_from_grade) + attested_from, attested_to, valid_from_grade, end_after_period) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, COALESCE($11, now()), - CASE WHEN $9::text = 'unknown' THEN COALESCE($11, now()) END, $12)" + CASE WHEN $9::text = 'unknown' THEN COALESCE($11, now()) END, $12, $13)" } }; let mut ins = sqlx::query(insert_sql) @@ -856,6 +911,7 @@ pub(crate) async fn insert_fact_on( .bind(confidence) .bind(validity.attested_at) .bind(validity.from_grade) + .bind(validity.end_after_period) .execute(&mut *conn) .await?; @@ -1741,7 +1797,7 @@ pub async fn entity_detail( WHERE fe.fact_id = f.id AND {chunk_live}) ) AS stale, (f.supersedes IS NOT NULL) AS corrected, - f.end_derived, {corrected_ends} AS corrected_ends, + f.end_derived, f.end_after_period, {corrected_ends} AS corrected_ends, {corrected_ends} IS NOT NULL AS time_corrected, closer.closed_by_id, closer.closed_by, closer.closed_by_value, CASE WHEN {corrected_ends} IS NOT NULL THEN {correction_note} END @@ -2536,6 +2592,7 @@ mod temporal_shape_tests { to_precision: Some("day"), attested_at: None, from_grade: None, + end_after_period: false, } .under(Temporal::Event) .unwrap(); @@ -2553,6 +2610,7 @@ mod temporal_shape_tests { to_precision: Some("month"), attested_at: None, from_grade: None, + end_after_period: false, } .under(Temporal::Event) .unwrap(); @@ -2597,6 +2655,7 @@ mod temporal_shape_tests { to_precision: Some("day"), attested_at: None, from_grade: None, + end_after_period: false, }; let refused = moment .under(Temporal::State) diff --git a/crates/utopia-store/src/materialize.rs b/crates/utopia-store/src/materialize.rs index a07f8749a..2383072be 100644 --- a/crates/utopia-store/src/materialize.rs +++ b/crates/utopia-store/src/materialize.rs @@ -156,14 +156,11 @@ async fn materialize_in_tx( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, kb_id: Uuid, ) -> AppResult<(Outcome, Vec)> { - // Plan every statement of the recompute for this base. Prepared statements switch to a - // generic plan after a few runs, and a generic plan cannot see how many rows `kb_id = $1` - // selects: on a production base of 200,000 typed-fact sources the source cleanup below - // chose a nested loop over a materialized anti-join and ran for hours, where the plan for - // the base's own id takes two seconds. LOCAL ends with the transaction. - sqlx::query("SET LOCAL plan_cache_mode = force_custom_plan") - .execute(&mut **tx) - .await?; + // The statements that scan the whole base (steps 0 to 3, the implication scans and the + // attestation sync) are not cached: a cached statement switches to a generic plan after a few + // runs, and a generic plan cannot see how many rows `kb_id = $1` selects. The source cleanup's + // generic plan ran for hours on a production base where the plan for its own id takes two + // seconds (#1298). The per-row statements in the loops stay cached. // 这一轮写下的行（新建的和并入的），提交后对账 let mut written: Vec = Vec::new(); // 0. 起止同值的物化状态行（#966 之前照抄一刻写下的）任何时刻都不成立：作废，它的陈述 @@ -189,6 +186,7 @@ async fn materialize_in_tx( OR s.id IN (SELECT i.statement_id FROM implied_fact_sources i WHERE i.fact_id = t.id)))", ) + .persistent(false) .bind(kb_id) .execute(&mut **tx) .await? @@ -206,6 +204,7 @@ async fn materialize_in_tx( LEFT JOIN entities oe ON oe.id = s.object_id JOIN phrase_bindings b ON {MATCH} WHERE s.id = src.statement_id + AND s.kb_id = $1 AND s.invalidated_at IS NULL AND t.invalidated_at IS NULL AND b.status = 'bound' @@ -216,6 +215,7 @@ async fn materialize_in_tx( WHERE q.fact_id = s.id AND q.role = 'mood') AND NOT {MOMENT_UNREAD})" )) + .persistent(false) .bind(kb_id) .execute(&mut **tx) .await?; @@ -240,6 +240,7 @@ async fn materialize_in_tx( RULE_MATCH = crate::implication_rules::RULE_MATCH, KIND = kind_word_sql("e.specific_type"), )) + .persistent(false) .bind(kb_id) .execute(&mut **tx) .await?; @@ -253,6 +254,7 @@ async fn materialize_in_tx( AND NOT EXISTS (SELECT 1 FROM typed_fact_sources src WHERE src.fact_id = t.id) AND NOT EXISTS (SELECT 1 FROM implied_fact_sources i WHERE i.fact_id = t.id)", ) + .persistent(false) .bind(kb_id) .execute(&mut **tx) .await? @@ -288,6 +290,7 @@ async fn materialize_in_tx( AND t.predicate_id = b.relation_type_id) ORDER BY s.id" )) + .persistent(false) .bind(kb_id) .fetch_all(&mut **tx) .await?; @@ -297,7 +300,8 @@ async fn materialize_in_tx( let reverse = d.direction == "reverse"; // 一刻的陈述在状态属性下按绑定说的读（#966）：开始写成从那一刻起，结束写成到那一刻 // 为止（走 `insert_fact_on`「某天结束了」那条路，关上开放的那段）。什么都不算的 - // 在查询里已经排掉了（[`MOMENT_UNREAD`]） + // 在查询里已经排掉了（[`MOMENT_UNREAD`]）。陈述不带 `end_after_period`：终点挪没挪到 + // 时段的尽头由写入路径定（#994 第一刀） let moment = d.temporal.as_deref() == Some("state") && d.valid_from.is_some() && d.valid_from == d.valid_to; @@ -309,6 +313,7 @@ async fn materialize_in_tx( to: d.valid_to, to_precision: d.valid_to_precision.as_deref(), attested_at: d.attested_from, + end_after_period: false, }, (true, Some("start")) => Validity { from: d.valid_from, @@ -317,6 +322,7 @@ async fn materialize_in_tx( to: None, to_precision: None, attested_at: d.attested_from, + end_after_period: false, }, (true, Some("end")) => Validity { from: None, @@ -325,6 +331,7 @@ async fn materialize_in_tx( to: d.valid_from, to_precision: d.valid_from_precision.as_deref(), attested_at: d.attested_from, + end_after_period: false, }, (true, _) => continue, }; @@ -500,6 +507,7 @@ pub async fn sync_typed_attestation<'e>( OR t.attested_by IS DISTINCT FROM src.by) RETURNING t.id", ) + .persistent(false) .bind(kb_id) .bind(only) .fetch_all(ex) @@ -565,6 +573,7 @@ async fn imply_in_tx( TEXT = crate::implication_rules::object_text_sql(), RULE_MATCH = crate::implication_rules::RULE_MATCH, )) + .persistent(false) .bind(kb_id) .fetch_all(&mut **tx) .await?; @@ -587,6 +596,7 @@ async fn imply_in_tx( ORDER BY e.id", KIND = kind_word_sql("e.specific_type"), )) + .persistent(false) .bind(kb_id) .fetch_all(&mut **tx) .await?; @@ -600,6 +610,7 @@ async fn imply_in_tx( to: d.valid_to, to_precision: d.valid_to_precision.as_deref(), attested_at: d.attested_from, + end_after_period: false, }; let value; let object = match (d.object_id, &d.object_value) { diff --git a/crates/utopia-store/src/materialize_plan_tests.rs b/crates/utopia-store/src/materialize_plan_tests.rs index efa06d92c..507b93667 100644 --- a/crates/utopia-store/src/materialize_plan_tests.rs +++ b/crates/utopia-store/src/materialize_plan_tests.rs @@ -1,23 +1,31 @@ -//! The recompute plans each statement for the base it is given (#1298: a generic plan of the -//! source cleanup ran for hours on a production base where the custom plan takes two seconds). -use super::materialize_in_tx; -use sqlx::PgPool; +//! The recompute's whole-base statements are planned for the base they are given (#1298): a cached +//! statement switches to a generic plan after a few runs, and the generic plan of the source +//! cleanup ran for hours on a production base where the plan for the base's own id takes seconds. +use super::materialize; +use sqlx::postgres::PgPoolOptions; use uuid::Uuid; #[tokio::test] -async fn a_recompute_plans_its_statements_for_the_base_it_is_given() -> anyhow::Result<()> { +async fn a_recompute_leaves_none_of_its_whole_base_statements_cached_on_its_connection( +) -> anyhow::Result<()> { let Some(url) = crate::test_db::url() else { return Ok(()); }; - let pool = PgPool::connect(&url).await?; - let mut tx = pool.begin().await?; - - materialize_in_tx(&mut tx, Uuid::now_v7()).await?; - let mode: String = sqlx::query_scalar("SELECT current_setting('plan_cache_mode')") - .fetch_one(&mut *tx) + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(&url) .await?; - tx.rollback().await?; + let base = Uuid::now_v7(); + + materialize(&pool, base).await?; + materialize(&pool, base).await?; + let cached: Vec = sqlx::query_scalar( + "SELECT statement FROM pg_prepared_statements + WHERE statement LIKE '%kb_id = $1%' AND statement NOT LIKE '%pg_prepared_statements%'", + ) + .fetch_all(&pool) + .await?; - assert_eq!(mode, "force_custom_plan"); + assert_eq!(cached, Vec::::new()); Ok(()) } diff --git a/crates/utopia-store/src/pending.rs b/crates/utopia-store/src/pending.rs index 822803515..8d581902d 100644 --- a/crates/utopia-store/src/pending.rs +++ b/crates/utopia-store/src/pending.rs @@ -315,6 +315,7 @@ pub async fn confirm(pool: &PgPool, kb_id: Uuid, id: Uuid) -> AppResult { diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index 913b46e9d..283ad7ea5 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -193,7 +193,28 @@ struct Candidate { canonical_name: String, profile_embedding: Option, profile_n: i32, - degree: i64, +} + +/// 这些实体各自的度数（不数名字事实），与 `ids` 同序；没有事实的是 0。 +async fn degrees(pool: &PgPool, kb_id: Uuid, ids: &[Uuid]) -> AppResult> { + let rows: HashMap = sqlx::query_as(&format!( + "SELECT n.id, count(*) + FROM unnest($2::uuid[]) AS n(id) + JOIN facts f ON f.kb_id = $1 AND (f.subject_id = n.id OR f.object_id = n.id) + AND f.invalidated_at IS NULL AND {not_name} + GROUP BY n.id", + not_name = crate::names::not_a_name("f"), + )) + .bind(kb_id) + .bind(ids) + .fetch_all(pool) + .await? + .into_iter() + .collect(); + Ok(ids + .iter() + .map(|id| rows.get(id).copied().unwrap_or(0)) + .collect()) } /// 消解结果：mention 落到了哪个实体；附带需要入队的疑似重复审核对 @@ -428,11 +449,9 @@ async fn resolve_by_name( let keys = recall_keys(&name); // 名字召回读名字事实（0041 决定 1）：本名、简称、曾用名都是 `known_as` 上的一条。 // 度数不数名字事实——名字不是一条「关于它的事」，数进去每个实体都凭空多一 + // 度数要等真有几个候选得比时才数：每个候选一条计数子查询，大库上很贵（#1296） let candidates: Vec = sqlx::query_as(&format!( - "SELECT e.id, e.canonical_name, e.profile_embedding, e.profile_n, - (SELECT count(*) FROM facts f - WHERE f.kb_id = e.kb_id AND (f.subject_id = e.id OR f.object_id = e.id) - AND f.invalidated_at IS NULL AND {not_name}) AS degree + "SELECT e.id, e.canonical_name, e.profile_embedding, e.profile_n FROM entities e -- IS NOT DISTINCT FROM 而不是 =（0009 的那个陷阱）：开放图谱里的实体都没有类 -- （类由对齐来定），`type_id = NULL` 永远不成立，同名的它就永远撞不上—— @@ -442,7 +461,6 @@ async fn resolve_by_name( -- 不是召回的桥——两篇文档里描述得一样的两个东西不能因此接到一起 AND e.description IS NULL AND (lower(e.canonical_name) = ANY($3) OR {named})", - not_name = crate::names::not_a_name("f"), named = crate::names::has_name_in("e", 1, 3), )) .bind(kb_id) @@ -463,10 +481,19 @@ async fn resolve_by_name( let Some(ctx) = context else { // 无向量可比：v1 兼容 —— 归并到事实最多的同名候选 - let best = candidates - .iter() - .max_by_key(|c| c.degree) - .expect("non-empty"); + let best = match candidates.as_slice() { + [only] => only, + _ => { + let ids: Vec = candidates.iter().map(|c| c.id).collect(); + let degree = degrees(pool, kb_id, &ids).await?; + candidates + .iter() + .zip(degree) + .max_by_key(|(_, d)| *d) + .map(|(c, _)| c) + .expect("non-empty") + } + }; touch_entity(pool, best.id).await?; return Ok(Resolution { entity_id: best.id, @@ -486,7 +513,7 @@ async fn resolve_by_name( // 有画像的候选算相似度；无画像（历史数据/无 embedding 期创建）单独归类 let mut scored: Vec<(&Candidate, f32)> = Vec::new(); - let mut unprofiled: Option<&Candidate> = None; + let mut unprofiled: Vec<&Candidate> = Vec::new(); for c in &candidates { match c .profile_embedding @@ -494,11 +521,7 @@ async fn resolve_by_name( .and_then(|p| cosine(p.as_slice(), ctx)) { Some(sim) => scored.push((c, sim)), - None => { - if unprofiled.map(|u| c.degree > u.degree).unwrap_or(true) { - unprofiled = Some(c); - } - } + None => unprofiled.push(c), } } // 最高分候选：并列时保留先遇到的那个（与旧的 `sim > s` 严格大于一致） @@ -573,6 +596,19 @@ async fn resolve_by_name( }); } } + let unprofiled = match unprofiled.as_slice() { + [] => None, + [only] => Some(*only), + many => { + let ids: Vec = many.iter().map(|c| c.id).collect(); + let degree = degrees(pool, kb_id, &ids).await?; + many.iter() + .copied() + .zip(degree) + .reduce(|acc, cur| if cur.1 > acc.1 { cur } else { acc }) + .map(|(c, _)| c) + } + }; if let Some(c) = unprofiled { // 无画像候选无从判别：v1 兼容归并，并用本次上下文初始化画像 update_profile(pool, c.id, c.profile_n, ctx).await?; @@ -2060,14 +2096,15 @@ pub async fn merge_entities( .collect(); // 合并后 SPO+valid_from 重复的 live 事实：留最早 recorded_at 的一条，其余作废。 + // 没有谓词的行不参与：开放陈述的不同短语都带 NULL，不能因此当成同一句话。 // // **宾语两侧都要分组。** 字面值事实的 object_id 全是 NULL，只按它分组就等于 // 把同主同谓下的**所有值**当成同一条断言：一次合并之后， // (公司, 成立年份, 2015) 与 (公司, 注册资本, …) 之外，同谓词的多个值只活得下来 // 最早记的那一个，其余无声消失，且没有 supersedes 可查。 let dups: Vec<(Vec,)> = sqlx::query_as( - "SELECT (array_agg(id ORDER BY recorded_at))[2:] FROM facts - WHERE kb_id = $1 AND invalidated_at IS NULL + "SELECT array_agg(id ORDER BY recorded_at, id) FROM facts + WHERE kb_id = $1 AND invalidated_at IS NULL AND predicate_id IS NOT NULL AND (subject_id = $2 OR object_id = $2) GROUP BY subject_id, predicate_id, object_id, object_value, valid_from HAVING count(*) > 1", @@ -2076,7 +2113,21 @@ pub async fn merge_entities( .bind(target_id) .fetch_all(&mut *tx) .await?; - let dup_ids: Vec = dups.into_iter().flat_map(|(ids,)| ids).collect(); + let mut dup_ids = Vec::new(); + for (ids,) in dups { + // 先给存活行接上来源，否则下一次物化可能作废它，再按原文重建，丢掉人的修正。 + // 旧行的链接暂留：撤回合并还用得上；物化按它原有的规则清理失效链接。 + sqlx::query( + "INSERT INTO typed_fact_sources (fact_id, statement_id) + SELECT $1, statement_id FROM typed_fact_sources WHERE fact_id = ANY($2) + ON CONFLICT DO NOTHING", + ) + .bind(ids[0]) + .bind(&ids[1..]) + .execute(&mut *tx) + .await?; + dup_ids.extend_from_slice(&ids[1..]); + } if !dup_ids.is_empty() { sqlx::query("UPDATE facts SET invalidated_at = now() WHERE id = ANY($1)") .bind(&dup_ids) diff --git a/crates/utopia-store/src/temporal.rs b/crates/utopia-store/src/temporal.rs index 7b21cef75..85e6d9853 100644 --- a/crates/utopia-store/src/temporal.rs +++ b/crates/utopia-store/src/temporal.rs @@ -590,7 +590,7 @@ async fn tidy( for (id, end) in diff(&rows, ends) { // 重新打开的行只是开着，终点不是谁推出来的 let derived = end != End::Open; - if let Some(corrected) = rewrite_end_tx(tx, id, &end, derived, false).await? { + if let Some(corrected) = rewrite_end_tx(tx, id, &end, derived, false, false).await? { rewritten.insert(id, corrected); report.corrected.push(corrected); } @@ -873,13 +873,51 @@ pub async fn retract(pool: &PgPool, kb_id: Uuid, fact_id: Uuid) -> AppResult( pool: impl sqlx::Acquire<'a, Database = Postgres>, fact_id: Uuid, valid_to: DateTime, valid_to_precision: &str, +) -> AppResult> { + let valid_to = crate::graph::truncate_to(valid_to, Some(valid_to_precision)); + let mut tx = pool.begin().await?; + let row: Option<(Option>, Option)> = sqlx::query_as( + "SELECT valid_from, valid_from_precision FROM facts + WHERE id = $1 AND invalidated_at IS NULL FOR UPDATE", + ) + .bind(fact_id) + .fetch_optional(&mut *tx) + .await?; + let Some((from, from_precision)) = row else { + return Ok(None); + }; + // 锁住起点后再比，避免并发修正穿过检查；两端都按各自精度截断（#1294）。 + // 放在审核的 store 入口，也管住冲突默认取新事实起点的闭合，不改原文写入的规则。 + if from + .is_some_and(|from| valid_to < crate::graph::truncate_to(from, from_precision.as_deref())) + { + return Err(utopia_core::AppError::invalid( + "interval_inverted", + "The start of the interval is after its end.", + )); + } + // 按日期关上的（审核里关、冲突的 close_at、违规的 fact_closed）给的就是终点那一刻，不标 + let end = End::At(valid_to, valid_to_precision.to_string()); + let corrected = rewrite_end_tx(&mut tx, fact_id, &end, false, false, false).await?; + tx.commit().await?; + Ok(corrected) +} + +/// [`close_superseded`]，外加这个终点是不是原文所说时段之后的那一刻（`end_after_period`， +/// #994）：写入路径把「2024 年离开」关在 2025-01-01 时标上，读者才印得出 2024 +pub(crate) async fn close_stated<'a>( + pool: impl sqlx::Acquire<'a, Database = Postgres>, + fact_id: Uuid, + valid_to: DateTime, + valid_to_precision: &str, + after_period: bool, ) -> AppResult> { // 闭合点截到它的精度（0024）：月精度的闭合就是那个月的 1 日 0 点 let end = End::At( @@ -887,7 +925,7 @@ pub async fn close_superseded<'a>( valid_to_precision.to_string(), ); let mut tx = pool.begin().await?; - let corrected = rewrite_end_tx(&mut tx, fact_id, &end, false, false).await?; + let corrected = rewrite_end_tx(&mut tx, fact_id, &end, false, false, after_period).await?; tx.commit().await?; Ok(corrected) } @@ -903,19 +941,28 @@ pub async fn close_with_unknown_end<'a>( attested_at: Option>, ) -> AppResult> { let mut tx = pool.begin().await?; - let corrected = - rewrite_end_tx(&mut tx, fact_id, &End::Unknown(attested_at), false, true).await?; + let corrected = rewrite_end_tx( + &mut tx, + fact_id, + &End::Unknown(attested_at), + false, + true, + false, + ) + .await?; tx.commit().await?; Ok(corrected) } /// 原文说出了一条**引擎关上**的行的终点：改写成原文说的，此后不再重算（#679 第三轮评审）。 -/// `valid_to` 为 `None` 是「结束了，不知哪天」，锚在 `attested_at`。返回修正行 id； -/// 行已作废，或它的终点本来就是写明的，返回 `None` +/// `valid_to` 为 `None` 是「结束了，不知哪天」，锚在 `attested_at`；`after_period`：原文说的 +/// 是那一刻之前的那个时段（#994）。返回修正行 id；行已作废，或它的终点本来就是写明的， +/// 返回 `None` pub async fn state_derived_end<'a>( pool: impl sqlx::Acquire<'a, Database = Postgres>, fact_id: Uuid, valid_to: Option<(DateTime, &str)>, + after_period: bool, attested_at: Option>, ) -> AppResult> { let end = match valid_to { @@ -933,15 +980,60 @@ pub async fn state_derived_end<'a>( .fetch_optional(&mut *tx) .await?; let corrected = match derived { - Some(true) => rewrite_end_tx(&mut tx, fact_id, &end, false, false).await?, + Some(true) => rewrite_end_tx(&mut tx, fact_id, &end, false, false, after_period).await?, _ => None, }; tx.commit().await?; Ok(corrected) } +/// 一条写明的终点又被另一种说法说了一遍：「until 2025」关上的行又读到「left in 2024」。 +/// 时刻不变，改写成带标记的一行（#994）；反过来的顺序在收尾那条路上已经标了，两种顺序 +/// 写下同一行。 +/// +/// 调用方（`insert_fact_on`）只比了时刻，这里锁住那一行再查两件事：精度与这次说的相同—— +/// 日精度写明的 2025-01-01 是那一天，不是 2024 年的尽头；终点不是人改过的 +/// （`corrected_ends` 为空或只有起点）——人把显示的 2024 改成 2025，就是要到 2025 年。 +/// 已经标过的不再标。那一行带不带得了标记（引擎画的终点、没有起点的行带不了），读出来交给 +/// `graph::period_mark_fits` 判。返回修正行 id；不必标的返回 `None` +pub(crate) async fn mark_end_after_period<'a>( + pool: impl sqlx::Acquire<'a, Database = Postgres>, + fact_id: Uuid, + precision: &str, +) -> AppResult> { + let mut tx = pool.begin().await?; + #[allow(clippy::type_complexity)] + let locked_row: Option<(Option>, Option>, bool, bool)> = + sqlx::query_as( + "SELECT f.valid_from, f.valid_to, f.end_derived, + COALESCE(r.temporal = 'state', false) AS declared_state + FROM facts f LEFT JOIN relation_types r ON r.id = f.predicate_id + WHERE f.id = $1 AND f.invalidated_at IS NULL + AND f.valid_to_precision = $2 AND NOT f.end_after_period + AND (f.corrected_ends IS NULL OR f.corrected_ends = 'start') + FOR UPDATE OF f", + ) + .bind(fact_id) + .bind(precision) + .fetch_optional(&mut *tx) + .await?; + let marked = match locked_row { + // 起点那一半人改过的照旧是人的：改写的 CASE 把 'start' 原样带过去 + Some((from, Some(at), end_derived, declared_state)) + if crate::graph::period_mark_fits(declared_state, from, Some(at), end_derived) => + { + let end = End::At(at, precision.to_string()); + rewrite_end_tx(&mut tx, fact_id, &end, false, false, true).await? + } + _ => None, + }; + tx.commit().await?; + Ok(marked) +} + /// 作废 + 改写一行的终点。`derived`：这个终点是引擎按时间线推出来的（0057），之后随 -/// 时间线重算；`require_open`：只改还开着的行。 +/// 时间线重算；`require_open`：只改还开着的行；`after_period`：终点是原文所说时段之后的 +/// 那一刻（#994）。 /// /// 先 `FOR UPDATE` 锁住旧行：并行的两次改写只有一次看得见它还活着，另一次拿到 `None`， /// 不会各插一条修正行；往这一行上添证据的也要等它（`graph::add_evidence`），证据不会落在 @@ -952,6 +1044,7 @@ async fn rewrite_end_tx( end: &End, derived: bool, require_open: bool, + after_period: bool, ) -> AppResult> { let alive: Option<(Option>, Option)> = sqlx::query_as( "SELECT valid_from, predicate_id FROM facts @@ -983,13 +1076,22 @@ async fn rewrite_end_tx( End::At(at, precision) => (Some(*at), Some(precision.as_str()), None), End::Unknown(anchor) => (None, Some(crate::graph::ENDED_UNKNOWN), *anchor), }; + // 标记只落在带得了它的终点上（`graph::period_mark_fits`）：时间线画的、不知哪天的、重新 + // 打开的终点，没有起点或属性不是状态的行，一律不带——起点和谓词是上面锁住时读的。要标时 + // 才去查属性：不标的改写（时间线重画、关成不知哪天）不多一次查询 + let after_period = after_period && { + let declared_state = predicate.is_some() + && crate::graph::predicate_temporal(&mut **tx, predicate).await? + == crate::graph::Temporal::State; + crate::graph::period_mark_fits(declared_state, from, valid_to, derived) + }; let corrected = Uuid::now_v7(); sqlx::query( "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, object_value, valid_from, valid_from_precision, valid_from_grade, valid_to, valid_to_precision, confidence, supersedes, attested_from, attested_by, attested_to, end_derived, - from_statement_id, implied, corrected_ends) + from_statement_id, implied, corrected_ends, end_after_period) SELECT $1, kb_id, subject_id, predicate_id, object_id, object_value, valid_from, valid_from_precision, valid_from_grade, $3, $4, confidence, id, attested_from, attested_by, @@ -998,7 +1100,8 @@ async fn rewrite_end_tx( -- 终点在这里被重新写下（文档说了终点，或时间线推了一个），人改过的终点就不在了； -- 起点那一半照旧 CASE corrected_ends WHEN 'both' THEN 'start' WHEN 'end' THEN NULL - ELSE corrected_ends END + ELSE corrected_ends END, + $7 FROM facts WHERE id = $2", ) .bind(corrected) @@ -1007,6 +1110,7 @@ async fn rewrite_end_tx( .bind(precision) .bind(anchor) .bind(derived) + .bind(after_period) .execute(&mut **tx) .await?; carry_open_conflicts(tx, fact_id, corrected).await?; @@ -1073,11 +1177,11 @@ pub async fn rehome_tx( valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, derived_by_rule, supersedes, attested_from, attested_to, end_derived, - from_statement_id, implied, corrected_ends) + from_statement_id, implied, corrected_ends, end_after_period) SELECT $1, kb_id, COALESCE($3, subject_id), predicate_id, COALESCE($4, object_id), object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, derived_by_rule, id, attested_from, attested_to, end_derived, - from_statement_id, implied, corrected_ends + from_statement_id, implied, corrected_ends, end_after_period FROM facts WHERE id = $2", ) .bind(moved) @@ -1184,12 +1288,15 @@ async fn copy_evidence( /// 改写同一条规矩（[`copy_materialization_links`]）。不带过去的话，陈述不再被活着的 /// 类型化行代表，画面上它按原话、原来的区间回来，下一轮物化还照它再算一行。 /// -/// 改了哪一端记在修正行上（`corrected_ends`，#970）：值或精度变了的那一端，与旧行已有的 -/// 取并集，与修正同一个事务——事实行据此说「这个日期是人改的」，不再靠事后那条审计。 +/// 改了哪一端记在修正行上（`corrected_ends`，#970）：值或精度（终点还有标记）变了的那一端， +/// 与旧行已有的取并集，与修正同一个事务——事实行据此说「这个日期是人改的」，不再靠事后那条审计。 /// 人交来的是整段区间，改完两端都是人的（`end_derived` 清掉，时间线不再重画它）：原来 /// 是时间线推出来的终点，人没改它的值也是把它钉住了，一样算人的；原来就是原文说的那一端 /// 没改，仍是原文的 /// +/// 终点是不是原文所说时段之后的那一刻（`end_after_period`，#994）由调用方给，这里只归一： +/// 修正路由按请求定它，从不替人换算 +/// /// 返回修正行 id；`None` 表示这条已被并发改写或作废，本次没有动手。 pub async fn correct_interval( pool: &PgPool, @@ -1209,15 +1316,17 @@ pub async fn correct_interval( let validity = match predicate { Some(_) => validity.truncated().under(temporal)?, None => validity.truncated(), - }; + } + .drop_period_mark_unless(predicate.is_some() && temporal == crate::graph::Temporal::State); let mut tx = pool.begin().await?; let corrected = Uuid::now_v7(); + // 标记变了也算终点变了（#994）：人把显示的 2024 改成 2025，存的时刻没动，说的却是另一年 let inserted: Option<(Uuid,)> = sqlx::query_as( "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, supersedes, attested_from, attested_to, - from_statement_id, implied, corrected_ends) + from_statement_id, implied, corrected_ends, end_after_period) SELECT $1, kb_id, subject_id, predicate_id, object_id, object_value, $3, $4, $5, $6, confidence, id, attested_from, @@ -1225,7 +1334,8 @@ pub async fn correct_interval( from_statement_id, implied, CASE WHEN start_marked AND end_marked THEN 'both' WHEN start_marked THEN 'start' - WHEN end_marked THEN 'end' END + WHEN end_marked THEN 'end' END, + $7 FROM (SELECT f.*, c.start_changed OR COALESCE(f.corrected_ends IN ('start', 'both'), false) AS start_marked, @@ -1238,6 +1348,7 @@ pub async fn correct_interval( AS start_changed, f.valid_to IS DISTINCT FROM $5::timestamptz OR f.valid_to_precision IS DISTINCT FROM $6::text + OR f.end_after_period IS DISTINCT FROM $7::boolean AS end_changed) c WHERE f.id = $2 AND f.invalidated_at IS NULL) old RETURNING id", @@ -1248,6 +1359,7 @@ pub async fn correct_interval( .bind(validity.from_precision) .bind(validity.to) .bind(validity.to_precision) + .bind(validity.end_after_period) .fetch_optional(&mut *tx) .await?; // 已被并发修正过：不重复动手（与 close_superseded 同一防线） diff --git a/crates/utopia-store/tests/a_corrected_row_says_which_end_changed.rs b/crates/utopia-store/tests/a_corrected_row_says_which_end_changed.rs index 581d46b22..a421e6b72 100644 --- a/crates/utopia-store/tests/a_corrected_row_says_which_end_changed.rs +++ b/crates/utopia-store/tests/a_corrected_row_says_which_end_changed.rs @@ -36,6 +36,7 @@ fn between(from: &str, to: Option<&str>) -> Validity<'static> { to_precision: to.map(|_| "day"), attested_at: None, from_grade: None, + ..Default::default() } } diff --git a/crates/utopia-store/tests/store/a_fact_awaits_a_nod.rs b/crates/utopia-store/tests/store/a_fact_awaits_a_nod.rs index 0249cc06e..c2a0a0825 100644 --- a/crates/utopia-store/tests/store/a_fact_awaits_a_nod.rs +++ b/crates/utopia-store/tests/store/a_fact_awaits_a_nod.rs @@ -120,6 +120,7 @@ async fn a_remembered_fact_waits_for_a_nod() -> anyhow::Result<()> { to_precision: None, attested_at: None, from_grade: None, + ..Default::default() }, confidence: 0.9, chunk_id: f.chunk, diff --git a/crates/utopia-store/tests/store/a_merge_keeps_distinct_statements.rs b/crates/utopia-store/tests/store/a_merge_keeps_distinct_statements.rs new file mode 100644 index 000000000..1f0774a8d --- /dev/null +++ b/crates/utopia-store/tests/store/a_merge_keeps_distinct_statements.rs @@ -0,0 +1,108 @@ +//! A merge keeps distinct open phrases and the sources of deduplicated typed facts. + +use sqlx::PgPool; +use utopia_store::resolution; +use uuid::Uuid; + +#[tokio::test] +async fn merging_preserves_open_phrases_and_the_surviving_typed_rows_sources() -> anyhow::Result<()> +{ + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let [org, ws, kb, class, target, source, place, property, binding, withdrawn, ships, receives, based, kept, dropped] = + std::array::from_fn(|_| Uuid::now_v7()); + let run = async { + // Locally generated UUIDs only. The keeper has a corrected end, but its + // original source was withdrawn; the duplicate has the only live source. + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','merge-statements'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','merge-statements'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','merge-statements'); + INSERT INTO entity_types(id,kb_id,key,label) VALUES ('{class}','{kb}','thing','Thing'); + INSERT INTO entities(id,kb_id,type_id,canonical_name) VALUES + ('{target}','{kb}','{class}','Harbor Bakery'), + ('{source}','{kb}','{class}','Harbor'), + ('{place}','{kb}','{class}','Port Ellen'); + INSERT INTO relation_types (id,kb_id,key,label,temporal) + VALUES ('{property}','{kb}','headquartered_in','headquartered in','state'); + INSERT INTO phrase_bindings + (id,kb_id,phrase,subject_type_id,object_type_id,object_is_value, + relation_type_id,direction,status) + VALUES ('{binding}','{kb}','is based in','{class}','{class}',false, + '{property}','forward','bound'); + INSERT INTO facts (id,kb_id,subject_id,object_id,layer,phrase,confidence) VALUES + ('{withdrawn}','{kb}','{target}','{place}','open','is based in',0.9), + ('{ships}','{kb}','{target}','{place}','open','ships to',0.9), + ('{receives}','{kb}','{target}','{place}','open','receives from',0.9), + ('{based}','{kb}','{source}','{place}','open','is based in',0.9); + UPDATE facts SET invalidated_at = now() WHERE id = '{withdrawn}'; + INSERT INTO facts + (id,kb_id,subject_id,predicate_id,object_id,confidence,from_statement_id, + recorded_at,valid_to,valid_to_precision,corrected_ends) VALUES + ('{kept}','{kb}','{target}','{property}','{place}',0.9,'{withdrawn}', + '2024-01-01','2025-01-01','year','end'), + ('{dropped}','{kb}','{source}','{property}','{place}',0.9,'{based}', + '2024-02-01',NULL,NULL,NULL); + INSERT INTO typed_fact_sources (fact_id,statement_id) VALUES + ('{kept}','{withdrawn}'), ('{dropped}','{based}');" + )) + .execute(&pool) + .await?; + + let merge = resolution::merge_entities(&pool, kb, source, target, None, "test").await?; + let live: Vec = sqlx::query_scalar( + "SELECT id FROM facts WHERE kb_id = $1 AND subject_id = $2 + AND layer = 'open' AND invalidated_at IS NULL ORDER BY id", + ) + .bind(kb) + .bind(target) + .fetch_all(&pool) + .await?; + let mut expected = vec![ships, receives, based]; + expected.sort(); + assert_eq!(live, expected, "different phrases, including those already on the target, survive"); + let invalidated: Vec = sqlx::query_scalar( + "SELECT invalidated_facts FROM entity_merges WHERE id = $1", + ) + .bind(merge) + .fetch_one(&pool) + .await?; + assert_eq!(invalidated, vec![dropped]); + let has_source: bool = sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM typed_fact_sources WHERE fact_id = $1 AND statement_id = $2)", + ) + .bind(kept) + .bind(based) + .fetch_one(&pool) + .await?; + assert!(has_source, "the surviving typed row inherits the duplicate's live source"); + + assert_eq!( + utopia_store::materialize::materialize(&pool, kb).await?, + utopia_store::materialize::Outcome::default(), + "materialization must neither retire the keeper nor rederive its uncorrected interval" + ); + let unchanged: bool = sqlx::query_scalar( + "SELECT invalidated_at IS NULL AND valid_to = '2025-01-01'::timestamptz + AND valid_to_precision = 'year' AND corrected_ends = 'end' + FROM facts WHERE id = $1", + ) + .bind(kept) + .fetch_one(&pool) + .await?; + assert!(unchanged); + anyhow::Ok(()) + } + .await; + sqlx::query("DELETE FROM knowledge_bases WHERE id = $1") + .bind(kb) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + run +} diff --git a/crates/utopia-store/tests/store/a_moment_marks_its_state.rs b/crates/utopia-store/tests/store/a_moment_marks_its_state.rs index 7f2634eb8..d454a9979 100644 --- a/crates/utopia-store/tests/store/a_moment_marks_its_state.rs +++ b/crates/utopia-store/tests/store/a_moment_marks_its_state.rs @@ -14,6 +14,10 @@ //! 照旧；已有的空段不并进后来的观察，也不被它关上；勘误改不了一段不成立的时间，也不能把 //! 一刻的事件改成状态；规则没有 marks，只说了一刻的陈述不蕴含状态，只有起点的照旧。 //! +//! 终点因此挪到时段尽头的行记着原文说的是那个时段（`end_after_period`，#994）：收尾、精化 +//! 两条路都记；关在「until 2025」的行再读到「left in 2024」也记（人只改过起点的也记），时间线画 +//! 的终点被它说出来也记；搬主宾、勘误改出来的行带着它，改成事件的不带。 +//! //! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 use chrono::{DateTime, Utc}; @@ -45,6 +49,7 @@ fn moment() -> Validity<'static> { to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() } } @@ -57,6 +62,7 @@ fn ending(at: DateTime) -> Validity<'static> { to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() } } @@ -274,6 +280,21 @@ impl Base { ) } + /// Lin Zhao —works_for→ object，走写入口（抽取、点头、人写的事实同一个门） + async fn write_to(&self, object: Uuid, v: Validity<'_>) -> anyhow::Result { + let (fact, _) = graph::insert_fact( + &self.pool, + self.kb, + self.lin, + Some(self.works_for), + object, + v, + 0.9, + ) + .await?; + Ok(fact) + } + async fn remove(self) -> anyhow::Result<()> { sqlx::query("DELETE FROM organizations WHERE id = $1") .bind(self.org) @@ -997,7 +1018,8 @@ async fn a_row_a_person_set_to_a_single_date_is_not_retired() -> anyhow::Result< /// 终点说的是一个时段，比的却是时刻（#975 评审）：「left … in 2024」存成 2024-01-01，年精度。 /// 「works for … from 2024-03-01」从这个时段里开始，关在时段的尽头 2025-01-01，而不是另立一行 -/// `- → 2024`、让它一直开着到 2025 年以后。先读到哪一句都一样 +/// `- → 2024`、让它一直开着到 2025 年以后。先读到哪一句都一样；行上记着终点说的是 2024 +/// （#994：收尾那条路和精化那条路都记） #[tokio::test] async fn a_coarse_end_closes_a_row_that_starts_inside_its_period() -> anyhow::Result<()> { let Some(pool) = pool().await? else { @@ -1008,6 +1030,7 @@ async fn a_coarse_end_closes_a_row_that_starts_inside_its_period() -> anyhow::Re t("2024-01-01T00:00:00Z"), t("2025-01-01T00:00:00Z"), ); + let mut marks: Vec> = Vec::new(); for order in [["works for", "left"], ["left", "works for"]] { let base = Base::new(&pool).await?; let run = async { @@ -1037,12 +1060,22 @@ async fn a_coarse_end_closes_a_row_that_starts_inside_its_period() -> anyhow::Re "{order:?}: not still working there in 2025" ); assert_eq!(materialize(&pool, base.kb).await?, Outcome::default()); + marks.push( + sqlx::query_scalar( + "SELECT end_after_period FROM facts + WHERE kb_id = $1 AND layer = 'typed' AND invalidated_at IS NULL", + ) + .bind(base.kb) + .fetch_all(&pool) + .await?, + ); anyhow::Ok(()) } .await; base.remove().await?; run?; } + assert_eq!(marks, [[true], [true]], "the end names 2024, not 2025"); Ok(()) } @@ -1108,6 +1141,245 @@ async fn a_coarse_end_leaves_rows_outside_its_period_as_before() -> anyhow::Resu Ok(()) } +/// 一个终点，两种说法（#994）：人写的 2024-03-01 起的一行关在「until 2025」，文档又说「left in +/// 2024」，同一个终点：标上，并进那一行，它仍是人的（反过来的顺序是收尾那条路）；另一份文档再说 +/// 一次，标过的原样留着。人只把起点挪到 4 月的一行（Dune）也标。日精度写明的 2025-01-01（Aster） +/// 是那一天，人改过的终点（Cobalt）是人的，都不改写 +#[tokio::test] +async fn one_end_named_two_ways_is_marked() -> anyhow::Result<()> { + let Some(pool) = pool().await? else { + return Ok(()); + }; + let base = Base::new(&pool).await?; + let run = async { + let (year, next_year) = (t("2024-01-01T00:00:00Z"), t("2025-01-01T00:00:00Z")); + let since = Validity::starting(Some(t("2024-03-01T00:00:00Z")), Some("day")); + let until_2025 = Validity { + to_precision: Some("year"), + ..ending(next_year) + }; + let (cobalt, dune) = (Uuid::now_v7(), Uuid::now_v7()); + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) + VALUES ($1, $2, $3, 'Cobalt'), ($4, $2, $3, 'Dune')", + ) + .bind(cobalt) + .bind(base.kb) + .bind(base.company) + .bind(dune) + .execute(&pool) + .await?; + for (object, end) in [(base.meridian, until_2025), (base.aster, ending(next_year))] { + base.write_to(object, since).await?; + base.write_to(object, end).await?; + } + let by_hand = Validity { + to: Some(next_year), + to_precision: Some("year"), + ..since + }; + let open = base.write_to(cobalt, since).await?; + utopia_store::temporal::correct_interval(&pool, open, by_hand).await?; + let stated = base.write_to(dune, by_hand).await?; + let april = Validity { + from: Some(t("2024-04-01T00:00:00Z")), + ..by_hand + }; + utopia_store::temporal::correct_interval(&pool, stated, april).await?; + for object in [base.meridian, base.meridian, base.aster, cobalt, dune] { + base.statement_to("left", object, Some(year), Some(year), "year") + .await?; + } + base.bind("left", Some("end")).await?; + materialize(&pool, base.kb).await?; + // 每个宾语活着的那一行：标记，仍是人的（不是这句算出来的），那边被取代过几行 + let rows: Vec<(String, bool, bool, i64)> = sqlx::query_as( + "SELECT o.canonical_name, f.end_after_period, f.from_statement_id IS NULL, + (SELECT count(*) FROM facts g WHERE g.layer = 'typed' + AND g.object_id = f.object_id AND g.invalidated_at IS NOT NULL) + FROM facts f JOIN entities o ON o.id = f.object_id + WHERE f.kb_id = $1 AND f.layer = 'typed' AND f.invalidated_at IS NULL + ORDER BY 1", + ) + .bind(base.kb) + .fetch_all(&pool) + .await?; + // Meridian 开着的那行先被「until 2025」关上、再被标上，各取代一次；Dune 改过一次起点 + let expected = [ + ("Aster Labs", false, 1), + ("Cobalt", false, 1), + ("Dune", true, 2), + ("Meridian Systems", true, 2), + ] + .map(|(o, mark, gone)| (o.to_string(), mark, true, gone)); + assert_eq!(rows, expected); + anyhow::Ok(()) + } + .await; + base.remove().await?; + run +} + +/// 时间线画的终点被原文说出来（#994）：works_for 声明成函数性、继任者 2025-01-01 起，时间线在那天 +/// 画上 Meridian 的终点（日精度）；「left in 2024」说的是同一刻，改成写明的年精度终点，记着说的是 +/// 2024 +#[tokio::test] +async fn a_drawn_end_the_text_names_as_a_period_is_marked() -> anyhow::Result<()> { + let Some(pool) = pool().await? else { + return Ok(()); + }; + let base = Base::new(&pool).await?; + let run = async { + let next_year = t("2025-01-01T00:00:00Z"); + sqlx::query("UPDATE relation_types SET functional = true WHERE id = $1") + .bind(base.works_for) + .execute(&pool) + .await?; + let since = Validity::starting(Some(t("2024-03-01T00:00:00Z")), Some("day")); + base.write_to(base.meridian, since).await?; + let next = Validity::starting(Some(next_year), Some("day")); + let next = base.write_to(base.aster, next).await?; + utopia_store::temporal::reconcile_moved_facts(&pool, base.kb, &[next]).await?; + // Meridian 那一行的终点：精度、是不是时间线画的、标记 + let end = || { + sqlx::query_as::<_, (String, bool, bool)>( + "SELECT valid_to_precision, end_derived, end_after_period FROM facts + WHERE object_id = $1 AND invalidated_at IS NULL", + ) + .bind(base.meridian) + .fetch_one(&pool) + }; + let drawn = end().await?; + let left_in_2024 = Validity { + to_precision: Some("year"), + ..ending(t("2024-01-01T00:00:00Z")) + }; + base.write_to(base.meridian, left_in_2024).await?; + assert_eq!( + (drawn, end().await?), + (("day".into(), true, false), ("year".into(), false, true)), + "drawn, then stated as the end of 2024" + ); + anyhow::Ok(()) + } + .await; + base.remove().await?; + run +} + +/// 标记随行走（#994）：撤回合并搬主宾用的 `rehome_tx` 带着它；勘误改出来的行接旧行的标记——改到 +/// 同起点、还开着的一行的宾语上，那一行经同起点那条路关上；改到同起点、终点是时间线画的一行的 +/// 宾语上，那个终点改成写明的——都带着它；属性改成事件的，写入口丢掉它 +#[tokio::test] +async fn a_marked_end_rides_through_a_rehome_and_a_revision() -> anyhow::Result<()> { + let Some(pool) = pool().await? else { + return Ok(()); + }; + let base = Base::new(&pool).await?; + let run = async { + let kb = base.kb; + let since = Validity::starting(Some(t("2024-03-01T00:00:00Z")), Some("day")); + let closed = Validity { + to: Some(t("2025-01-01T00:00:00Z")), + to_precision: Some("year"), + ..since + }; + let row = base.write_to(base.meridian, closed).await?; + let (cobalt, drawn, doc) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (company, lin, works_for) = (base.company, base.lin, base.works_for); + // Only locally generated UUIDs are interpolated into fixture SQL. + sqlx::raw_sql(&format!( + "UPDATE facts SET end_after_period = TRUE WHERE id = '{row}'; + INSERT INTO entities (id, kb_id, type_id, canonical_name) + VALUES ('{cobalt}', '{kb}', '{company}', 'Cobalt'); + INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, layer, confidence, + valid_from, valid_from_precision, valid_to, valid_to_precision, + end_derived) + VALUES ('{drawn}', '{kb}', '{lin}', '{works_for}', '{cobalt}', 'typed', 0.9, + '2024-03-01T00:00:00Z', 'day', '2025-01-01T00:00:00Z', 'day', TRUE); + INSERT INTO documents (id, kb_id, filename, sha256) + VALUES ('{doc}', '{kb}', 'cv.txt', 'x');" + )) + .execute(&pool) + .await?; + let mut tx = pool.begin().await?; + let moved = utopia_store::temporal::rehome_tx(&mut tx, row, None, Some(base.aster)) + .await? + .expect("the row was live"); + tx.commit().await?; + + // 勘误改三次：宾语改回 Meridian，那边已有一行 2024-03-01 起开着；改到 Cobalt，那边同起点的 + // 一行终点是时间线画的（日精度，上面写下的）；属性改成 acquired（事件） + let open = base.write_to(base.meridian, since).await?; + let mut candidate = Candidate { + fact_id: moved, + statement_id: None, + predicate_id: base.works_for, + subject_id: base.lin, + subject: "Lin Zhao".into(), + subject_class: Some("person".into()), + property: "works_for".into(), + property_label: "works for".into(), + object_id: Some(base.aster), + object: "Aster Labs".into(), + object_class: Some("organization".into()), + flag: Some("name_absent".into()), + quote: None, + }; + // 一行的标记，和它取代的是哪一行 + let read = |id: Uuid| { + sqlx::query_as::<_, (bool, Option)>( + "SELECT end_after_period, supersedes FROM facts WHERE id = $1", + ) + .bind(id) + .fetch_one(&pool) + }; + let mut rows = vec![read(moved).await?]; + let run_id = errata::start_run(&pool, kb, doc, 3, 0).await?; + for (property, object) in [ + (None, Some("Meridian Systems")), + (None, Some("Cobalt")), + (Some("acquired"), None), + ] { + let recorded = errata::record( + &pool, + kb, + ActionInput { + run_id, + document_id: doc, + candidate: Some(&candidate), + proposed: Proposed::Revise { + property: property.map(Into::into), + object: object.map(Into::into), + }, + reason: "the document says otherwise", + quote: Some("Lin Zhao left Aster Labs for Meridian Systems"), + document_text: "Lin Zhao left Aster Labs for Meridian Systems in 2024.", + }, + ) + .await?; + candidate.fact_id = recorded + .new_fact_id + .unwrap_or_else(|| panic!("{recorded:?}")); + rows.push(read(candidate.fact_id).await?); + } + assert_eq!( + rows, + [ + (true, Some(row)), + (true, Some(open)), + (true, Some(drawn)), + (false, None) + ], + "rehomed; revised onto an open row and a drawn end; onto an event" + ); + anyhow::Ok(()) + } + .await; + base.remove().await?; + run +} + /// 两个值同一刻开始的冲突（works_for 声明成函数性）：人选「关上旧的」却不给日期，旧的就关在 /// 新的起点——也是它自己的起点，一段不成立的状态，拒绝（`empty_state_span`）。冲突留着等人 /// 给一个日期 diff --git a/crates/utopia-store/tests/store/a_path_joins_two_entities.rs b/crates/utopia-store/tests/store/a_path_joins_two_entities.rs index 36d516ae6..4ba81a867 100644 --- a/crates/utopia-store/tests/store/a_path_joins_two_entities.rs +++ b/crates/utopia-store/tests/store/a_path_joins_two_entities.rs @@ -109,6 +109,7 @@ async fn seed(pool: &PgPool) -> anyhow::Result { to_precision: to.map(|_| "month"), attested_at: None, from_grade: None, + ..Default::default() }; let mut direct = Uuid::nil(); for (s, p, o, v) in [ diff --git a/crates/utopia-store/tests/store/a_retired_account.rs b/crates/utopia-store/tests/store/a_retired_account.rs index fd4374c81..ce84193ac 100644 --- a/crates/utopia-store/tests/store/a_retired_account.rs +++ b/crates/utopia-store/tests/store/a_retired_account.rs @@ -130,3 +130,103 @@ async fn the_last_admin_and_oneself_are_protected() -> anyhow::Result<()> { .await?; run } + +async fn concurrent_admin_removal(deactivate: [bool; 2]) -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let (org, a, b) = org_with_two_admins(&pool).await?; + let run = async { + let mut gate = pool.begin().await?; + sqlx::query("SELECT id FROM users WHERE org_id = $1 FOR UPDATE") + .bind(org) + .execute(&mut *gate) + .await?; + let mut tasks = tokio::task::JoinSet::new(); + let mut pids = Vec::new(); + for ((actor, target), retire) in [(a, b), (b, a)].into_iter().zip(deactivate) { + let writer = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await?; + pids.push( + sqlx::query_scalar::<_, i32>("SELECT pg_backend_pid()") + .fetch_one(&writer) + .await?, + ); + tasks.spawn(async move { + if retire { + utopia_store::accounts::deactivate_user(&writer, target, actor).await + } else { + utopia_store::accounts::set_system_admin(&writer, target, actor, false).await + } + }); + } + // Observe real lock waits, not elapsed time: before the fix both writers + // have passed their checks; afterwards one waits on the organization lock. + let waiting = tokio::time::timeout(std::time::Duration::from_secs(10), async { + loop { + let blocked: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity + WHERE pid = ANY($1) AND cardinality(pg_blocking_pids(pid)) > 0", + ) + .bind(&pids) + .fetch_one(&pool) + .await?; + if blocked == 2 { + break anyhow::Ok(()); + } + tokio::task::yield_now().await; + } + }) + .await; + gate.commit().await?; + waiting??; + let mut succeeded = 0; + while let Some(result) = + tokio::time::timeout(std::time::Duration::from_secs(10), tasks.join_next()).await? + { + match result? { + Ok(()) => succeeded += 1, + Err(utopia_core::AppError::Validation(_)) => {} + Err(error) => return Err(error.into()), + } + } + let remaining: i64 = sqlx::query_scalar( + "SELECT count(*) FROM users + WHERE org_id = $1 AND is_admin AND deactivated_at IS NULL", + ) + .bind(org) + .fetch_one(&pool) + .await?; + anyhow::Ok((succeeded, remaining)) + } + .await; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + let (succeeded, remaining) = run?; + assert_eq!( + remaining, 1, + "concurrent removals must retain an active administrator" + ); + assert_eq!(succeeded, 1, "exactly one removal must succeed"); + Ok(()) +} + +#[tokio::test] +async fn concurrent_revocations_keep_an_active_admin() -> anyhow::Result<()> { + concurrent_admin_removal([false, false]).await +} + +#[tokio::test] +async fn concurrent_deactivations_keep_an_active_admin() -> anyhow::Result<()> { + concurrent_admin_removal([true, true]).await +} + +#[tokio::test] +async fn concurrent_revocation_and_deactivation_keep_an_active_admin() -> anyhow::Result<()> { + concurrent_admin_removal([false, true]).await +} diff --git a/crates/utopia-store/tests/store/a_single_name_candidate_is_not_counted.rs b/crates/utopia-store/tests/store/a_single_name_candidate_is_not_counted.rs new file mode 100644 index 000000000..63febee75 --- /dev/null +++ b/crates/utopia-store/tests/store/a_single_name_candidate_is_not_counted.rs @@ -0,0 +1,84 @@ +//! 同名候选只有一个时没有可比的，不必数它的事实（#1296）。 +//! 一条连接的池子：sqlx 把它跑过的语句都缓存在 `pg_prepared_statements` 里，没跑过的就不在。 + +use sqlx::postgres::PgPoolOptions; +use utopia_store::names; +use uuid::Uuid; + +#[tokio::test] +async fn a_single_name_candidate_is_attached_without_counting_its_facts() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await?; + let (org, ws, kb) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (person, entity) = (Uuid::now_v7(), Uuid::now_v7()); + let tag = "single-name-candidate"; + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, $2)") + .bind(org) + .bind(tag) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, $3)") + .bind(ws) + .bind(org) + .bind(tag) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, $3)") + .bind(kb) + .bind(ws) + .bind(tag) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO entity_types (id, kb_id, key, label) VALUES ($1, $2, $3, $4)") + .bind(person) + .bind(kb) + .bind("person") + .bind("Person") + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO entities (id, kb_id, type_id, canonical_name) VALUES ($1, $2, $3, $4)", + ) + .bind(entity) + .bind(kb) + .bind(person) + .bind("林小满") + .execute(&pool) + .await?; + names::record(&pool, kb, entity, "林小满", None, None).await?; + + let run = async { + let r = utopia_store::resolution::resolve_mention( + &pool, + kb, + Some(person), + "林小满", + None, + None, + None, + &[], + ) + .await?; + assert!(!r.created && r.entity_id == entity, "唯一的同名候选就是它"); + let counted: Vec = sqlx::query_scalar( + "SELECT statement FROM pg_prepared_statements WHERE statement LIKE '%count(*)%facts f%'", + ) + .persistent(false) + .fetch_all(&pool) + .await?; + assert_eq!(counted, Vec::::new(), "只有一个候选，不该数事实"); + anyhow::Ok(()) + } + .await; + + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + run +} diff --git a/crates/utopia-store/tests/store/a_timeline_is_recomputed_from_the_rows_it_has.rs b/crates/utopia-store/tests/store/a_timeline_is_recomputed_from_the_rows_it_has.rs index f6216db57..079f7c750 100644 --- a/crates/utopia-store/tests/store/a_timeline_is_recomputed_from_the_rows_it_has.rs +++ b/crates/utopia-store/tests/store/a_timeline_is_recomputed_from_the_rows_it_has.rs @@ -120,6 +120,7 @@ async fn observe(pool: &PgPool, f: &Fixture, x: Seen<'_>) -> anyhow::Result anyhow::Result<()> to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await? diff --git a/crates/utopia-store/tests/store/a_wrong_time_can_be_corrected.rs b/crates/utopia-store/tests/store/a_wrong_time_can_be_corrected.rs index d06286d1d..c0e85d1f8 100644 --- a/crates/utopia-store/tests/store/a_wrong_time_can_be_corrected.rs +++ b/crates/utopia-store/tests/store/a_wrong_time_can_be_corrected.rs @@ -257,6 +257,7 @@ async fn an_end_that_was_never_there_can_be_taken_back() -> anyhow::Result<()> { to_precision: Some("year"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await? diff --git a/crates/utopia-store/tests/store/an_admin_grants_system_admin.rs b/crates/utopia-store/tests/store/an_admin_grants_system_admin.rs index c1b733648..553bfe1c4 100644 --- a/crates/utopia-store/tests/store/an_admin_grants_system_admin.rs +++ b/crates/utopia-store/tests/store/an_admin_grants_system_admin.rs @@ -44,6 +44,13 @@ async fn granting_a_member_makes_them_admin() -> anyhow::Result<()> { let run = async { utopia_store::accounts::set_system_admin(&pool, member, admin, true).await?; assert!(is_admin(&pool, member).await?); + utopia_store::accounts::set_system_admin(&pool, member, admin, false).await?; + assert!(!is_admin(&pool, member).await?); + // A request admitted before the actor's revocation still cannot remove + // the last administrator. Repeating a non-admin's revocation is harmless. + let refused = utopia_store::accounts::set_system_admin(&pool, admin, member, false).await; + assert!(matches!(refused, Err(utopia_core::AppError::Validation(_)))); + utopia_store::accounts::set_system_admin(&pool, member, admin, false).await?; Ok::<_, anyhow::Error>(()) } .await; @@ -90,9 +97,11 @@ async fn granting_a_deactivated_or_unknown_user_is_not_found() -> anyhow::Result let run = async { utopia_store::accounts::deactivate_user(&pool, member, admin).await?; for target in [member, Uuid::now_v7()] { - let missing = - utopia_store::accounts::set_system_admin(&pool, target, admin, true).await; - assert!(matches!(missing, Err(utopia_core::AppError::NotFound))); + for value in [true, false] { + let missing = + utopia_store::accounts::set_system_admin(&pool, target, admin, value).await; + assert!(matches!(missing, Err(utopia_core::AppError::NotFound))); + } } assert!(!is_admin(&pool, member).await?); Ok::<_, anyhow::Error>(()) diff --git a/crates/utopia-store/tests/store/an_end_date_closes_the_open_span.rs b/crates/utopia-store/tests/store/an_end_date_closes_the_open_span.rs index 0d2a2cd2a..da6c351ea 100644 --- a/crates/utopia-store/tests/store/an_end_date_closes_the_open_span.rs +++ b/crates/utopia-store/tests/store/an_end_date_closes_the_open_span.rs @@ -26,6 +26,7 @@ fn span( to_precision: to.map(|_| "day"), attested_at: Some(day(attested)), from_grade: None, + ..Default::default() } } @@ -54,6 +55,7 @@ fn ends(to: &str, attested: &str) -> utopia_store::graph::Validity<'static> { to_precision: Some("day"), attested_at: Some(day(attested)), from_grade: None, + ..Default::default() } } @@ -147,6 +149,7 @@ async fn an_end_date_closes_the_open_span() -> anyhow::Result<()> { to_precision: None, attested_at: Some(day("2025-08-01")), from_grade: None, + ..Default::default() }, 0.9, ) diff --git a/crates/utopia-store/tests/store/an_event_holds_at_the_moment_it_names.rs b/crates/utopia-store/tests/store/an_event_holds_at_the_moment_it_names.rs index 53ac19c73..784d50a3a 100644 --- a/crates/utopia-store/tests/store/an_event_holds_at_the_moment_it_names.rs +++ b/crates/utopia-store/tests/store/an_event_holds_at_the_moment_it_names.rs @@ -312,6 +312,7 @@ async fn a_span_an_ending_or_an_unknown_end_given_to_an_event_collapses() -> any to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await?; @@ -338,6 +339,7 @@ async fn a_span_an_ending_or_an_unknown_end_given_to_an_event_collapses() -> any to_precision: Some("month"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await?; @@ -581,6 +583,7 @@ async fn a_person_correcting_an_event_leaves_it_one_moment() -> anyhow::Result<( to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await? diff --git a/crates/utopia-store/tests/store/an_sso_login_finds_or_opens_its_account.rs b/crates/utopia-store/tests/store/an_sso_login_finds_or_opens_its_account.rs index 7aa956c40..a8ea51095 100644 --- a/crates/utopia-store/tests/store/an_sso_login_finds_or_opens_its_account.rs +++ b/crates/utopia-store/tests/store/an_sso_login_finds_or_opens_its_account.rs @@ -5,6 +5,55 @@ use uuid::Uuid; const ISSUER: &str = "https://id.sso.test"; +#[tokio::test] +async fn ambiguous_active_emails_leave_accounts_and_identities_unchanged() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let org = Uuid::now_v7(); + sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'sso-ambiguity-test')") + .bind(org) + .execute(&pool) + .await?; + let run = async { + let email = format!("{}-ann@sso.test", Uuid::now_v7().simple()); + let first = Uuid::now_v7(); + for (id, address) in [(first, email.to_uppercase()), (Uuid::now_v7(), email.clone())] { + sqlx::query("INSERT INTO users (id, org_id, email, display_name, password_hash) VALUES ($1, $2, $3, 'Ann', 'x')") + .bind(id).bind(org).bind(address).execute(&pool).await?; + } + let snapshot = || async { + sqlx::query_scalar::<_, serde_json::Value>( + "SELECT jsonb_build_object( + 'users', (SELECT jsonb_agg(to_jsonb(u) ORDER BY id) FROM users u WHERE org_id = $1), + 'identities', (SELECT jsonb_agg(to_jsonb(i) ORDER BY subject) FROM oidc_identities i JOIN users u ON u.id = i.user_id WHERE u.org_id = $1))", + ).bind(org).fetch_one(&pool).await + }; + for already_linked in [false, true] { + if already_linked { + sqlx::query("INSERT INTO oidc_identities (issuer, subject, user_id) VALUES ($1, $2, $3)") + .bind(ISSUER).bind(format!("existing-{first}")).bind(first).execute(&pool).await?; + } + let before = snapshot().await?; + // 两种精确拼写都必须拒绝；已有绑定也不能让另一个候选变得可选。 + for address in [&email, &email.to_uppercase()] { + let result = utopia_store::accounts::account_for_sso( + &pool, ISSUER, &format!("new-{first}"), address, "Changed", + ).await; + assert!(matches!(result, Err(utopia_core::AppError::CodedConflict { code: "unlinked", ref message }) if message == "Multiple active accounts match the SSO email")); + assert_eq!(snapshot().await?, before); + } + } + anyhow::Ok(()) + }.await; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + run +} + /// 一个比库里其它组织都早的组织，开户会落进它的默认工作区 /// 用「最早的组织」的测试一个一个跑，组织的时间也比库里现有的都早一天： /// 并行时谁更早说不准，中断的旧测试留下的组织也会抢走「最早」 diff --git a/crates/utopia-store/tests/store/an_unknown_date_is_not_an_open_one.rs b/crates/utopia-store/tests/store/an_unknown_date_is_not_an_open_one.rs index 302a93db1..010a99b3d 100644 --- a/crates/utopia-store/tests/store/an_unknown_date_is_not_an_open_one.rs +++ b/crates/utopia-store/tests/store/an_unknown_date_is_not_an_open_one.rs @@ -392,6 +392,7 @@ async fn the_anchor_moves_earlier_and_is_inherited() -> anyhow::Result<()> { to_precision: Some("day"), attested_at: None, from_grade: None, + ..Default::default() }, ) .await? diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rs index 175efdfc9..f495d9721 100644 --- a/crates/utopia-store/tests/store/main.rs +++ b/crates/utopia-store/tests/store/main.rs @@ -55,6 +55,7 @@ mod a_judged_entity_waits_its_turn; mod a_kind_word_binds_to_a_class; mod a_late_value_takes_its_place_in_history; mod a_mapping_is_not_a_fact; +mod a_merge_keeps_distinct_statements; mod a_merge_rewinds_with_the_second_clock; mod a_merged_target_stays_out_of_the_export; mod a_moment_marks_its_state; @@ -88,6 +89,7 @@ mod a_secret_is_sealed_at_rest; mod a_signature_holds_on_every_path; mod a_signature_vector_lasts_while_its_text_does; mod a_similar_name_is_proposed_not_merged; +mod a_single_name_candidate_is_not_counted; mod a_source_kind_is_listed_once; mod a_source_reaches_only_where_it_was_granted; mod a_statement_is_attested_by_what_its_document_says; diff --git a/crates/utopia-store/tests/the_engine_redraws_only_what_it_drew.rs b/crates/utopia-store/tests/the_engine_redraws_only_what_it_drew.rs index 8108ce5f3..ad013843c 100644 --- a/crates/utopia-store/tests/the_engine_redraws_only_what_it_drew.rs +++ b/crates/utopia-store/tests/the_engine_redraws_only_what_it_drew.rs @@ -152,6 +152,7 @@ async fn observe(pool: &PgPool, f: &Fixture, x: Seen<'_>) -> anyhow::Result A CI job pushes a test as a statement: `test-parses-a-date covers parse.rs`, qualifier `lines: "40-52"`. A week later lines are added above the test and the job pushes the same `external_id` with `lines: "58-70"`. The push is accepted, a new version is recorded, the chunk is replaced. The statement still says `40-52`, and will say it forever. The same goes for a `status` that went from `failing` to `passing`, or a `mood` that a later observation drops. + +## The problem + +`add_statement_qualifier` writes `ON CONFLICT (fact_id, role) DO NOTHING`. That was 0037's rule for typed qualifiers, carried over to the open layer by migration 0061: two observations that disagree should become two rows and a conflict, and until that cut exists, the first value stays. For a document that is mentioned once, the rule is invisible. For a source that pushes the same observation again with a corrected value (0054), it is wrong every time. + +Three facts of the code make it so: + +- **The statement survives the re-push.** `insert_open_statement` dedups open statements on `(kb, subject, phrase, object | value)` among live rows. A re-push with the same subject, phrase and object finds the existing statement and only adds evidence on the new chunk. The new role word arrives on a fact that already has one, and is dropped. +- **The old chunk is superseded, not deleted.** 0054 decision 7: the earlier chunk is marked `superseded_at`, and a statement whose evidence is only on superseded chunks is stale, handed to review, not invalidated. The statement's identity is right to survive; its role words are not right to stay frozen. +- **Typed rows copy once.** `materialize` copies a statement's evidence and qualifiers onto a typed row only when it creates the row. A typed row made from the first push keeps the first push's role words, and its only evidence is the superseded chunk, so every read marks it stale although the statement it came from was just confirmed. The three `q.role = 'mood'` checks look at `statement_qualifiers` as a whole, so a `mood` written once can never be taken back. + +## Decisions + +**1. One qualifier row per evidence chunk.** + +`statement_qualifiers` gains `chunk_id`, a reference to `chunks`, `NOT NULL`, and its key becomes `(fact_id, role, chunk_id)`. Each passage that states a role word records it against itself, the way `fact_evidence` already records one row per chunk. Writes stay append-only and idempotent: the same push twice writes nothing new, a new push writes rows for its new chunk and touches no old row. Every open statement has evidence, and a typed or implied row gets its statements' evidence with its role words (decision 4), so there is no row without a chunk to allow for. + +The backfill copies each existing `(fact, role)` value onto every evidence chunk of its fact, superseded ones included. Attaching it only to the earliest chunk would hide role words that show today: for a statement already re-pushed under 0054 decision 7, the earliest chunk is superseded, the live chunk carries no row because `DO NOTHING` dropped it, and under decision 2 the role would disappear. With the value on every chunk, every read after the migration returns exactly what it returned before; only the pushes that follow can change a value. + +`chunk_id` is a new provenance edge, and 0048's invariant covers it: the owner-derived trigger `statement_qualifiers_same_kb` checks that the chunk's document is in the fact's base, and the export preflight scans the edge. + +**2. A superseded chunk stops speaking for the statement.** + +The rows that speak for `(fact, role)` are the rows whose chunk is live at the read's `as_of` (`chunk_live_at`, and `document_live_at` for the chunk's document). For a stale fact, one with no live evidence at all, they are the rows on its evidence chunks that were superseded last, so a fact waiting in review still shows what it last said. This is one SQL fragment beside the `record_axis` helpers, and every reader of role words uses it: the entity panel, the tools, the export and `materialize`. + +There is no ordering among live rows. Two live documents that give different values for one role both speak, and a read returns each value with the chunk it came from, the way it returns two evidence passages. This is 0037 decision 3's answer, that two observations that disagree stay two and neither hides the other, applied to role words on open statements. Flagging such a pair as contested is left open below. + +What follows: + +- A re-push that changes a value supersedes the old chunk, so only the new value speaks. One that drops a role leaves no live row for it, and the role disappears from the read. The old rows stay in the table, as everything in the ledger does. +- Deleting a document supersedes its chunks; restoring it brings them back. The rule reads chunk and document liveness, so neither needs a recomputation. + +**3. A read in the past shows what was said then, on the chunk's clock.** + +A role word has no record time of its own. It arrives as a row on its evidence chunk, and its record time is that chunk's: it starts speaking when the chunk becomes live and stops when the chunk is superseded. An `as_of` read applies decision 2 at that instant, so a role word that changed shows the value its live evidence carried at `as_of`. In the example above, a read at a moment between the two pushes says `lines: "40-52"`; a read today says `58-70`. This is what `chunk_live_at` already does for evidence, where a passage replaced today is still live evidence on a graph read as of March, and a role word is part of that evidence. + +The copies `materialize` makes onto typed and implied rows (decision 4) keep the `chunk_id` of the row they copy, so they take that chunk's liveness too. A copy is written on the first run after its chunk arrives, and an `as_of` read between the chunk's arrival and that run shows it on the typed row a little before it was made. The read answers what the evidence said at that moment, not when the copy was written. + +**4. `materialize` keeps typed and implied rows in step.** + +On each run, scoped to the base it runs for, `materialize` copies onto every live typed row the evidence and qualifier rows of its source statements (`typed_fact_sources`) that the row does not have yet, and does the same for every live implied row from the statement that triggered it (`implied_fact_sources`). The copy is the same insert it does today at creation, `ON CONFLICT DO NOTHING`, now run for existing rows too. A typed or implied row then reads its role words through the same rule as its statements, and is stale exactly when they are. + +The three `mood` checks use the rule instead of any row: a statement has a mood when a row that speaks for `(fact, 'mood')` exists. A re-push that adds `mood` retires the typed row on the next run; one that removes it lets the row be created. + +**5. Typed qualifiers keep 0037's rule.** + +This record covers role words, the `statement_qualifiers` rows of the open layer and the copies `materialize` makes of them. The typed `fact_qualifiers` of 0037, one value per declared attribute per edge, keep that record's rule: a second mention with a different value does not overwrite the first, and is reported as `qualifier_conflict`. + +## Dead ends + +- **Last write wins on `(fact, role)`.** An `ON CONFLICT ... DO UPDATE` is one line. It overwrites an observation, which the ledger never does. It makes the value depend on the order of writes, so deleting the newer document leaves the newer value in place unless something recomputes it from what is left, and restoring needs the same. And an `as_of` read could only show today's value for any moment. +- **Qualifiers in the dedup key.** Each value would make its own statement: `covers` with `40-52` and `covers` with `58-70` as two facts. That splits one observation's identity on every change, and it contradicts 0037 decision 3, which keeps a qualifier out of identity so that a second mention adds to the same row. +- **Role words only on the chunk text.** The payload is stored verbatim as the chunk (0054 decision 2), so the value can be parsed back from it. Every reader would then parse JSON or prose to answer a question the table answers, and a document's chunk has no structure to parse. + +## Open questions + +- **Contested role words.** Two live documents that disagree on one role both speak (decision 2). Flagging such a pair the way the export flags a contested statement (0062) would make the disagreement easier to find; this record does not decide it. +- **The cost of the re-sync.** Copying evidence and role words onto every live typed and implied row on each `materialize` run is a few set-based inserts per base. It has not been measured on a large base. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 3d6c51ebc..f4ef89cb4 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -73,6 +73,7 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | 0010 | [An unnamed relation stays empty](0010-no-relation-is-no-relation.md) | Implemented | partly (by 0044) | | 0037 | [A relation carries its own attributes](0037-a-relation-carries-its-own-attributes.md) | In progress | partly (by 0044) | | 0048 | [Provenance references stay inside the knowledge base](0048-provenance-references-stay-inside-the-knowledge-base.md) | Implemented | | +| 0070 | [A role word speaks while its chunk is live](0070-a-role-word-speaks-while-its-chunk-is-live.md) | Proposed | | ### [governance](../design/governance.md) diff --git a/docs/design/ledger.md b/docs/design/ledger.md index b9b5b7141..34b908222 100644 --- a/docs/design/ledger.md +++ b/docs/design/ledger.md @@ -8,7 +8,7 @@ in [time](time.md), derived facts in [rules](rules.md). **A fact is a row that is never updated in place.** `facts (kb, subject_id, predicate_id NULL, object_id | object_value, confidence, valid_*, *_precision, attested_from, attested_to, recorded_at, -invalidated_at, supersedes, end_derived, layer, phrase)`. A correction inserts a new row that +invalidated_at, supersedes, end_derived, end_after_period, layer, phrase)`. A correction inserts a new row that `supersedes` the old one and invalidates it; a retraction invalidates; `entity_history` reads the chain back as "recorded as X, refined to Y, by whom" [0001, 0003]. Confirming a mapping by `UPDATE ... confidence = 1.0` was the one violation and is why mappings left the ledger [0011]. diff --git a/docs/design/time.md b/docs/design/time.md index 0369d15a6..f4f3724ac 100644 --- a/docs/design/time.md +++ b/docs/design/time.md @@ -67,7 +67,10 @@ graph with its date. An end names its bucket: an open row that starts inside it bucket's end ("left in 2024" closes a row from March 2024 at 2025-01-01), a row that starts before it closes at its start. Materialized state rows with equal ends written before this are retired and computed again when one of their source statements has those ends; a row a person set to a single -date is kept [0053 revision, #966]. The `[event]` / `[eternal]` prompt marks of 0031 d4 and the zoned-time sentence of 0024 +date is kept [0053 revision, #966]. An end moved to its bucket's end is marked `end_after_period`, +by the closing path and the refinement alike, and an open row closed by both "until 2025" and "left +in 2024" is marked in either order; a correction keeps the mark while the end stays as stored and a +start remains [0053 revision 2026-10-17, #994]. The `[event]` / `[eternal]` prompt marks of 0031 d4 and the zoned-time sentence of 0024 belonged to the typed contract and are gone with it; the write rule in the store is what remains. **Derived rows** store the interval as read: the intersection of premise read intervals, an anchored @@ -116,6 +119,8 @@ only when that date came from the content or a dating source, else null [#731]. bench no lower than 17 of 18 on both runs [0045]. - The `as_of` control on the graph page; the anchor shown beside a blank start; an event drawn as a point; retrieval that takes `at` [0019, 0022, 0031]. +- Every reader prints a marked end as the period it names: chat and MCP, the review lines, the + export, the web views and the time dialog (#994 cut 1) [0053 revision 2026-10-17]. ## Open questions diff --git a/migrations/0114_an_end_that_names_a_period_says_so.sql b/migrations/0114_an_end_that_names_a_period_says_so.sql new file mode 100644 index 000000000..ec5435b46 --- /dev/null +++ b/migrations/0114_an_end_that_names_a_period_says_so.sql @@ -0,0 +1,19 @@ +-- 终点说的是它之前那个时段时，行上记一笔（#994，0053 的修订）。 +-- +-- 「works for Meridian from 2024-03-01」再加「left in 2024」，#975 之后存成 2024-03-01 → +-- 2025-01-01（年精度）：世界轴上这是对的，2024 年全年在职，2025 年不在。可「until 2025」 +-- 关上同一行，存的也是 2025-01-01（年精度）。只看值分不出原文说的是 2024 还是 2025，读者 +-- 只能按存的时刻印，印出来是下一个时段。 +-- +-- `end_after_period`：终点是原文所说时段之后的那一刻。存的时刻不变，世界轴照旧读它，读者 +-- 据此印原文说的时段。写入路径在 #975 挪终点的两处设它（收尾、精化）；一条开着的行被 +-- 「until 2025」与「left in 2024」两句关上，哪一句先到都设；改写、搬主宾、勘误改出来的行 +-- 带着它；时间线画的终点、不知哪天的终点从不带。约束要求有起点：终点只在起点落在那个时段 +-- 里时才被挪到时段的尽头，没有起点的行第一刀不标。第二刀让时间解析自己写这种终点时，按 +-- 名字换掉这条约束，去掉起点那一条（Rust 一侧同一句话在 `graph::period_mark_fits`）。 +-- +-- 约束先删再加，整份文件可以重放。 +ALTER TABLE facts ADD COLUMN IF NOT EXISTS end_after_period BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE facts DROP CONSTRAINT IF EXISTS facts_end_after_period_needs_a_stated_end; +ALTER TABLE facts ADD CONSTRAINT facts_end_after_period_needs_a_stated_end + CHECK (NOT end_after_period OR (valid_from IS NOT NULL AND valid_to IS NOT NULL AND NOT end_derived)); diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 2eb3e0d88..a7109fcb7 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -140,6 +140,8 @@ export const en = { inference_off: "Materialized inference is off for this knowledge base. Turn it on in Settings.", bad_resolution: "That is not a valid decision.", + interval_inverted: + "The end is before the start. Give an end after the start.", empty_state_span: "A state that ends where it starts holds at no moment. Give an end after the start, or leave it open.", unknown_marks: "A single date marks the start, the end, or neither.", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index ddda99a15..9222d4edc 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -127,6 +127,7 @@ export const zh: Strings = { concurrency_range: "填一个 1 到 256 之间的数。", inference_off: "这个库没有打开物化推理。到设置里开。", bad_resolution: "这不是一个有效的裁决。", + interval_inverted: "终点早于起点。请给一个晚于起点的终点。", empty_state_span: "起止相同的状态任何时刻都不成立。给一个晚于起点的终点，或者让它开着。", unknown_marks: "单个日期标的是开始、结束，或者都不是。", marks_needs_state: "只有状态属性才说单个日期标的是什么，事件与恒常属性不说。", diff --git a/web/src/pages/Chat.tsx b/web/src/pages/Chat.tsx index 40ffcf73c..5abe02f1e 100644 --- a/web/src/pages/Chat.tsx +++ b/web/src/pages/Chat.tsx @@ -74,8 +74,8 @@ import { import { NodCard } from "./PendingFacts"; import { stepDetail, stepLabel } from "../steps"; import { NextStep, nextStep, useReadiness } from "./NextStep"; -import { DRAFT_KEY, fetchMe, lastKey } from "../authSession"; import { KbMenuList } from "./KbMenuList"; +import { DRAFT_KEY, fetchMe, lastKey } from "../authSession"; /* `Turn` 定义在 liveAnswer 里：进行中的那一次也是一串 Turn， 而它必须活得比这个组件长（见那个文件顶上的说明） */ diff --git a/web/src/pages/Graph.tsx b/web/src/pages/Graph.tsx index a878a7853..fc4e4cb9d 100644 --- a/web/src/pages/Graph.tsx +++ b/web/src/pages/Graph.tsx @@ -397,7 +397,9 @@ export function Graph() { canUpload: kb?.my_role !== "viewer", }); const data = useQuery({ - queryKey: ["graph", kb?.id, focusEntity, nodeBudget], + queryKey: focusEntity + ? ["graph", kb?.id, "neighborhood", focusEntity] + : ["graph", kb?.id, "overview", nodeBudget], queryFn: () => focusEntity ? api.graphNeighborhood(kb!.id, focusEntity) diff --git a/web/src/pages/KbMenuList.tsx b/web/src/pages/KbMenuList.tsx index 0e5a3afab..12e017d6d 100644 --- a/web/src/pages/KbMenuList.tsx +++ b/web/src/pages/KbMenuList.tsx @@ -9,7 +9,7 @@ Esc / ← 关上并把焦点还给那一行（kbMenuKeys）。没用 DropdownMenuSub：它只能 长在 DropdownMenu 里，而两个宿主一个是带查找框的 Popover（菜单的 typeahead 与焦点管理会跟查找框抢键），一个是手写的 div。 */ -import { useRef, useState, useSyncExternalStore, type KeyboardEvent } from "react"; +import { useEffect, useRef, useState, useSyncExternalStore, type KeyboardEvent } from "react"; import { Check, ChevronRight } from "lucide-react"; import type { Kb } from "../api"; import { Row, cn } from "../ui"; @@ -139,6 +139,11 @@ function KbGroupRow({ if (open) submenu.current?.querySelector("button")?.focus(); else show(true); }; + useEffect(() => { + if (!narrow || !open || !focusInside.current) return; + submenu.current?.querySelector("button")?.focus(); + focusInside.current = false; + }, [narrow, open]); const onSubmenuKey = (e: KeyboardEvent) => { const action = submenuKey(e.key); if (action === undefined) return; @@ -169,25 +174,34 @@ function KbGroupRow({ if (narrow) return ( <> - setOpen((v) => !v)} - > - {name} - +
+ setOpen((v) => !v)} + onKeyDown={onRowKey} + > + {name} + +
{open && - items.map((item) => ( - onPick(item.kb.id)} - /> - ))} +
+ {items.map((item) => ( + { + setOpen(false); + onPick(item.kb.id); + }} + /> + ))} +
+ } ); return ( @@ -216,8 +230,11 @@ function KbGroupRow({ onOpenAutoFocus={(e) => { if (!focusInside.current) e.preventDefault(); }} - // Esc 交给 onSubmenuKey：它还要把焦点还给那一行 - onEscapeKeyDown={(e) => e.preventDefault()} + // 焦点在子菜单里时 Esc 交给 onSubmenuKey（它还要把焦点还给那一行）； + // 焦点还在搜索框里时照常关掉 + onEscapeKeyDown={(e) => { + if (submenu.current?.contains(document.activeElement)) e.preventDefault(); + }} // 按在组那一行上不算外面：否则按下就关、松开的 click 又开，闪一下 onInteractOutside={(e) => { if (anchor.current?.contains(e.target as Node)) e.preventDefault(); diff --git a/web/src/pages/KbSwitcher.tsx b/web/src/pages/KbSwitcher.tsx index 45ee1341d..9ce534f62 100644 --- a/web/src/pages/KbSwitcher.tsx +++ b/web/src/pages/KbSwitcher.tsx @@ -18,8 +18,8 @@ import { PopoverContent, PopoverTrigger, } from "@/components/ui/popover"; -import { fetchMe } from "../authSession"; import { KbMenuList } from "./KbMenuList"; +import { fetchMe } from "../authSession"; export function KbSwitcher({ kb, diff --git a/web/src/pages/Review.tsx b/web/src/pages/Review.tsx index 88e3a1eb7..ac6c213a6 100644 --- a/web/src/pages/Review.tsx +++ b/web/src/pages/Review.tsx @@ -1551,6 +1551,7 @@ export function Review() { resolution: ViolationResolution; opts?: DecideOpts; }) => api.decideViolation(kb!.id, id, resolution, opts), + onError: (e) => toast.error(e instanceof Error ? e.message : String(e)), onSettled: invalidate, }); // 检查是同步的纯计算,所以直接 mutate 不排队。跑完把报告留在按钮旁边—— @@ -1580,6 +1581,7 @@ export function Review() { close_at: closeAt, close_at_precision: closeAtPrecision, }), + onError: (e) => toast.error(e instanceof Error ? e.message : String(e)), onSettled: invalidate, }); @@ -1593,6 +1595,7 @@ export function Review() { validTo: string; precision: string; }) => api.closeFact(kb!.id, id, validTo, precision), + onError: (e) => toast.error(e instanceof Error ? e.message : String(e)), onSettled: invalidate, }); diff --git a/web/src/useEventReconnect.test.ts b/web/src/useEventReconnect.test.ts index 2e37fe5c4..c54d22f15 100644 --- a/web/src/useEventReconnect.test.ts +++ b/web/src/useEventReconnect.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { QueryClient, QueryObserver, type QueryKey } from "@tanstack/react-query"; import { DEFAULT_STALE_MS, STREAM_KEYS, STREAM_STALE_MS, applyQueryDefaults } from "./queryDefaults"; -import { SETTLE_MS, useKbEvents } from "./useKbEvents"; +import { OVERVIEW_SETTLE_MS, SETTLE_MS, useKbEvents } from "./useKbEvents"; import { useAlertEvents } from "./useAlertEvents"; import { HIDDEN_RELEASE_MS } from "./eventStream"; @@ -178,6 +178,7 @@ describe("KB stream reconnect", () => { it("flushes a pending burst together with reconnect instead of scheduling another refetch", async () => { const client = createClient(); const graph = await watch(client, ["graph", "kb-a"]); + const overview = await watch(client, ["graph", "kb-a", "overview", 200]); const documents = await watch(client, ["documents", "kb-a"]); const badge = await watch(client, ["alerts", "unread"]); const list = await watch(client, ["alerts", "list", "", 0]); @@ -190,11 +191,13 @@ describe("KB stream reconnect", () => { source.emit("error"); source.emit("open"); expect(graph.fetch).toHaveBeenCalledTimes(2); + expect(overview.fetch).toHaveBeenCalledTimes(2); expect(documents.fetch).toHaveBeenCalledTimes(2); expect(badge.fetch).toHaveBeenCalledTimes(2); expect(list.fetch).toHaveBeenCalledTimes(2); - await vi.advanceTimersByTimeAsync(SETTLE_MS); + await vi.advanceTimersByTimeAsync(OVERVIEW_SETTLE_MS); expect(graph.fetch).toHaveBeenCalledTimes(2); + expect(overview.fetch).toHaveBeenCalledTimes(2); expect(documents.fetch).toHaveBeenCalledTimes(2); expect(badge.fetch).toHaveBeenCalledTimes(2); expect(list.fetch).toHaveBeenCalledTimes(2); @@ -203,7 +206,7 @@ describe("KB stream reconnect", () => { it("keeps normal business events coalesced", async () => { const client = createClient(); - const graph = await watch(client, ["graph", "kb-a"]); + const graph = await watch(client, ["graph", "kb-a", "neighborhood", "e1"]); const documents = await watch(client, ["documents", "kb-a"]); const { source } = mount(() => useKbEvents("kb-a")); source.emit("open"); @@ -219,6 +222,26 @@ describe("KB stream reconnect", () => { expect(documents.fetch).toHaveBeenCalledTimes(2); }); + it("a burst of graph events refetches the overview once after OVERVIEW_SETTLE_MS and the neighbourhood after SETTLE_MS", async () => { + const client = createClient(); + const overview = await watch(client, ["graph", "kb-a", "overview", 200]); + const neighborhood = await watch(client, ["graph", "kb-a", "neighborhood", "e1"]); + const { source } = mount(() => useKbEvents("kb-a")); + source.emit("open"); + source.emit("graph"); + await vi.advanceTimersByTimeAsync(SETTLE_MS); + expect(neighborhood.fetch).toHaveBeenCalledTimes(2); + expect(overview.fetch).toHaveBeenCalledTimes(1); + for (let elapsed = SETTLE_MS; elapsed < 1_000; elapsed += 100) { + source.emit("graph"); + await vi.advanceTimersByTimeAsync(100); + } + await vi.advanceTimersByTimeAsync(OVERVIEW_SETTLE_MS - 1_000 - 1); + expect(overview.fetch).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1); + expect(overview.fetch).toHaveBeenCalledTimes(2); + }); + it("ignores late callbacks from the old connection after switching KB", async () => { const client = createClient(); const oldKb = await watch(client, ["graph", "kb-a"]); diff --git a/web/src/useKbEvents.ts b/web/src/useKbEvents.ts index 8ca6a47f3..ea7c39fe9 100644 --- a/web/src/useKbEvents.ts +++ b/web/src/useKbEvents.ts @@ -18,6 +18,9 @@ import { STREAM_KEYS } from "./queryDefaults"; /** 一阵事件之间的静默期。抽取落事实的间隔远小于它，人眼看不出这点延迟 */ export const SETTLE_MS = 300; +/** 全图 overview 要数遍整个库：推送期间几秒重数一次就够了（#1296） */ +export const OVERVIEW_SETTLE_MS = 3_000; + /** * 把一阵失效合并成一次：`push` 只记键，静默 `settleMs` 后一起交给 `invalidate`， * 同一个键及其前缀只失效一次；`flush` 给卸载与重连用，攒着的刷掉而不是丢掉。 @@ -37,7 +40,7 @@ export function createInvalidationCoalescer( const keys = [...pending.values()]; pending.clear(); for (const key of keys) { - // 正常 graph 事件可能已排了 ["graph"]，重连又补上 ["graph", kbId]： + // 正常 graph 事件可能已排了 ["graph", kbId, "neighborhood"]，重连又补上 ["graph", kbId]： // 前缀已经覆盖后者，不能取消刚开始的请求再重取一遍。 if (keys.some((prefix) => prefix.length < key.length && partialMatchKey(key, prefix))) continue; invalidate(key); @@ -59,12 +62,31 @@ export function useKbEvents(kbId: string | undefined) { (key) => queryClient.invalidateQueries({ queryKey: key }), SETTLE_MS, ); + // overview 到点也经快的那层刷：重连补上的 ["graph", kbId] 已覆盖它，不再多取一遍 + const overview = createInvalidationCoalescer((key) => { + push(key); + flush(); + }, OVERVIEW_SETTLE_MS); + const pushGraph = () => { + push(["graph", kbId, "neighborhood"]); + overview.push(["graph", kbId, "overview"]); + }; + const flushAll = () => { + overview.flush(); + flush(); + }; const unsubscribe = subscribeEvents( `/api/v1/kbs/${kbId}/events`, { - document: () => push(["documents", kbId], ["graph"]), - graph: () => push(["graph"], ["ontologyRegressions", kbId]), + document: () => { + push(["documents", kbId]); + pushGraph(); + }, + graph: () => { + push(["ontologyRegressions", kbId]); + pushGraph(); + }, // 映射探索跑完发的也是 review：Pending 那一栏得跟着刷新 review: () => push(["review", kbId], ["mappings", kbId], ["ontologyRegressions", kbId]), // 一句记忆抽出了等人点头的事实（0015）：对话里那张确认卡跟着长出来 @@ -77,13 +99,13 @@ export function useKbEvents(kbId: string | undefined) { // 事件没有回放，staleTime 到期也不会自己发请求。断线后恢复必须补刷； // 首次正常连接交给页面已有的读取，首次连接失败再恢复则同样可能漏事件。 push(...STREAM_KEYS.map((head) => [head, kbId]), ["alerts"]); - flush(); + flushAll(); }, ); return () => { unsubscribe(); // 卸载时把攒着的刷掉而不是丢掉：换页回来看到的必须是新数据 - flush(); + flushAll(); }; }, [kbId, queryClient]); }