From c8aedf9206b8909c646be299510c54bc5ccf2532 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:56:50 -0400 Subject: [PATCH 01/27] =?UTF-8?q?The=20activity=20log=20filter=20covers=20?= =?UTF-8?q?the=20days=20it=20names,=20so=20=E6=88=AA=E6=AD=A2=E6=97=A5?= =?UTF-8?q?=E6=9C=9F=202026-10-24=20keeps=20that=20day's=20events=20(#1420?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rudy Celekli Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- web/src/pages/KbSettings.tsx | 16 ++++++++++++++-- web/src/pages/auditBounds.test.ts | 17 +++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 web/src/pages/auditBounds.test.ts diff --git a/web/src/pages/KbSettings.tsx b/web/src/pages/KbSettings.tsx index b8478e20c..916652d45 100644 --- a/web/src/pages/KbSettings.tsx +++ b/web/src/pages/KbSettings.tsx @@ -509,6 +509,19 @@ function auditDetailName(e: AuditEvent): string { const AUDIT_PAGE = 50; +/** 两个日期框说的是读者日历上的几天：起始那天零点起、截止那天过完为止。服务端的区间 + * 含起点不含终点、按时刻比，所以截止日要送下一天的零点，零点也按本地时区换成时刻 */ +export function auditBounds(since: string, until: string) { + const midnight = (day: string, later: number) => { + const [y, m, d] = day.split("-").map(Number); + return new Date(y, m - 1, d + later).toISOString(); + }; + return { + since: since ? midnight(since, 0) : undefined, + until: until ? midnight(until, 1) : undefined, + }; +} + function KbActivity({ kbId }: { kbId: string }) { // 筛选按真实查法来：查一类动作、查一个人、查一段时间。 // 动作前缀匹配——`entity.` 就能把 retyped / renamed 一族一起捞出来 @@ -521,8 +534,7 @@ function KbActivity({ kbId }: { kbId: string }) { queryFn: () => api.kbAudit(kbId, { action: action || undefined, - since: since || undefined, - until: until || undefined, + ...auditBounds(since, until), limit: AUDIT_PAGE, offset: page * AUDIT_PAGE, }), diff --git a/web/src/pages/auditBounds.test.ts b/web/src/pages/auditBounds.test.ts new file mode 100644 index 000000000..8d74daa55 --- /dev/null +++ b/web/src/pages/auditBounds.test.ts @@ -0,0 +1,17 @@ +import { beforeAll, expect, it } from "vitest"; +import { auditBounds } from "./KbSettings"; + +// 上海的 10 月 24 日是 UTC 的 23 日 16:00 到 24 日 16:00 +beforeAll(() => { + process.env.TZ = "Asia/Shanghai"; +}); + +it("filters the activity log by the days the reader picked, end day included", () => { + expect(auditBounds("2026-10-24", "2026-10-24")).toEqual({ + since: "2026-10-23T16:00:00.000Z", + until: "2026-10-24T16:00:00.000Z", + }); + // 截止日是月末：下一天是下个月的 1 日 + expect(auditBounds("", "2026-10-31").until).toBe("2026-10-31T16:00:00.000Z"); + expect(auditBounds("", "")).toEqual({ since: undefined, until: undefined }); +}); From 6431a5ad76cb7fcac4cbc549fab1e89dda31587a Mon Sep 17 00:00:00 2001 From: Daniel <118035379+Floating-Y@users.noreply.github.com> Date: Mon, 26 Oct 2026 12:54:41 -0700 Subject: [PATCH 02/27] Prevent GitHub and Jira sync checkpoints from skipping unread tickets (#1359) * Prevent GitHub and Jira sync checkpoints from skipping unread tickets Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> * Read complete GitHub discussions and bound rate-limit waits Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> * Exercise sync budgets with realistic test fixtures and reuse regression setup Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> * Keep equal-time Jira pages stable and align checkpoint documentation Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> --------- Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> --- crates/utopia-cli/src/main.rs | 2 +- crates/utopia-server/src/github_issues.rs | 562 +++++++++++++++++- crates/utopia-server/src/github_sync_tests.rs | 5 +- crates/utopia-server/src/ingest_sources.rs | 105 ++-- crates/utopia-server/src/jira_issues.rs | 514 ++++++++++++++-- .../src/timestamp_checkpoint_tests.rs | 495 +++++++++++++++ crates/utopia-store/src/sources.rs | 70 +++ ...checkpoint-ends-at-a-timestamp-boundary.md | 59 +- docs/decisions/README.md | 2 +- docs/design/sources.md | 17 + ...heckpoint_ends_at_a_timestamp_boundary.sql | 4 + 11 files changed, 1733 insertions(+), 102 deletions(-) create mode 100644 crates/utopia-server/src/timestamp_checkpoint_tests.rs create mode 100644 migrations/0119_a_source_checkpoint_ends_at_a_timestamp_boundary.sql diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index f3683aabd..1d93a525b 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -82,7 +82,7 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**，不是最大的编号（守卫 `schema_version_policy_compares_against_current` // 按个数比）：编号有空缺时两者不同——0071 由一个开放 PR 占着，0072 先落，个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 106; +const CURRENT_SCHEMA_VERSION: u32 = 107; fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); diff --git a/crates/utopia-server/src/github_issues.rs b/crates/utopia-server/src/github_issues.rs index 2dee2a118..0d9aa0118 100644 --- a/crates/utopia-server/src/github_issues.rs +++ b/crates/utopia-server/src/github_issues.rs @@ -39,13 +39,16 @@ use chrono::{DateTime, Utc}; use serde::Deserialize; +use std::collections::BTreeMap; +use std::time::Duration; use crate::ticket::stamp; -/// 一次同步最多取多少页（每页 100）。GitHub 的分页没有天然终点， -/// 一个活跃仓库能翻很久；这里沿用单端点的分页预算。 -const MAX_PAGES: u32 = 10; +/// 列表预算可在完整时间组边界延长；单张工单的详情不受此预算限制。 +const MAX_LISTING_PAGES: u32 = 10; const PER_PAGE: u32 = 100; +const MAX_RATE_LIMIT_RETRIES: u32 = 3; +const MAX_RATE_LIMIT_WAIT: Duration = Duration::from_secs(30); #[derive(Debug, Deserialize)] pub struct Issue { @@ -190,16 +193,39 @@ pub(crate) async fn get_checked( auth: Option<&str>, accept: Option<&str>, ) -> anyhow::Result { - let mut req = http.get(url); - if let Some(a) = auth { - req = req.header(reqwest::header::AUTHORIZATION, a); - } - if let Some(a) = accept { - req = req.header(reqwest::header::ACCEPT, a); - } - let resp = req.send().await?; - let status = resp.status(); - if !status.is_success() { + let mut retries = 0; + loop { + let mut req = http.get(url.clone()); + if let Some(a) = auth { + req = req.header(reqwest::header::AUTHORIZATION, a); + } + if let Some(a) = accept { + req = req.header(reqwest::header::ACCEPT, a); + } + let resp = req.send().await?; + let status = resp.status(); + if status.is_success() { + return Ok(resp); + } + if matches!( + status, + reqwest::StatusCode::FORBIDDEN | reqwest::StatusCode::TOO_MANY_REQUESTS + ) && retries < MAX_RATE_LIMIT_RETRIES + { + if let Some(delay) = rate_limit_delay(resp.headers(), Utc::now()) { + // 等待留在当前请求内，否则超大时间组会在每次重试时重新耗尽前缀的配额。 + let delay = delay.max(Duration::from_secs(1 << retries)); + retries += 1; + tracing::info!( + retries, + wait_seconds = delay.as_secs(), + "GitHub rate limited; waiting before retry" + ); + drop(resp); + tokio::time::sleep(delay).await; + continue; + } + } // 限流说清楚是限流：未认证时每小时 60 次，一个中等仓库一次同步就能吃光。 // 报成通用 HTTP 错会让人去查网络，而正确的动作是配一个令牌 let remaining = resp @@ -214,10 +240,163 @@ pub(crate) async fn get_checked( } anyhow::bail!("HTTP {status} from GitHub"); } - Ok(resp) } -/// 分页取一个端点，直到空页或触顶。 +fn rate_limit_delay(headers: &reqwest::header::HeaderMap, now: DateTime) -> Option { + let retry_after = headers + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| { + value + .trim() + .parse::() + .ok() + .map(Duration::from_secs) + .or_else(|| { + DateTime::parse_from_rfc2822(value) + .ok() + .map(|at| at.signed_duration_since(now).to_std().unwrap_or_default()) + }) + }); + let delay = retry_after.or_else(|| { + if headers.get("x-ratelimit-remaining")?.to_str().ok()? != "0" { + return None; + } + let reset = headers + .get("x-ratelimit-reset")? + .to_str() + .ok()? + .parse::() + .ok()?; + let reset = DateTime::from_timestamp(reset, 0)?; + Some( + reset + .signed_duration_since(now) + .to_std() + .unwrap_or_default(), + ) + })?; + // 长等待不能占住 worker；拒绝本次重试，而不是早于服务端允许的时间重发。 + (delay <= MAX_RATE_LIMIT_WAIT).then_some(delay) +} + +pub struct IssueWindow { + pub issues: Vec, + pub checkpoint: Option>, +} + +/// offset 分页不是快照；复读同一前缀，变化时让调用者保留旧 checkpoint 后重试。 +pub async fn fetch_window( + http: &reqwest::Client, + base: &str, + since: Option>, + item_limit: usize, + include_prs: bool, + auth: Option<&str>, +) -> anyhow::Result { + let (window, positions) = + fetch_window_pass(http, base, since, item_limit, include_prs, auth, None).await?; + if let Some(checkpoint) = window + .checkpoint + .filter(|checkpoint| Some(*checkpoint) != since) + { + let (verified, verified_positions) = fetch_window_pass( + http, + base, + since, + item_limit, + include_prs, + auth, + Some(checkpoint), + ) + .await?; + anyhow::ensure!( + verified.checkpoint == Some(checkpoint) && positions == verified_positions, + "GitHub issues changed during pagination; retry the previous checkpoint" + ); + } + Ok(window) +} + +async fn fetch_window_pass( + http: &reqwest::Client, + base: &str, + since: Option>, + item_limit: usize, + include_prs: bool, + auth: Option<&str>, + through: Option>, +) -> anyhow::Result<(IssueWindow, BTreeMap>)> { + let mut query = vec![ + ("state", "all".to_owned()), + ("sort", "updated".to_owned()), + ("direction", "asc".to_owned()), + ]; + if let Some(since) = since { + query.push(("since", since.to_rfc3339())); + } + let mut window = IssueWindow { + issues: Vec::new(), + checkpoint: since, + }; + let mut positions = BTreeMap::new(); + let mut previous_time = None; + let mut stop_after = through; + let mut listing_pages = 0; + let mut page = 1; + loop { + let mut url = reqwest::Url::parse(base)?; + { + let mut parameters = url.query_pairs_mut(); + for (key, value) in &query { + parameters.append_pair(key, value); + } + parameters.append_pair("per_page", &PER_PAGE.to_string()); + parameters.append_pair("page", &page.to_string()); + } + let batch: Vec = get_checked(http, url, auth, None).await?.json().await?; + let count = batch.len(); + let mut page_counted = false; + for issue in batch { + let updated = issue.updated_at; + anyhow::ensure!( + previous_time.is_none_or(|previous| updated >= previous), + "GitHub issues are not ordered by update time; retry the previous checkpoint" + ); + previous_time = Some(updated); + if stop_after.is_some_and(|boundary| updated > boundary) { + return Ok((window, positions)); + } + // 已消费条目和排除的 PR 也能移动页边界，必须参加复读核对。 + anyhow::ensure!( + positions.insert(issue.number, updated).is_none(), + "GitHub issue repeated during pagination; retry the previous checkpoint" + ); + if since.is_some_and(|checkpoint| updated <= checkpoint) { + continue; + } + if !page_counted { + listing_pages += 1; + page_counted = true; + } + window.checkpoint = Some(updated); + if include_prs || issue.pull_request.is_none() { + window.issues.push(issue); + } + if through.is_none() + && (window.issues.len() >= item_limit || listing_pages >= MAX_LISTING_PAGES) + { + stop_after = Some(updated); + } + } + if count < PER_PAGE as usize { + return Ok((window, positions)); + } + page += 1; + } +} + +/// 详情必须读到末页，不能用截断的评论或事件覆盖已有完整正文。 pub async fn fetch_all Deserialize<'de>>( http: &reqwest::Client, base: &str, @@ -225,7 +404,8 @@ pub async fn fetch_all Deserialize<'de>>( auth: Option<&str>, ) -> anyhow::Result> { let mut out = Vec::new(); - for page in 1..=MAX_PAGES { + let mut page = 1; + loop { // 自己拼 query：这套 reqwest 特性组合里没有 RequestBuilder::query， // 而 sync_custom 本来也是这么拼的 let mut url = reqwest::Url::parse(base)?; @@ -244,6 +424,7 @@ pub async fn fetch_all Deserialize<'de>>( if n < PER_PAGE as usize { break; } + page += 1; } Ok(out) } @@ -251,6 +432,355 @@ pub async fn fetch_all Deserialize<'de>>( #[cfg(test)] mod tests { use super::*; + use serde_json::{json, Value}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate}; + + fn listing_row(number: i64, updated: &str) -> Value { + json!({ + "number": number, "title": format!("Issue {number}"), "state": "open", + "created_at": "2026-01-01T00:00:00Z", "updated_at": updated, + }) + } + + fn requested_page(request: &Request) -> usize { + request + .url + .query_pairs() + .find(|(key, _)| key == "page") + .unwrap() + .1 + .parse() + .unwrap() + } + + #[test] + fn rate_limit_headers_supply_a_bounded_wait_without_guessing() { + let now: DateTime = "2026-01-02T00:00:00Z".parse().unwrap(); + let mut headers = reqwest::header::HeaderMap::new(); + for (value, expected) in [ + ("2", Some(2)), + ("0", Some(0)), + ("30", Some(30)), + ("31", None), + ("-1", None), + ("broken", None), + ("86401", None), + ] { + headers.insert(reqwest::header::RETRY_AFTER, value.parse().unwrap()); + assert_eq!( + rate_limit_delay(&headers, now), + expected.map(Duration::from_secs) + ); + } + for (seconds, expected) in [(3, Some(3)), (31, None)] { + headers.insert( + reqwest::header::RETRY_AFTER, + (now + chrono::Duration::seconds(seconds)) + .to_rfc2822() + .parse() + .unwrap(), + ); + assert_eq!( + rate_limit_delay(&headers, now), + expected.map(Duration::from_secs) + ); + } + headers.remove(reqwest::header::RETRY_AFTER); + headers.insert("x-ratelimit-remaining", "0".parse().unwrap()); + headers.insert( + "x-ratelimit-reset", + (now.timestamp() + 30).to_string().parse().unwrap(), + ); + assert_eq!( + rate_limit_delay(&headers, now), + Some(Duration::from_secs(30)) + ); + headers.insert("x-ratelimit-remaining", "1".parse().unwrap()); + assert_eq!(rate_limit_delay(&headers, now), None); + headers.insert("x-ratelimit-remaining", "0".parse().unwrap()); + headers.insert( + "x-ratelimit-reset", + (now.timestamp() + 31).to_string().parse().unwrap(), + ); + assert_eq!(rate_limit_delay(&headers, now), None); + } + + #[tokio::test] + async fn a_rate_limited_request_waits_and_preserves_its_url_and_headers() { + let server = MockServer::start().await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + for status in [403, 429] { + server.reset().await; + let attempts = AtomicUsize::new(0); + Mock::given(method("GET")) + .respond_with(move |_: &Request| { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + ResponseTemplate::new(status).insert_header("Retry-After", "0") + } else { + ResponseTemplate::new(200) + } + }) + .mount(&server) + .await; + let url = reqwest::Url::parse(&format!("{}/resource?page=2", server.uri())).unwrap(); + let started = std::time::Instant::now(); + get_checked( + &http, + url.clone(), + Some("Bearer fixture"), + Some("application/vnd.github.raw+json"), + ) + .await + .unwrap(); + assert!(started.elapsed() >= Duration::from_secs(1)); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2); + for request in &requests { + // wiremock 为 origin-form URI 补 localhost；实际目标主机保留在 Host 头中。 + assert_eq!(request.url.path(), url.path()); + assert_eq!(request.url.query(), url.query()); + assert_eq!( + request.headers.get("host").unwrap().to_str().unwrap(), + server.address().to_string() + ); + assert_eq!( + request.headers.get("authorization").unwrap(), + "Bearer fixture" + ); + assert_eq!( + request.headers.get("accept").unwrap(), + "application/vnd.github.raw+json" + ); + } + } + } + + #[tokio::test] + async fn persistent_rate_limits_stop_after_a_bounded_number_of_requests() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(429).insert_header("Retry-After", "0")) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + assert!( + get_checked(&http, server.uri().parse().unwrap(), None, None) + .await + .is_err() + ); + assert_eq!(server.received_requests().await.unwrap().len(), 4); + } + + #[tokio::test] + async fn long_rate_limit_waits_fail_without_sleeping_or_retrying() { + let server = MockServer::start().await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + for status in [403, 429] { + let responses = [ + ResponseTemplate::new(status).insert_header("Retry-After", "31"), + ResponseTemplate::new(status).insert_header( + "Retry-After", + (Utc::now() + chrono::Duration::seconds(60)).to_rfc2822(), + ), + ResponseTemplate::new(status) + .insert_header("x-ratelimit-remaining", "0") + .insert_header( + "x-ratelimit-reset", + (Utc::now().timestamp() + 60).to_string(), + ), + ]; + for response in responses { + server.reset().await; + Mock::given(method("GET")) + .respond_with(response) + .mount(&server) + .await; + let result = tokio::time::timeout( + Duration::from_secs(5), + get_checked(&http, server.uri().parse().unwrap(), None, None), + ) + .await + .expect("a long rate-limit wait must fail immediately"); + assert!(result.is_err()); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + } + } + + #[tokio::test] + async fn an_empty_window_retains_the_previous_checkpoint_without_verification() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!([]))) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let since = Some("2026-01-02T00:00:00Z".parse().unwrap()); + let window = fetch_window(&http, &server.uri(), since, 200, false, None) + .await + .unwrap(); + assert!(window.issues.is_empty()); + assert_eq!(window.checkpoint, since); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn a_large_timestamp_group_finishes_and_the_next_run_passes_its_old_pages() { + let server = MockServer::start().await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let group_time = "2026-01-02T00:00:00Z"; + let later_time = "2026-01-03T00:00:00Z"; + // PR 多时先耗尽十页预算；全是工单时先耗尽 200 条预算。 + for issue_interval in [1, 100] { + server.reset().await; + let mut rows: Vec<_> = (1..=1001) + .map(|number| { + let mut row = listing_row(number, group_time); + if (number - 1) % issue_interval as i64 != 0 { + row["pull_request"] = json!({}); + } + row + }) + .collect(); + rows.push(listing_row(1002, later_time)); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let page = requested_page(request); + let rows: Vec<_> = rows + .iter() + .skip((page - 1) * 100) + .take(100) + .cloned() + .collect(); + ResponseTemplate::new(200).set_body_json(rows) + }) + .mount(&server) + .await; + let first = fetch_window(&http, &server.uri(), None, 200, false, None) + .await + .unwrap(); + let numbers: Vec<_> = first.issues.iter().map(|issue| issue.number).collect(); + assert_eq!( + numbers, + (1..=1001).step_by(issue_interval).collect::>() + ); + assert_eq!(first.checkpoint, Some(group_time.parse().unwrap())); + + // 模拟端点仍返回 inclusive 下界；十页旧记录不应耗尽新一轮的任一预算。 + let next = fetch_window(&http, &server.uri(), first.checkpoint, 200, false, None) + .await + .unwrap(); + assert_eq!(next.issues.len(), 1); + assert_eq!(next.issues[0].number, 1002); + assert_eq!(next.checkpoint, Some(later_time.parse().unwrap())); + let requests = server.received_requests().await.unwrap(); + assert!(requests.iter().all(|request| { + request + .url + .query_pairs() + .any(|(key, value)| key == "sort" && value == "updated") + && request + .url + .query_pairs() + .any(|(key, value)| key == "direction" && value == "asc") + })); + assert!(requests.iter().any(|request| requested_page(request) == 11)); + } + } + + #[tokio::test] + async fn a_moved_issue_rejects_the_window_and_is_read_after_pagination_stabilizes() { + let server = MockServer::start().await; + let moved = AtomicBool::new(false); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let page = requested_page(request); + if page == 2 { + moved.store(true, Ordering::SeqCst); + } + let mut rows: Vec<_> = (1..=201) + .map(|number| listing_row(number, "2026-01-02T00:00:00Z")) + .collect(); + if moved.load(Ordering::SeqCst) { + rows.remove(0); + rows.push(listing_row(1, "2026-01-03T00:00:00Z")); + } + let rows: Vec<_> = rows.into_iter().skip((page - 1) * 100).take(100).collect(); + ResponseTemplate::new(200).set_body_json(rows) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_window(&http, &server.uri(), None, 200, false, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("changed during pagination")); + + let recovered = fetch_window(&http, &server.uri(), None, 200, false, None) + .await + .unwrap(); + let recovered_numbers: Vec<_> = recovered.issues.iter().map(|issue| issue.number).collect(); + assert_eq!(recovered_numbers, (2..=201).collect::>()); + assert_eq!( + recovered.checkpoint, + Some("2026-01-02T00:00:00Z".parse().unwrap()) + ); + let next = fetch_window(&http, &server.uri(), recovered.checkpoint, 200, false, None) + .await + .unwrap(); + assert_eq!(next.issues.len(), 1); + assert_eq!(next.issues[0].number, 1); + assert_eq!( + next.checkpoint, + Some("2026-01-03T00:00:00Z".parse().unwrap()) + ); + } + + #[tokio::test] + async fn a_complete_discussion_is_read_beyond_ten_pages() { + let server = MockServer::start().await; + for count in [1000, 1001, 1100] { + server.reset().await; + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let page = requested_page(request); + let rows: Vec<_> = (0..count).skip((page - 1) * 100).take(100).collect(); + ResponseTemplate::new(200).set_body_json(rows) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_all::(&http, &server.uri(), &[], None) + .await + .unwrap(); + assert_eq!(result, (0..count).collect::>()); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), count as usize / 100 + 1); + } + } + + #[tokio::test] + async fn a_failed_discussion_page_does_not_return_a_partial_history() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(|request: &Request| { + let page = requested_page(request); + if page == 11 { + return ResponseTemplate::new(503); + } + let rows: Vec<_> = (0..1001).skip((page - 1) * 100).take(100).collect(); + ResponseTemplate::new(200).set_body_json(rows) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_all::(&http, &server.uri(), &[], None).await; + assert!(result.unwrap_err().to_string().contains("503")); + assert_eq!(server.received_requests().await.unwrap().len(), 11); + } fn issue(json: serde_json::Value) -> Issue { serde_json::from_value(json).unwrap() diff --git a/crates/utopia-server/src/github_sync_tests.rs b/crates/utopia-server/src/github_sync_tests.rs index 39b38db2b..025acb57e 100644 --- a/crates/utopia-server/src/github_sync_tests.rs +++ b/crates/utopia-server/src/github_sync_tests.rs @@ -260,14 +260,15 @@ async fn incremental_github_sync_preserves_the_complete_discussion() -> anyhow:: .iter() .filter(|request| request.url.path() == "/repos/test/repo/issues") .collect(); - anyhow::ensure!(issue_requests.len() == 6); + // 每轮先选窗口，再复读检查分页期间是否发生移动。 + anyhow::ensure!(issue_requests.len() == 12); for (index, request) in issue_requests.iter().enumerate() { let lower_bound = request .url .query_pairs() .find(|(key, _)| key == "since") .map(|(_, value)| time(&value)); - anyhow::ensure!(lower_bound == if index == 0 { None } else { since }); + anyhow::ensure!(lower_bound == if index < 2 { None } else { since }); } anyhow::ensure!( requests diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs index a45359802..cb012ae3f 100644 --- a/crates/utopia-server/src/ingest_sources.rs +++ b/crates/utopia-server/src/ingest_sources.rs @@ -16,6 +16,10 @@ use uuid::Uuid; #[path = "source_checkpoint_tests.rs"] mod source_checkpoint_tests; +#[cfg(test)] +#[path = "timestamp_checkpoint_tests.rs"] +mod timestamp_checkpoint_tests; + /// 单次同步的新文档上限（防超长 feed/URL 列表拖垮任务） const MAX_NEW_PER_SYNC: usize = 200; const MAX_FEED_BYTES: usize = 4 * 1024 * 1024; @@ -39,6 +43,7 @@ pub struct SyncStats { pub discovered: usize, pub queued_for_content: usize, pub content_terminal: usize, + pub checkpoint: Option>, } #[derive(Debug)] @@ -69,9 +74,12 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() let source = utopia_store::sources::get(&state.pool, source_id).await?; let kind = SourceKind::parse(&source.kind); let since = match kind { - Some(SourceKind::Custom | SourceKind::GithubIssues | SourceKind::JiraIssues) => { + Some(SourceKind::Custom) => { utopia_store::sources::last_successful_sync_start(&state.pool, source_id).await? } + Some(SourceKind::GithubIssues | SourceKind::JiraIssues) => { + utopia_store::sources::sync_checkpoint(&state.pool, source_id).await? + } _ => None, }; utopia_store::sources::mark_running(&state.pool, source_id).await?; @@ -104,19 +112,51 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() None => Err(anyhow::anyhow!("unknown source kind `{}`", source.kind)), }; - match outcome { - Ok(stats) => { - utopia_store::sources::finish_run( + let outcome = match outcome { + Ok(stats) + if matches!( + kind, + Some(SourceKind::GithubIssues | SourceKind::JiraIssues) + ) => + { + utopia_store::sources::finish_timestamp_sync( &state.pool, run_id, source_id, - None, + since, + stats.checkpoint, stats.created as i32, stats.updated as i32, ) - .await?; - utopia_store::sources::finish_sync(&state.pool, source_id, None, stats.total() as i32) + .await + .map(|()| stats) + .map_err(anyhow::Error::from) + } + other => other, + }; + match outcome { + Ok(stats) => { + if !matches!( + kind, + Some(SourceKind::GithubIssues | SourceKind::JiraIssues) + ) { + utopia_store::sources::finish_run( + &state.pool, + run_id, + source_id, + None, + stats.created as i32, + stats.updated as i32, + ) + .await?; + utopia_store::sources::finish_sync( + &state.pool, + source_id, + None, + stats.total() as i32, + ) .await?; + } state.emit_source(source.kb_id); tracing::info!( %source_id, @@ -834,20 +874,20 @@ async fn sync_github_issues_with_client( let base = format!("{api_base}/repos/{repo}"); - // 增量：GitHub 的 since 是"这之后更新过的" - let mut issue_q: Vec<(&str, String)> = vec![("state", "all".into())]; - if let Some(t) = since { - issue_q.push(("since", t.to_rfc3339())); - } - - let issues: Vec = - crate::github_issues::fetch_all(http, &format!("{base}/issues"), &issue_q, auth).await?; - let mut stats = SyncStats::default(); - for issue in issues - .iter() - .filter(|i| include_prs || i.pull_request.is_none()) - .take(MAX_NEW_PER_SYNC) - { + let window = crate::github_issues::fetch_window( + http, + &format!("{base}/issues"), + since, + MAX_NEW_PER_SYNC, + include_prs, + auth, + ) + .await?; + let mut stats = SyncStats { + checkpoint: window.checkpoint, + ..SyncStats::default() + }; + for issue in &window.issues { // 只重建本轮选中的工单，但每张要读完整评论历史。仓库级评论即使 // 去掉 since，也可能被无关工单和 PR 的评论挤出分页上限。 let comments = crate::github_issues::fetch_all::( @@ -1207,21 +1247,14 @@ async fn sync_jira_issues( ) .await?; - let jql = crate::jira_issues::jql(project, since); - let (issues, total) = crate::jira_issues::fetch_all(&http, base_url, &jql, auth).await?; - // **截断了就说出来。** 一个跑了多年的项目动辄上万张工单，翻页上限意味着 - // 这一轮只覆盖了一段；不报的话界面上"同步完成"就是一句误导 - if total > issues.len() as i64 { - tracing::warn!( - source_id = %source.id, - fetched = issues.len(), - total, - "Jira 结果被翻页上限截断，本轮只覆盖了一部分；下一轮的 JQL 窗口会接上" - ); - } - - let mut stats = SyncStats::default(); - for issue in issues.iter().take(MAX_NEW_PER_SYNC) { + let window = + crate::jira_issues::fetch_window(&http, base_url, project, since, MAX_NEW_PER_SYNC, auth) + .await?; + let mut stats = SyncStats { + checkpoint: window.checkpoint, + ..SyncStats::default() + }; + for issue in &window.issues { let body = crate::jira_issues::render(issue); // 逻辑身份带上站点：一个知识库接两个 Jira 时，PROJ-1 不会互相覆盖 let host = reqwest::Url::parse(base_url) diff --git a/crates/utopia-server/src/jira_issues.rs b/crates/utopia-server/src/jira_issues.rs index 4ade51412..b01d36b33 100644 --- a/crates/utopia-server/src/jira_issues.rs +++ b/crates/utopia-server/src/jira_issues.rs @@ -40,13 +40,15 @@ use chrono::{DateTime, Utc}; use serde::Deserialize; +use std::collections::BTreeMap; +use std::time::Duration; use crate::ticket::stamp; -/// 一次同步最多翻多少页。Jira 的 `total` 常常是几万，全量拉回来没有意义—— -/// 增量窗口之外的等下一次 JQL 取。 +/// 达到列表预算后仍读完当前时间组，下次从完整组之后继续。 const MAX_PAGES: u32 = 10; const PAGE_SIZE: u32 = 50; +const MAX_RATE_LIMIT_RETRIES: u32 = 3; /// 想要的字段。**必须显式列**：不列 `comment` 就不返回评论， /// 而默认返回全部字段会把响应撑到几百 KB 一条。 @@ -57,7 +59,7 @@ const FIELDS: &str = "summary,status,issuetype,priority,created,updated,resoluti pub struct SearchPage { #[serde(default)] pub issues: Vec, - #[serde(default)] + // total 用来识别提前结束的分页，缺失时不能把空页当作完整结果。 pub total: i64, } @@ -243,67 +245,494 @@ pub fn jql(project: &str, since: Option>) -> String { t.format("%Y-%m-%d %H:%M") )); } - q.push_str(" ORDER BY updated ASC"); + q.push_str(" ORDER BY updated ASC, key ASC"); q } -/// 分页取。Jira 用 `startAt`/`maxResults`，`total` 常常是几万—— -/// 靠 MAX_PAGES 封顶，剩下的等下一次增量窗口。 -/// -/// **把 `total` 一起返回**：截断了就得说出来。取回 500 条而服务端有 14506 条时， -/// 界面上"同步完成"是一句误导——真实情况是"这一轮只覆盖了一小段"。 -pub async fn fetch_all( +pub struct IssueWindow { + pub issues: Vec, + pub checkpoint: Option>, +} + +/// 复读固定的已选前缀，检测更新导致的 offset 位移；变化时不确认本轮 checkpoint。 +pub async fn fetch_window( + http: &reqwest::Client, + base_url: &str, + project: &str, + since: Option>, + item_limit: usize, + auth: Option<&str>, +) -> anyhow::Result { + let query = jql(project, since); + let (window, positions) = + fetch_window_pass(http, base_url, &query, since, item_limit, auth, None).await?; + if let Some(checkpoint) = window + .checkpoint + .filter(|checkpoint| Some(*checkpoint) != since) + { + let (verified, verified_positions) = fetch_window_pass( + http, + base_url, + &query, + since, + item_limit, + auth, + Some(checkpoint), + ) + .await?; + anyhow::ensure!( + verified.checkpoint == Some(checkpoint) && positions == verified_positions, + "Jira issues changed during pagination; retry the previous checkpoint" + ); + } + Ok(window) +} + +async fn fetch_window_pass( http: &reqwest::Client, base_url: &str, - jql: &str, + query: &str, + since: Option>, + item_limit: usize, auth: Option<&str>, -) -> anyhow::Result<(Vec, i64)> { - let mut out = Vec::new(); - let mut total = 0i64; - for page in 0..MAX_PAGES { + through: Option>, +) -> anyhow::Result<(IssueWindow, BTreeMap>)> { + let mut window = IssueWindow { + issues: Vec::new(), + checkpoint: since, + }; + let mut positions = BTreeMap::new(); + let mut previous_time = None; + let mut stop_after = through; + let mut listing_pages = 0; + let mut start_at = 0; + loop { let mut url = reqwest::Url::parse(&format!( "{}/rest/api/2/search", base_url.trim_end_matches('/') ))?; { let mut q = url.query_pairs_mut(); - q.append_pair("jql", jql); + q.append_pair("jql", query); q.append_pair("expand", "changelog"); q.append_pair("fields", FIELDS); q.append_pair("maxResults", &PAGE_SIZE.to_string()); - q.append_pair("startAt", &(page * PAGE_SIZE).to_string()); + q.append_pair("startAt", &start_at.to_string()); + } + let page_data = fetch_page(http, url, auth).await?; + let count = page_data.issues.len(); + anyhow::ensure!( + page_data.total >= 0 && (start_at + count) as i64 <= page_data.total, + "Jira returned an inconsistent page total; retry the previous checkpoint" + ); + anyhow::ensure!( + count != 0 || start_at as i64 >= page_data.total, + "Jira returned an empty page before its total; retry the previous checkpoint" + ); + let mut page_counted = false; + for issue in page_data.issues { + let updated = issue + .fields + .updated + .ok_or_else(|| { + anyhow::anyhow!("Jira issue {} is missing its update time", issue.key) + })? + .0; + anyhow::ensure!( + previous_time.is_none_or(|previous| updated >= previous), + "Jira issues are not ordered by update time; retry the previous checkpoint" + ); + previous_time = Some(updated); + if stop_after.is_some_and(|boundary| updated > boundary) { + return Ok((window, positions)); + } + // 分钟取整返回的旧记录也可能改变 offset，复读时不能只比较新记录。 + anyhow::ensure!( + positions.insert(issue.key.clone(), updated).is_none(), + "Jira issue repeated during pagination; retry the previous checkpoint" + ); + if since.is_some_and(|checkpoint| updated <= checkpoint) { + continue; + } + if !page_counted { + listing_pages += 1; + page_counted = true; + } + window.checkpoint = Some(updated); + window.issues.push(issue); + if through.is_none() + && (window.issues.len() >= item_limit || listing_pages >= MAX_PAGES) + { + stop_after = Some(updated); + } + } + start_at += count; + if count == 0 || start_at as i64 >= page_data.total { + return Ok((window, positions)); } + } +} + +async fn fetch_page( + http: &reqwest::Client, + url: reqwest::Url, + auth: Option<&str>, +) -> anyhow::Result { + let mut retries = 0; + loop { let mut req = http - .get(url) + .get(url.clone()) .header(reqwest::header::ACCEPT, "application/json"); - if let Some(a) = auth { - req = req.header(reqwest::header::AUTHORIZATION, a); + if let Some(auth) = auth { + req = req.header(reqwest::header::AUTHORIZATION, auth); } let resp = req.send().await?; let status = resp.status(); - if !status.is_success() { - // Jira 把 JQL 语法错误也报成 400，正文里才是原因。 - // 只说 "HTTP 400" 会让人去查网络，而真正该改的是 project key 或时间格式 - let detail = resp.text().await.unwrap_or_default(); - anyhow::bail!( - "HTTP {status} from Jira: {}", - detail.chars().take(200).collect::() - ); + if status.is_success() { + return Ok(resp.json().await?); } - let page_data: SearchPage = resp.json().await?; - total = page_data.total; - let n = page_data.issues.len(); - out.extend(page_data.issues); - if n < PAGE_SIZE as usize { - break; + if status == reqwest::StatusCode::TOO_MANY_REQUESTS && retries < MAX_RATE_LIMIT_RETRIES { + if let Some(delay) = rate_limit_delay(resp.headers(), Utc::now()) { + // 保留已读取的组和当前页，不能把限流变成从头扫描的循环。 + let delay = delay.max(Duration::from_secs(1 << retries)); + retries += 1; + tracing::info!( + retries, + wait_seconds = delay.as_secs(), + "Jira rate limited; waiting before retry" + ); + drop(resp); + tokio::time::sleep(delay).await; + continue; + } } + // Jira 把 JQL 语法错误也报成 400，正文里才是原因。 + let detail = resp.text().await.unwrap_or_default(); + anyhow::bail!( + "HTTP {status} from Jira: {}", + detail.chars().take(200).collect::() + ); } - Ok((out, total)) +} + +fn rate_limit_delay(headers: &reqwest::header::HeaderMap, now: DateTime) -> Option { + let value = headers + .get(reqwest::header::RETRY_AFTER)? + .to_str() + .ok()? + .trim(); + let delay = value + .parse::() + .ok() + .map(Duration::from_secs) + .or_else(|| { + DateTime::parse_from_rfc2822(value) + .ok() + .map(|at| at.signed_duration_since(now).to_std().unwrap_or_default()) + })?; + // shortcut: 长等待走普通失败重试；严格遵循跨任务的 Retry-After 需要持久化调度时间。 + (delay <= Duration::from_secs(30)).then_some(delay) } #[cfg(test)] mod tests { use super::*; + use serde_json::{json, Value}; + use std::sync::atomic::{AtomicBool, Ordering}; + use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate}; + + fn listing_row(number: i64, updated: &str) -> Value { + json!({"key": format!("X-{number}"), "fields": {"summary": "Issue", "updated": updated}}) + } + + fn requested_offset(request: &Request) -> usize { + request + .url + .query_pairs() + .find(|(key, _)| key == "startAt") + .unwrap() + .1 + .parse() + .unwrap() + } + + #[test] + fn rate_limit_headers_supply_a_bounded_wait_without_guessing() { + let now: DateTime = "2026-01-02T00:00:00Z".parse().unwrap(); + let mut headers = reqwest::header::HeaderMap::new(); + assert_eq!(rate_limit_delay(&headers, now), None); + for (value, expected) in [ + ("2", Some(2)), + ("0", Some(0)), + ("30", Some(30)), + ("31", None), + ("-1", None), + ("broken", None), + ("86401", None), + ] { + headers.insert(reqwest::header::RETRY_AFTER, value.parse().unwrap()); + assert_eq!( + rate_limit_delay(&headers, now), + expected.map(Duration::from_secs) + ); + } + for (seconds, expected) in [(3, Some(3)), (30, Some(30)), (31, None)] { + headers.insert( + reqwest::header::RETRY_AFTER, + (now + chrono::Duration::seconds(seconds)) + .to_rfc2822() + .parse() + .unwrap(), + ); + assert_eq!( + rate_limit_delay(&headers, now), + expected.map(Duration::from_secs) + ); + } + } + + #[tokio::test] + async fn a_long_rate_limit_wait_fails_without_holding_the_sync_slot() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(429).insert_header("Retry-After", "31")) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let url = reqwest::Url::parse(&server.uri()).unwrap(); + let result = tokio::time::timeout(Duration::from_secs(1), fetch_page(&http, url, None)) + .await + .expect("a long Retry-After must return an error without waiting"); + assert!(result.unwrap_err().to_string().contains("HTTP 429")); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn a_rate_limited_page_resumes_without_rereading_the_previous_page() { + let server = MockServer::start().await; + let limited = AtomicBool::new(false); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let offset = requested_offset(request); + if offset == 50 && !limited.swap(true, Ordering::SeqCst) { + return ResponseTemplate::new(429).insert_header("Retry-After", "0"); + } + let rows: Vec<_> = (1..=51) + .skip(offset) + .take(50) + .map(|number| listing_row(number, "2026-01-02T00:00:00Z")) + .collect(); + ResponseTemplate::new(200).set_body_json(json!({"issues": rows, "total": 51})) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let started = std::time::Instant::now(); + let window = fetch_window(&http, &server.uri(), "X", None, 200, Some("Bearer fixture")) + .await + .unwrap(); + assert_eq!(window.issues.len(), 51); + assert!(started.elapsed() >= Duration::from_secs(1)); + let requests = server.received_requests().await.unwrap(); + assert_eq!( + requests.iter().map(requested_offset).collect::>(), + [0, 50, 50, 0, 50] + ); + assert!(requests + .iter() + .all( + |request| request.headers.get("authorization").unwrap() == "Bearer fixture" + && request.headers.get("accept").unwrap() == "application/json" + )); + } + + #[tokio::test] + async fn an_empty_window_retains_the_previous_checkpoint_without_verification() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"issues": [], "total": 0})), + ) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let since = Some("2026-01-02T00:00:00Z".parse().unwrap()); + let window = fetch_window(&http, &server.uri(), "X", since, 200, None) + .await + .unwrap(); + assert!(window.issues.is_empty()); + assert_eq!(window.checkpoint, since); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn a_large_timestamp_group_finishes_and_the_next_run_passes_its_old_pages() { + // Jira may return fewer items per page than the requested maxResults. + for (group_size, page_size) in [(501, 50), (101, 10)] { + let server = MockServer::start().await; + let group_time = "2026-01-02T12:34:56Z"; + let later_time = "2026-01-02T12:34:57Z"; + let mut rows: Vec<_> = (1..=group_size) + .map(|number| listing_row(number, group_time)) + .collect(); + rows.push(listing_row(group_size + 1, later_time)); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let offset = requested_offset(request); + let page: Vec<_> = rows.iter().skip(offset).take(page_size).cloned().collect(); + ResponseTemplate::new(200).set_body_json( + json!({"issues": page, "total": rows.len(), "maxResults": page_size}), + ) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let first = fetch_window(&http, &server.uri(), "X", None, 200, None) + .await + .unwrap(); + assert_eq!(first.issues.len(), group_size as usize); + assert_eq!( + first + .issues + .iter() + .map(|issue| issue.key.clone()) + .collect::>(), + (1..=group_size) + .map(|number| format!("X-{number}")) + .collect::>() + ); + assert_eq!(first.checkpoint, Some(group_time.parse().unwrap())); + let next = fetch_window(&http, &server.uri(), "X", first.checkpoint, 200, None) + .await + .unwrap(); + assert_eq!(next.issues.len(), 1); + assert_eq!(next.issues[0].key, format!("X-{}", group_size + 1)); + assert_eq!(next.checkpoint, Some(later_time.parse().unwrap())); + let requests = server.received_requests().await.unwrap(); + assert!(requests.iter().any(|request| { + requested_offset(request) == 10 * page_size + && request.url.query_pairs().any(|(key, value)| { + key == "jql" + && value + == "project = X AND updated >= \"2026-01-02 12:34\" ORDER BY updated ASC, key ASC" + }) + })); + } + } + + #[tokio::test] + async fn a_moved_issue_rejects_the_window_and_a_stable_retry_recovers_every_issue() { + let server = MockServer::start().await; + let moved = AtomicBool::new(false); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let offset = requested_offset(request); + if offset == 50 { + moved.store(true, Ordering::SeqCst); + } + let mut rows: Vec<_> = (1..=201) + .map(|number| listing_row(number, "2026-01-02T00:00:00Z")) + .collect(); + if moved.load(Ordering::SeqCst) { + rows.remove(0); + rows.push(listing_row(1, "2026-01-03T00:00:00Z")); + } + let page: Vec<_> = rows.into_iter().skip(offset).take(50).collect(); + ResponseTemplate::new(200).set_body_json(json!({"issues": page, "total": 201})) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_window(&http, &server.uri(), "X", None, 200, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("changed during pagination")); + + let recovered = fetch_window(&http, &server.uri(), "X", None, 200, None) + .await + .unwrap(); + let mut keys: Vec<_> = recovered.issues.iter().map(|issue| &issue.key).collect(); + keys.sort(); + let mut expected_keys: Vec<_> = (2..=201).map(|number| format!("X-{number}")).collect(); + expected_keys.sort(); + assert_eq!(keys, expected_keys.iter().collect::>()); + assert_eq!( + recovered.checkpoint, + Some("2026-01-02T00:00:00Z".parse().unwrap()) + ); + + let next = fetch_window(&http, &server.uri(), "X", recovered.checkpoint, 200, None) + .await + .unwrap(); + assert_eq!(next.issues.len(), 1); + assert_eq!(next.issues[0].key, "X-1"); + assert_eq!( + next.checkpoint, + Some("2026-01-03T00:00:00Z".parse().unwrap()) + ); + } + + #[tokio::test] + async fn an_issue_without_an_update_time_cannot_advance_a_checkpoint() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "issues": [{"key": "X-1", "fields": {"summary": "Missing time"}}], "total": 1, + }))) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_window(&http, &server.uri(), "X", None, 200, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("missing its update time")); + } + + #[tokio::test] + async fn an_empty_page_cannot_finish_a_group_when_jira_reports_unread_items() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let rows: Vec<_> = if requested_offset(request) == 0 { + (1..=50) + .map(|number| listing_row(number, "2026-01-02T00:00:00Z")) + .collect() + } else { + Vec::new() + }; + ResponseTemplate::new(200).set_body_json(json!({"issues": rows, "total": 51})) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_window(&http, &server.uri(), "X", None, 200, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("empty page before its total")); + assert!(serde_json::from_value::(json!({"issues": []})).is_err()); + for response in [ + json!({"issues": [], "total": -1}), + json!({"issues": [listing_row(1, "2026-01-02T00:00:00Z")], "total": 0}), + ] { + server.reset().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_json(response)) + .mount(&server) + .await; + let result = fetch_window(&http, &server.uri(), "X", None, 200, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("inconsistent page total")); + } + } /// **Jira 的时间戳不是 RFC3339。** `+0000` 没有冒号，chrono 的默认实现解不了。 /// 这一条是整个模块最容易悄悄坏掉的地方：解不出来就是整页工单丢掉。 @@ -322,13 +751,14 @@ mod tests { let t = DateTime::parse_from_rfc3339("2026-08-30T12:34:56Z") .unwrap() .with_timezone(&Utc); - let q = jql("KAFKA", Some(t)); - assert!(q.contains("project = KAFKA"), "{q}"); - assert!(q.contains("updated >= \"2026-08-30 12:34\""), "{q}"); - assert!(q.contains("ORDER BY updated ASC"), "{q}"); - // 没有 since 时不该带上 updated 子句。**查子句不查词**—— - // ORDER BY updated ASC 里也有 "updated"，第一版断言就栽在这上面 - assert!(!jql("KAFKA", None).contains("updated >=")); + assert_eq!( + jql("KAFKA", Some(t)), + "project = KAFKA AND updated >= \"2026-08-30 12:34\" ORDER BY updated ASC, key ASC" + ); + assert_eq!( + jql("KAFKA", None), + "project = KAFKA ORDER BY updated ASC, key ASC" + ); } /// **拿真实响应钉住字段形状。** diff --git a/crates/utopia-server/src/timestamp_checkpoint_tests.rs b/crates/utopia-server/src/timestamp_checkpoint_tests.rs new file mode 100644 index 000000000..f97ae4561 --- /dev/null +++ b/crates/utopia-server/src/timestamp_checkpoint_tests.rs @@ -0,0 +1,495 @@ +//! ADR 0072: persisted progress describes complete source timestamp groups. +use super::{sync_github_issues_with_client, sync_source}; +use chrono::{DateTime, Utc}; +use serde_json::{json, Value}; +use std::sync::{Arc, Mutex}; +use utopia_store::sources; +use uuid::Uuid; +use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate}; + +const FIRST: &str = "2026-01-02T12:34:20Z"; +const NEXT: &str = "2026-01-02T12:34:40Z"; + +fn time(value: &str) -> DateTime { + value.parse().unwrap() +} + +#[derive(Clone, Copy)] +enum Provider { + Github, + Jira, +} + +impl Provider { + fn kind(self) -> &'static str { + match self { + Self::Github => "github_issues", + Self::Jira => "jira_issues", + } + } + + fn row(self, number: usize, updated: &str) -> Value { + match self { + Self::Github => json!({"number":number, "title":format!("Issue {number}"), + "state":"open", "body":format!("Body {number}"), + "created_at":"2026-01-01T00:00:00Z", "updated_at":updated}), + Self::Jira => json!({"key":format!("TEST-{number}"), "fields":{ + "summary":format!("Issue {number}"), "description":format!("Body {number}"), + "created":"2026-01-01T00:00:00Z", "updated":updated}}), + } + } +} + +#[derive(Default)] +struct Feed { + rows: Vec, + fail_page: Option, + oversized_discussion: Option<(&'static str, usize)>, + fail_detail_page: Option, +} + +struct Fixture { + provider: Provider, + state: crate::state::AppState, + source_id: Uuid, + org_id: Uuid, + server: MockServer, + feed: Arc>, + _directory: tempfile::TempDir, +} + +impl Fixture { + async fn new(provider: Provider) -> anyhow::Result> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(None); + }; + let pool = sqlx::PgPool::connect(&url).await?; + let (org_id, workspace_id, kb_id, source_id) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'timestamp-test')") + .bind(org_id) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'timestamp-test')") + .bind(workspace_id) + .bind(org_id) + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'timestamp-test')", + ) + .bind(kb_id) + .bind(workspace_id) + .execute(&pool) + .await?; + let server = MockServer::start().await; + let config = match provider { + Provider::Github => json!({"repo":"test/repo"}), + Provider::Jira => json!({"base_url":server.uri(), "project":"TEST"}), + }; + sqlx::query( + "INSERT INTO sources(id,kb_id,kind,name,config) VALUES($1,$2,$3,'timestamp-test',$4)", + ) + .bind(source_id) + .bind(kb_id) + .bind(provider.kind()) + .bind(config) + .execute(&pool) + .await?; + let directory = tempfile::tempdir()?; + let config = utopia_core::config::AppConfig { + data_dir: directory.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open( + &directory.path().join("search"), + )?); + let state = crate::state::AppState::new(pool, &config, search, "test-only".into()); + let feed = Arc::new(Mutex::new(Feed::default())); + let current = feed.clone(); + Mock::given(method("GET")).respond_with(move |request: &Request| { + let feed = current.lock().unwrap(); + let query_number = |key: &str, default: usize| request.url.query_pairs() + .find(|(name, _)| name == key) + .map(|(_, value)| value.parse::().unwrap()).unwrap_or(default); + let path = request.url.path(); + if path.ends_with("/comments") || path.ends_with("/events") { + if let Some((kind, count)) = feed.oversized_discussion { + if path.ends_with(kind) && path.contains("/issues/1/") { + let page = query_number("page", 1); + if feed.fail_detail_page == Some(page) { + return ResponseTemplate::new(503); + } + let rows: Vec<_> = (1..=count).skip((page - 1) * 100).take(100) + .map(|number| if kind == "comments" { + json!({"issue_url":"https://api.github.com/repos/test/repo/issues/1", + "created_at":FIRST, "body":format!("Discussion item {number}")}) + } else { + json!({"event":"closed", "created_at":FIRST, + "actor":{"login":format!("Discussion item {number}")}}) + }).collect(); + return ResponseTemplate::new(200).set_body_json(rows); + } + } + return ResponseTemplate::new(200).set_body_json(json!([])); + } + let (offset, limit) = match provider { + Provider::Github => ((query_number("page", 1) - 1) * 100, 100), + Provider::Jira => (query_number("startAt", 0), query_number("maxResults", 50)), + }; + if feed.fail_page == Some(offset / limit + 1) { + return ResponseTemplate::new(503); + } + // Return the consumed prefix deliberately: provider bounds are inclusive/rounded. + let rows: Vec<_> = feed.rows.iter().skip(offset).take(limit).cloned().collect(); + let body = match provider { + Provider::Github => json!(rows), + Provider::Jira => json!({"issues":rows,"total":feed.rows.len(),"startAt":offset,"maxResults":limit}), + }; + ResponseTemplate::new(200).set_body_json(body) + }).mount(&server).await; + Ok(Some(Self { + provider, + state, + source_id, + org_id, + server, + feed, + _directory: directory, + })) + } + + fn set_rows(&self, first_count: usize, next_count: usize) { + self.feed.lock().unwrap().rows = (1..=first_count + next_count) + .map(|number| { + self.provider + .row(number, if number <= first_count { FIRST } else { NEXT }) + }) + .collect(); + } + + async fn sync(&self) -> anyhow::Result<()> { + if matches!(self.provider, Provider::Jira) { + sync_source(&self.state, self.source_id).await?; + return Ok(()); + } + let previous = sources::sync_checkpoint(&self.state.pool, self.source_id).await?; + let source = sources::get(&self.state.pool, self.source_id).await?; + let run = sources::start_run(&self.state.pool, self.source_id).await?; + let http = reqwest::Client::builder().no_proxy().build()?; + let stats = sync_github_issues_with_client( + &self.state, + &source, + previous, + &http, + &self.server.uri(), + ) + .await?; + sources::finish_timestamp_sync( + &self.state.pool, + run, + self.source_id, + previous, + stats.checkpoint, + stats.created as i32, + stats.updated as i32, + ) + .await?; + Ok(()) + } + + async fn checkpoint(&self) -> anyhow::Result>> { + Ok(sources::sync_checkpoint(&self.state.pool, self.source_id).await?) + } + + async fn counts(&self) -> anyhow::Result<(i64, i64, i64)> { + Ok(sqlx::query_as("SELECT + (SELECT count(*) FROM documents WHERE source_id=$1), + (SELECT count(*) FROM document_versions v JOIN documents d ON d.id=v.document_id WHERE d.source_id=$1), + (SELECT count(*) FROM jobs j JOIN documents d ON d.id::text=j.payload->>'document_id' WHERE d.source_id=$1)") + .bind(self.source_id).fetch_one(&self.state.pool).await?) + } + + async fn cleanup(self) -> anyhow::Result<()> { + sqlx::query("DELETE FROM jobs WHERE payload->>'document_id' IN (SELECT id::text FROM documents WHERE source_id=$1)") + .bind(self.source_id).execute(&self.state.pool).await?; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(self.org_id) + .execute(&self.state.pool) + .await?; + Ok(()) + } +} + +async fn large_group_progress(provider: Provider, group_size: usize) -> anyhow::Result<()> { + let Some(fixture) = Fixture::new(provider).await? else { + return Ok(()); + }; + fixture.set_rows(group_size, 1); + fixture.sync().await?; + assert_eq!(fixture.counts().await?.0, group_size as i64); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + fixture.sync().await?; + let counts = fixture.counts().await?; + assert_eq!( + counts, + ( + (group_size + 1) as i64, + (group_size + 1) as i64, + (group_size + 1) as i64 + ) + ); + assert_eq!(fixture.checkpoint().await?, Some(time(NEXT))); + fixture.sync().await?; + assert_eq!(fixture.counts().await?, counts); + assert_eq!(fixture.checkpoint().await?, Some(time(NEXT))); + let requests = fixture.server.received_requests().await.unwrap(); + let incremental = requests.iter().any(|request| match provider { + Provider::Github => request.url.query_pairs().any(|(key, value)| { + key == "since" + && (time(&value) == time(FIRST) + || time(&value) == time(FIRST) + chrono::Duration::seconds(1)) + }), + Provider::Jira => request + .url + .query_pairs() + .any(|(key, value)| key == "jql" && value.contains("2026-01-02 12:34")), + }); + assert!( + incremental, + "the next run must send the persisted source bound" + ); + fixture.cleanup().await +} + +#[tokio::test] +async fn github_imports_1001_equal_time_items_then_skips_the_old_prefix() -> anyhow::Result<()> { + large_group_progress(Provider::Github, 1001).await +} + +#[tokio::test] +async fn jira_imports_501_equal_time_items_then_skips_the_rounded_prefix() -> anyhow::Result<()> { + large_group_progress(Provider::Jira, 501).await +} + +#[tokio::test] +async fn processing_budget_finishes_item_240_and_leaves_the_next_group() -> anyhow::Result<()> { + for provider in [Provider::Github, Provider::Jira] { + large_group_progress(provider, 240).await?; + } + Ok(()) +} + +#[tokio::test] +async fn failed_listing_does_not_advance_and_retry_deduplicates() -> anyhow::Result<()> { + for provider in [Provider::Github, Provider::Jira] { + let Some(fixture) = Fixture::new(provider).await? else { + return Ok(()); + }; + fixture.set_rows(240, 0); + fixture.feed.lock().unwrap().fail_page = Some(2); + assert!(fixture.sync().await.is_err()); + assert_eq!(fixture.checkpoint().await?, None); + fixture.feed.lock().unwrap().fail_page = None; + fixture.sync().await?; + assert_eq!(fixture.counts().await?, (240, 240, 240)); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + fixture.cleanup().await?; + } + Ok(()) +} + +#[tokio::test] +async fn github_failed_discussion_tail_preserves_the_document_and_recovers() -> anyhow::Result<()> { + for (endpoint, count) in [("comments", 1001), ("events", 1100)] { + let Some(fixture) = Fixture::new(Provider::Github).await? else { + return Ok(()); + }; + fixture.set_rows(1, 0); + fixture.sync().await?; + let original: String = + sqlx::query_scalar("SELECT sha256 FROM documents WHERE source_id=$1") + .bind(fixture.source_id) + .fetch_one(&fixture.state.pool) + .await?; + { + let mut feed = fixture.feed.lock().unwrap(); + feed.rows = vec![Provider::Github.row(1, NEXT), Provider::Github.row(2, NEXT)]; + feed.oversized_discussion = Some((endpoint, count)); + feed.fail_detail_page = Some(11); + } + assert!( + fixture.sync().await.is_err(), + "failed {endpoint} tail must not replace a document" + ); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + let current: String = sqlx::query_scalar("SELECT sha256 FROM documents WHERE source_id=$1") + .bind(fixture.source_id) + .fetch_one(&fixture.state.pool) + .await?; + assert_eq!(current, original); + assert_eq!(fixture.counts().await?, (1, 1, 1)); + fixture.feed.lock().unwrap().fail_detail_page = None; + fixture.sync().await?; + assert_eq!(fixture.checkpoint().await?, Some(time(NEXT))); + assert_eq!(fixture.counts().await?, (2, 3, 3)); + let updated: String = sqlx::query_scalar( + "SELECT sha256 FROM documents WHERE source_id=$1 AND external_key='github:test/repo#1'", + ) + .bind(fixture.source_id) + .fetch_one(&fixture.state.pool) + .await?; + let body = String::from_utf8(fixture.state.blob.get(&updated).await?)?; + assert!(body.contains(&format!("Discussion item {count}"))); + fixture.sync().await?; + assert_eq!(fixture.counts().await?, (2, 3, 3)); + fixture.cleanup().await?; + } + Ok(()) +} + +#[tokio::test] +async fn recovery_ignores_old_success_and_retains_history_after_audit_pruning() -> anyhow::Result<()> +{ + for provider in [Provider::Github, Provider::Jira] { + let Some(fixture) = Fixture::new(provider).await? else { + return Ok(()); + }; + fixture.set_rows(1, 0); + fixture.sync().await?; + let before = fixture.counts().await?; + sqlx::query("UPDATE sources SET sync_checkpoint=NULL WHERE id=$1") + .bind(fixture.source_id) + .execute(&fixture.state.pool) + .await?; + assert_eq!(fixture.checkpoint().await?, None); + fixture.sync().await?; + assert_eq!( + fixture.counts().await?, + before, + "recovery must retain versions and jobs" + ); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + for _ in 0..51 { + let run = sources::start_run(&fixture.state.pool, fixture.source_id).await?; + sources::finish_run( + &fixture.state.pool, + run, + fixture.source_id, + Some("fixture failure"), + 0, + 0, + ) + .await?; + } + assert_eq!( + sources::list_runs(&fixture.state.pool, fixture.source_id, 100) + .await? + .len(), + 50 + ); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + fixture.feed.lock().unwrap().rows.clear(); + fixture.sync().await?; + assert_eq!( + fixture.checkpoint().await?, + Some(time(FIRST)), + "empty reads must not advance to wall clock" + ); + assert_eq!(fixture.counts().await?, before); + fixture.cleanup().await?; + } + Ok(()) +} + +#[tokio::test] +async fn checkpoint_and_run_success_commit_atomically() -> anyhow::Result<()> { + let Some(fixture) = Fixture::new(Provider::Github).await? else { + return Ok(()); + }; + let pool = &fixture.state.pool; + assert!(sources::finish_timestamp_sync( + pool, + Uuid::now_v7(), + fixture.source_id, + None, + Some(time(FIRST)), + 1, + 0 + ) + .await + .is_err()); + assert_eq!( + fixture.checkpoint().await?, + None, + "missing run must roll back source checkpoint" + ); + let run = sources::start_run(pool, fixture.source_id).await?; + sources::finish_timestamp_sync(pool, run, fixture.source_id, None, Some(time(FIRST)), 1, 0) + .await?; + let stale_run = sources::start_run(pool, fixture.source_id).await?; + assert!(sources::finish_timestamp_sync( + pool, + stale_run, + fixture.source_id, + None, + Some(time(NEXT)), + 1, + 0 + ) + .await + .is_err()); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + let status: String = sqlx::query_scalar("SELECT status FROM source_sync_runs WHERE id=$1") + .bind(stale_run) + .fetch_one(pool) + .await?; + assert_eq!( + status, "running", + "checkpoint conflict must not mark the stale run successful" + ); + fixture.cleanup().await +} + +#[tokio::test] +async fn a_document_write_failure_keeps_the_checkpoint_and_retry_deduplicates() -> anyhow::Result<()> +{ + for provider in [Provider::Github, Provider::Jira] { + let Some(fixture) = Fixture::new(provider).await? else { + return Ok(()); + }; + fixture.set_rows(3, 0); + // The constraint affects only this fixture's second document, including on shared test databases. + let constraint = format!("checkpoint_write_failure_{}", fixture.source_id.simple()); + let second_key = match provider { + Provider::Github => "github:test/repo#2".to_owned(), + Provider::Jira => "jira:127.0.0.1/TEST-2".to_owned(), + }; + sqlx::query(&format!("ALTER TABLE documents ADD CONSTRAINT {constraint} CHECK \ + (source_id IS DISTINCT FROM '{}'::uuid OR external_key IS DISTINCT FROM '{}') NOT VALID", + fixture.source_id, second_key)) + .execute(&fixture.state.pool).await?; + let failed = fixture.sync().await; + sqlx::query(&format!( + "ALTER TABLE documents DROP CONSTRAINT {constraint}" + )) + .execute(&fixture.state.pool) + .await?; + assert!(failed.is_err(), "the second document write must fail"); + assert_eq!(fixture.checkpoint().await?, None); + assert_eq!( + fixture.counts().await?, + (1, 1, 1), + "the first item was committed before the failure" + ); + fixture.sync().await?; + assert_eq!(fixture.counts().await?, (3, 3, 3)); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + fixture.cleanup().await?; + } + Ok(()) +} diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs index 6eb3d01c0..29e4d7d9a 100644 --- a/crates/utopia-store/src/sources.rs +++ b/crates/utopia-store/src/sources.rs @@ -459,6 +459,76 @@ pub async fn touch_sync_time(pool: &PgPool, id: Uuid, at: DateTime) -> AppR // 同步运行记录（渠道审计历史） // --------------------------------------------------------------------------- +/// 工单来源的进度独立于只保留 50 条的审计历史；NULL 也不能回退到旧的 ok 行。 +pub async fn sync_checkpoint(pool: &PgPool, source_id: Uuid) -> AppResult>> { + sqlx::query_scalar("SELECT sync_checkpoint FROM sources WHERE id = $1") + .bind(source_id) + .fetch_optional(pool) + .await? + .ok_or(AppError::NotFound) +} + +/// 完整时间戳组的进度与成功记录一起提交，任一写入失败都保留旧游标。 +pub async fn finish_timestamp_sync( + pool: &PgPool, + run_id: Uuid, + source_id: Uuid, + previous_checkpoint: Option>, + checkpoint: Option>, + created_docs: i32, + updated_docs: i32, +) -> AppResult<()> { + if checkpoint < previous_checkpoint { + return Err(AppError::Validation( + "Source checkpoint cannot move backwards".into(), + )); + } + let mut tx = pool.begin().await?; + let source = sqlx::query( + "UPDATE sources SET sync_checkpoint = $3, last_sync_status = 'ok', + last_sync_error = NULL, last_sync_added = $4, last_sync_at = now() + WHERE id = $1 AND kind IN ('github_issues', 'jira_issues') + AND sync_checkpoint IS NOT DISTINCT FROM $2::timestamptz", + ) + .bind(source_id) + .bind(previous_checkpoint) + .bind(checkpoint) + .bind(created_docs + updated_docs) + .execute(&mut *tx) + .await?; + if source.rows_affected() != 1 { + return Err(AppError::Conflict( + "Source checkpoint changed during sync".into(), + )); + } + let run = sqlx::query( + "UPDATE source_sync_runs SET finished_at = now(), status = 'ok', error = NULL, + created_docs = $3, updated_docs = $4 + WHERE id = $1 AND source_id = $2 AND status = 'running'", + ) + .bind(run_id) + .bind(source_id) + .bind(created_docs) + .bind(updated_docs) + .execute(&mut *tx) + .await?; + if run.rows_affected() != 1 { + return Err(AppError::Conflict( + "Source sync run is no longer running".into(), + )); + } + sqlx::query( + "DELETE FROM source_sync_runs WHERE source_id = $1 AND id NOT IN + (SELECT id FROM source_sync_runs WHERE source_id = $1 + ORDER BY started_at DESC LIMIT 50)", + ) + .bind(source_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + /// 调度时钟包含失败尝试，不能作增量游标。回到上次成功运行的开始， /// 让那次拉取期间发生的更新也能在下一轮读到；没有成功记录就重新全量读取。 pub async fn last_successful_sync_start( diff --git a/docs/decisions/0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md b/docs/decisions/0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md index d0f0bbbc9..5f143ac41 100644 --- a/docs/decisions/0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md +++ b/docs/decisions/0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md @@ -1,6 +1,6 @@ # A source checkpoint ends at a timestamp boundary -Proposed 2026-10-19 (#1212, #1213); open: implementation and regression coverage. +In progress 2026-10-25 (#1359, migration 0119); open: custom batch progression and recovery (#1397). ## Context @@ -27,8 +27,14 @@ This follows the [maintainer's decision](https://github.com/deeplethe/utopia/iss ### GitHub and Jira -Read items in ascending update-time order. Skip items at or before the -checkpoint before counting either budget. When the 200-item processing budget +GitHub/Jira store their watermark in `sources.sync_checkpoint`, initially NULL even +when successful audit rows exist. Progress and the successful run commit together. +The 2026-10-23 revision below records the listing verification, complete GitHub +discussion reads and bounded waits used by these readers. + +Read items in ascending update-time order. Jira breaks ties by key so an unchanged +result keeps the same page boundaries. Skip items at or before the checkpoint +before counting either budget. When the 200-item processing budget or ten-page listing budget is exhausted inside a group, continue reading and processing until that group is complete. Stop before consuming the next timestamp group, then save the completed group's time as the checkpoint. @@ -58,6 +64,45 @@ group cap or a group-size alert; use the existing rate-limit backoff. The exception applies to source listing and processing, not permission to replace a GitHub document with a truncated comments or events history. +### Revision 2026-10-23: verification, complete discussions and bounded waits + +Keep the second listing pass before ingestion. Re-read through the selected +timestamp boundary and compare the listed identities and update times with the +first pass, including skipped old rows and excluded GitHub PRs. Those rows can +still move offsets. For example, after reading issues 1–100, updating issue 1 +can move it to the tail; the next page then starts at issue 102 and silently +misses issue 101. Advancing the checkpoint without detecting that movement can +make issue 101 unreachable on later incremental runs. Jira has the same risk. + +The second pass roughly doubles requests for the selected listing prefix, not +GitHub's per-ticket comments and events requests. It does not extend the chosen +window, and changes only beyond that boundary do not invalidate the prefix. +Comparison ignores ordering within one timestamp group. A changed identity or +update time fails the run before ingestion, preserves the previous checkpoint, +sets `last_sync_status` to `failed` and raises the existing `source.sync_failed` +alert. A later run starts at the same checkpoint. Keep this failure behavior; +do not add repeated scans inside the same run. This detects observed movement, +not an atomic snapshot. A continuously changing prefix can repeatedly fail; +progress under sustained concurrent updates is not guaranteed. + +GitHub comments and events must be read to the end, even beyond ten pages. +The listing budget is not a limit on a selected ticket's discussion history. +Do not skip a large discussion, return truncated content or advance progress +past a ticket whose details could not be read completely. A detail-page failure +keeps the old document and checkpoint. Large discussions therefore increase +requests, memory use and run duration. Jira's search `total` remains required: +without it, a premature empty page could be mistaken for a complete group. + +For explicit rate limits, keep at most three retries of the current request, +preserving its in-memory position. Honor a valid wait hint only when it is at +most 30 seconds per retry, retaining the existing minimum backoff. An invalid +or longer hint fails the run immediately without sleeping or shortening the +provider's requested wait. This prevents one long wait from occupying a sync +slot for hours; it is not a total run-duration limit. The existing job retry +and source schedule govern later attempts and do not persist `Retry-After`, so +they do not guarantee that a later attempt waits until the provider's deadline. +Strict cross-job rate-limit scheduling remains outside this revision. + ### Custom The endpoint already returns the whole requested response. Walk its items in @@ -73,7 +118,7 @@ incomplete runs. No source-update field, pagination token or saved response is added to the custom protocol; `doc_time` remains document time. The custom fix is being reviewed in -[PR #1246](https://github.com/deeplethe/utopia/pull/1246). It does not resolve +[PR #1397](https://github.com/deeplethe/utopia/pull/1397). It does not resolve the GitHub and Jira parts of #1212. ## Recovery and tradeoffs @@ -120,6 +165,12 @@ must demonstrate: this progress without omissions or duplicate versions and processing jobs. - A failed page fetch, detail fetch or write cannot advance the checkpoint; retry recovers the whole group. Updates during pagination remain reachable. +- A page-moving update rejects the window; once the listing stabilizes, a + retry recovers the skipped issue and a later run reads the moved issue. +- GitHub comments and events beyond ten pages are fully read; failure on a + later detail page preserves the old document and checkpoint. +- A 30-second rate-limit hint is accepted; a 31-second hint returns an error + without sleeping or retrying that request. Cover seconds and HTTP-date hints. - Custom's 201 changed items finish on the second run; 450 finish over three capped runs. Cover an unchanged prefix and no custom update field. - The one-time recovery scan preserves existing data and returns to incremental diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 64b16381b..3ac9cafb3 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -115,7 +115,7 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | 0052 | [Document content is a read contract over the retained ledger](0052-document-content-is-a-read-contract.md) | Proposed | | | 0054 | [A source may push statements in the open contract](0054-a-source-may-push-statements-in-the-open-contract.md) | Implemented | | | 0065 | [The OCR reader chooses its provider](0065-readers-choose-their-provider.md) | Implemented | | -| 0072 | [A source checkpoint ends at a timestamp boundary](0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md) | Proposed | | +| 0072 | [A source checkpoint ends at a timestamp boundary](0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md) | In progress | | | 0074 | [A Google Drive source is connected by a person](0074-a-google-drive-source-is-connected-by-a-person.md) | Implemented | | ### [lakehouse-and-actions](../design/lakehouse-and-actions.md) diff --git a/docs/design/sources.md b/docs/design/sources.md index 8af6d16c9..5cd6fca9b 100644 --- a/docs/design/sources.md +++ b/docs/design/sources.md @@ -28,6 +28,23 @@ from a `HEAD` on the file before anything is downloaded [#1327]. **Google Drive.** A person connects it with OAuth and it reads Drive as them; the refresh token is a sealed source secret and the browser never holds a Google token; the flow is bound to the session that started it; only the person who connected picks folders, subfolders come along, and a sync stops once that person is no longer an editor; a run downloads at most 2000 files and says so when it stops there; Docs, Sheets and Slides are exported to DOCX, XLSX and PPTX; change is Drive's marker (`md5Checksum`, or `modifiedTime` for Google formats) because an export's bytes differ every time; "not in source" only after a complete walk [0074]. +**Ticket checkpoints.** GitHub/Jira read by ascending update time and stop only after +a whole timestamp group, even when that exceeds 200 items or ten listing pages. +Already consumed timestamps use neither budget. Their private source checkpoint is +committed with the successful run, independent of retained audit history; empty or +failed runs do not advance it. Upgrades start these two readers from NULL and reuse +document/version/job deduplication. A second listing pass checks the selected prefix +for page movement before ingestion, roughly doubling listing requests. A changed +prefix fails with the existing sync alert and keeps the old checkpoint; neither +API provides a snapshot, and a constantly changing prefix may repeatedly fail. +GitHub reads selected tickets' comments and events to the end, including beyond +ten pages, before replacing a document. Explicit rate limits retry the current +request at most three times with waits of at most 30 seconds each; longer hints +fail immediately. Later ordinary job retries and source schedules do not preserve +the provider's `Retry-After` deadline. Jira requires the search `total` to detect +premature pagination endings. Custom batch progression is separate work in #1397 +[0072, revision 2026-10-23]. + **Statements.** A `statements` source takes the open extraction contract itself (`e`, `s`, `n`) on `POST /sources/{id}/statements`, with the `api` push's identity, versions, tombstones and run history; the payload is the document and its single chunk, and extraction parses the chunk instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement with the item as its evidence (null offsets) and reaches the typed graph only through alignment. The contract has no slot for a property or a class and unknown keys are refused; a second push under the same identity marks the earlier statements stale rather than closing them; tables belong on a mount, not here [0054]. **RSS.** Observations are not documents: one table with baseline, candidate and no_source rows; diff --git a/migrations/0119_a_source_checkpoint_ends_at_a_timestamp_boundary.sql b/migrations/0119_a_source_checkpoint_ends_at_a_timestamp_boundary.sql new file mode 100644 index 000000000..5659156b0 --- /dev/null +++ b/migrations/0119_a_source_checkpoint_ends_at_a_timestamp_boundary.sql @@ -0,0 +1,4 @@ +-- 不从旧的 ok 审计记录回填：旧版本可能只读完前 200 条就记成功。 +-- GitHub/Jira 从 NULL 开始恢复扫描，保留全部文档、版本、任务与审计历史。 +-- Custom 的批次推进由 #1397 单独实现，这里不改变它的游标。 +ALTER TABLE sources ADD COLUMN sync_checkpoint timestamptz; From fb0caed87f7170efb73eb855ed19d80edc59273c Mon Sep 17 00:00:00 2001 From: Quintin Botes Date: Mon, 26 Oct 2026 21:00:53 +0100 Subject: [PATCH 03/27] A deployment that refuses reasoning_effort none on tool requests too is not sent none again (#1353) (#1372) * A deployment that refuses reasoning_effort none on tool requests too is not sent none again (#1353) Signed-off-by: Quintin Botes * Only a retry refused for reasoning_effort forgets none, so an unrelated 400 keeps it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Quintin Botes --------- Signed-off-by: Quintin Botes Co-authored-by: Claude Opus 5.5 Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-llm/src/lib.rs | 76 ++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rs index bd95945d0..b4df40d5e 100644 --- a/crates/utopia-llm/src/lib.rs +++ b/crates/utopia-llm/src/lib.rs @@ -953,6 +953,7 @@ impl LlmClient { stream: bool, ) -> anyhow::Result { use std::sync::atomic::Ordering; + let mut retried = false; loop { let learned = self.tools_without_effort.load(Ordering::Relaxed); let resp = self @@ -975,8 +976,19 @@ impl LlmClient { { tracing::info!(model = %self.model, "端点不收带工具的推理强度，带工具的请求改发 none 并记住"); self.tools_without_effort.store(true, Ordering::Relaxed); + retried = true; continue; } + // 改发 none 也被拒，且错误还是在说推理强度：这个端点连 none 都不收，记住它只会让以后每一问都带着一个错的值。 + // 别的 400（上下文超长、工具定义不对）跟 none 无关，记住的值留着 + if retried + && status == reqwest::StatusCode::BAD_REQUEST + && err_detail(&parsed, &raw) + .to_lowercase() + .contains("reasoning_effort") + { + self.tools_without_effort.store(false, Ordering::Relaxed); + } return Err(failure("LLM", status, retry_after, &parsed, &raw)); } } @@ -1764,6 +1776,70 @@ mod tests { ); } + #[tokio::test] + async fn a_deployment_that_refuses_none_too_is_not_sent_none_again() { + use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400).set_body_json(serde_json::json!({ "error": { + "message": "Function tools with reasoning_effort are not supported for gpt-x." } }))) + .mount(&server) + .await; + let client = + LlmClient::new(&server.uri(), None, "m").with_reasoning_effort(Some("low".into())); + let tool = serde_json::json!([{ "type": "function", "function": { "name": "ping" } }]); + let ask = [serde_json::json!({ "role": "user", "content": "x" })]; + assert!(client.chat_tools(&ask, &tool).await.is_err()); + let sent: Vec = server + .received_requests() + .await + .unwrap() + .iter() + .map(|r| serde_json::from_slice(&r.body).unwrap()) + .collect(); + assert_eq!( + sent.iter() + .map(|b| b["reasoning_effort"].clone()) + .collect::>(), + ["low", "none"] + ); + assert_eq!( + client.tools_body(&ask, Some(&tool), None, false)["reasoning_effort"], + "low", + "the flag is cleared" + ); + } + + #[tokio::test] + async fn an_unrelated_400_on_the_retry_keeps_none() { + use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400).set_body_json(serde_json::json!({ "error": { + "message": "Function tools with reasoning_effort are not supported for gpt-x." } }))) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(400).set_body_json(serde_json::json!({ "error": { + "message": "This model's maximum context length is 8192 tokens." } })), + ) + .mount(&server) + .await; + let client = + LlmClient::new(&server.uri(), None, "m").with_reasoning_effort(Some("low".into())); + let tool = serde_json::json!([{ "type": "function", "function": { "name": "ping" } }]); + let ask = [serde_json::json!({ "role": "user", "content": "x" })]; + assert!(client.chat_tools(&ask, &tool).await.is_err()); + assert_eq!(server.received_requests().await.unwrap().len(), 2); + assert_eq!( + client.tools_body(&ask, Some(&tool), None, false)["reasoning_effort"], + "none", + "the flag is kept" + ); + } + #[tokio::test] async fn another_400_on_a_tool_request_is_not_retried() { use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; From 6466a5e41b63c4df430792a7243c2747d62440fc Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Mon, 26 Oct 2026 21:06:37 +0100 Subject: [PATCH 04/27] A search may name a source and an external_id prefix, and both channels recall only those documents before fusion (#1427) Signed-off-by: Bogdan Szabo Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-core/src/models.rs | 9 +- crates/utopia-search/src/lib.rs | 21 ++- .../utopia-server/src/api/documents_routes.rs | 4 +- crates/utopia-server/src/api/search_routes.rs | 41 +++- .../utopia-server/src/api/sources_routes.rs | 14 +- .../utopia-server/src/api/token_read_tests.rs | 175 +++++++++++++++++- crates/utopia-server/src/main.rs | 1 + crates/utopia-server/src/pipeline_tests.rs | 5 +- crates/utopia-server/src/pushed_keys.rs | 23 +++ crates/utopia-server/src/retrieval.rs | 84 ++++++--- crates/utopia-server/src/retrieval_tests.rs | 83 ++++++++- crates/utopia-server/src/search_recovery.rs | 23 ++- crates/utopia-store/src/documents.rs | 39 +++- .../a_search_reads_the_base_as_it_was.rs | 6 +- ...st_chunk_is_found_however_it_is_reached.rs | 18 +- ...ty-from-the-person-scope-from-the-token.md | 8 +- docs/design/chat-and-mcp.md | 5 +- web/src/docs/ingest.md | 2 + 18 files changed, 497 insertions(+), 64 deletions(-) create mode 100644 crates/utopia-server/src/pushed_keys.rs diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 178a9fffc..14e8164eb 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -479,15 +479,18 @@ pub struct ChunkView { pub filename: String, } -/// 分块带上它来自哪个来源。`external_id` 只有推送来源（api / statements）才有： -/// 推送方给的那个 id，去掉库里加的 `{kind}:` 前缀，好让它拿回去对自己的记录（#1271） +/// 分块带上它来自哪个来源，连同来源的 kind 和文档的 `external_key`：推送来源的 +/// `external_id` 由这两样反推（`sources_routes::pushed_identity`，#1271） #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct SourcedChunk { #[serde(flatten)] #[sqlx(flatten)] pub chunk: ChunkView, pub source_id: Option, - pub external_id: Option, + #[serde(skip)] + pub source_kind: Option, + #[serde(skip)] + pub external_key: Option, } #[derive(Debug, Clone, Serialize, sqlx::FromRow)] diff --git a/crates/utopia-search/src/lib.rs b/crates/utopia-search/src/lib.rs index f518825bd..54ca0b6f2 100644 --- a/crates/utopia-search/src/lib.rs +++ b/crates/utopia-search/src/lib.rs @@ -10,7 +10,7 @@ use std::sync::Mutex; use anyhow::Context; use tantivy::collector::TopDocs; -use tantivy::query::{BooleanQuery, Occur, Query, TermQuery}; +use tantivy::query::{BooleanQuery, Occur, Query, TermQuery, TermSetQuery}; use tantivy::schema::{ Field, IndexRecordOption, Schema, TextFieldIndexing, TextOptions, Value, STORED, STRING, }; @@ -134,7 +134,15 @@ impl SearchIndex { /// BM25 检索（限定 kb）。 /// 查询用与索引完全一致的 jieba analyzer 切词后按 OR 组合—— /// 不能走 QueryParser：CJK 整句会被当成短语查询（要求词连续出现），召回归零。 - pub fn search(&self, kb_id: &str, query: &str, limit: usize) -> anyhow::Result> { + /// + /// `documents`：只在这些文档里找。它和 kb 一样是 `Must`，`limit` 数的就只是允许的块 + pub fn search( + &self, + kb_id: &str, + query: &str, + limit: usize, + documents: Option<&[String]>, + ) -> anyhow::Result> { let mut analyzer = self.analyzer.clone(); let mut stream = analyzer.token_stream(query); let mut term_queries: Vec<(Occur, Box)> = Vec::new(); @@ -161,7 +169,14 @@ impl SearchIndex { IndexRecordOption::Basic, )); let text_query: Box = Box::new(BooleanQuery::new(term_queries)); - let combined = BooleanQuery::new(vec![(Occur::Must, kb_filter), (Occur::Must, text_query)]); + let mut clauses = vec![(Occur::Must, kb_filter), (Occur::Must, text_query)]; + if let Some(documents) = documents { + let terms = documents + .iter() + .map(|id| Term::from_field_text(self.f_document_id, id)); + clauses.push((Occur::Must, Box::new(TermSetQuery::new(terms)))); + } + let combined = BooleanQuery::new(clauses); let searcher = self.reader.searcher(); let collector = TopDocs::with_limit(limit).order_by_score(); diff --git a/crates/utopia-server/src/api/documents_routes.rs b/crates/utopia-server/src/api/documents_routes.rs index 62746b758..f740cde67 100644 --- a/crates/utopia-server/src/api/documents_routes.rs +++ b/crates/utopia-server/src/api/documents_routes.rs @@ -227,11 +227,11 @@ pub async fn by_external_id( ) -> ApiResult { require_reader_kb(&state, &reader, kb_id).await?; let source = super::sources_routes::source_in_kb(&state, kb_id, source_id).await?; - if !super::sources_routes::is_push_kind(&source.kind) { + if !crate::pushed_keys::is_push_kind(&source.kind) { return Err(AppError::NotFound.into()); } let external_id = query.external_id.trim(); - let key = super::sources_routes::pushed_key(&source, external_id); + let key = crate::pushed_keys::pushed_key(&source, external_id); let found = utopia_store::documents::find_by_external_key(&state.pool, source.id, &key) .await? .ok_or(AppError::NotFound)?; diff --git a/crates/utopia-server/src/api/search_routes.rs b/crates/utopia-server/src/api/search_routes.rs index 10ae4066d..daba73234 100644 --- a/crates/utopia-server/src/api/search_routes.rs +++ b/crates/utopia-server/src/api/search_routes.rs @@ -10,7 +10,7 @@ use crate::error::ApiResult; use crate::retrieval; use crate::state::AppState; -/// 不认识的字段答 422：过滤条件还没做，悄悄忽略一个 `filters` 会让调用方以为它生效了 +/// 不认识的字段答 422：悄悄忽略一个拼错的过滤条件，调用方会以为它生效了 #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub struct SearchReq { @@ -23,6 +23,44 @@ pub struct SearchReq { /// 命中的是当时活着的块、当时还没被删的文档 #[serde(default)] pub as_of: Option, + /// 只检索本库这个来源的文档；别的库的来源答 404 + #[serde(default)] + pub source_id: Option, + /// 只检索 `external_id` 以它开头的文档。identity 是推送来源里的，所以要 `source_id` + /// 且它是推送来源，否则 422 + #[serde(default)] + pub external_id_prefix: Option, +} + +async fn source_filter( + state: &AppState, + kb_id: Uuid, + req: &SearchReq, +) -> ApiResult> { + let source = match req.source_id { + Some(source_id) => { + Some(super::sources_routes::source_in_kb(state, kb_id, source_id).await?) + } + None => None, + }; + let Some(prefix) = &req.external_id_prefix else { + return Ok(source.map(|source| retrieval::SourceFilter { + source_id: source.id, + key_prefix: None, + })); + }; + let Some(source) = source.filter(|source| crate::pushed_keys::is_push_kind(&source.kind)) + else { + return Err(AppError::invalid( + "prefix_needs_push_source", + "external_id_prefix needs the source_id of an API or Statements source", + ) + .into()); + }; + Ok(Some(retrieval::SourceFilter { + source_id: source.id, + key_prefix: Some(crate::pushed_keys::pushed_key(&source, prefix)), + })) } pub async fn search( @@ -50,6 +88,7 @@ pub async fn search( top_k, offset: req.offset, as_of, + filter: source_filter(&state, kb_id, &req).await?, }; let page = retrieval::hybrid_scored(&state, kb_id, kb.workspace_id, q, opts, None).await?; Ok(Json(json!({ diff --git a/crates/utopia-server/src/api/sources_routes.rs b/crates/utopia-server/src/api/sources_routes.rs index ddd859dd8..bcb62b728 100644 --- a/crates/utopia-server/src/api/sources_routes.rs +++ b/crates/utopia-server/src/api/sources_routes.rs @@ -12,6 +12,7 @@ use uuid::Uuid; use super::graph_routes::require_kb; use crate::auth::{AuthUser, SessionOrToken}; use crate::error::ApiResult; +use crate::pushed_keys::{is_push_kind, pushed_key}; use crate::state::AppState; /// 生成 api 来源的推送密钥。 @@ -166,19 +167,6 @@ pub async fn create( )) } -/// 有推送密钥的来源：`api` 推文档，`statements` 推陈述（0054）。两种的密钥同一套 -/// 生成、查看、轮换 -pub(crate) fn is_push_kind(kind: &str) -> bool { - matches!(kind, "api" | "statements") -} - -/// The `external_key` a push source files a pushed identity under (`api:…`, `statements:…`). -/// The push handlers write it and the read by `external_id` (#1276) looks it up, so the two -/// cannot drift apart. -pub(crate) fn pushed_key(source: &utopia_core::models::Source, identity: &str) -> String { - format!("{}:{identity}", source.kind) -} - /// 查看推送密钥（Editor；列表响应从不携带，查看走这里）。 /// /// 令牌要 write：密钥能往库里推东西，读令牌拿到它就等于绕过了自己的 scope diff --git a/crates/utopia-server/src/api/token_read_tests.rs b/crates/utopia-server/src/api/token_read_tests.rs index 339e7654f..632405ae9 100644 --- a/crates/utopia-server/src/api/token_read_tests.rs +++ b/crates/utopia-server/src/api/token_read_tests.rs @@ -20,6 +20,7 @@ struct Fixture { entity: Uuid, fact: Uuid, doc: Uuid, + search: Arc, _dir: tempfile::TempDir, } @@ -63,7 +64,7 @@ impl Fixture { let search = Arc::new(utopia_search::SearchIndex::open( &dir.path().join("search"), )?); - let state = AppState::new(pool.clone(), &cfg, search, "test-only".into()); + let state = AppState::new(pool.clone(), &cfg, search.clone(), "test-only".into()); let app = super::router(state, &cfg); Ok(Some(Self { pool, @@ -76,6 +77,7 @@ impl Fixture { entity, fact, doc, + search, _dir: dir, })) } @@ -133,6 +135,22 @@ impl Fixture { Ok(self.app.clone().oneshot(request).await?.status()) } + async fn search( + &self, + token: &str, + body: serde_json::Value, + ) -> anyhow::Result { + let request = Request::post(format!("/api/v1/kbs/{}/search", self.kb)) + .header("authorization", format!("Bearer {token}")) + .header("content-type", "application/json") + .body(Body::from(body.to_string()))?; + let response = self.app.clone().oneshot(request).await?; + let status = response.status().as_u16(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX).await?; + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap_or_default(); + Ok(json!({ "status": status, "body": body })) + } + async fn cleanup(self) -> anyhow::Result<()> { sqlx::query("DELETE FROM organizations WHERE id = $1") .bind(self.org) @@ -229,3 +247,158 @@ async fn a_search_past_depth_100_or_with_an_unknown_filters_field_gets_422() -> assert_eq!(statuses, [StatusCode::UNPROCESSABLE_ENTITY; 2]); Ok(()) } + +/// 一个 api 来源推了 `identities` 里的每一篇，都讲 harvest；另一个库有个来源 +async fn seed_pushed_and_foreign_sources( + f: &Fixture, + identities: &[&str], +) -> anyhow::Result<(Uuid, Uuid, Uuid)> { + let (pushed, custom, foreign) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (kb, other_kb) = (f.kb, f.other_kb); + sqlx::raw_sql(&format!( + "INSERT INTO sources(id,kb_id,kind,name) VALUES + ('{pushed}','{kb}','api','ci'), ('{custom}','{kb}','custom','poll'), + ('{foreign}','{other_kb}','api','ci');" + )) + .execute(&f.pool) + .await?; + for identity in identities { + let (doc, chunk) = (Uuid::now_v7(), Uuid::now_v7()); + sqlx::query( + "INSERT INTO documents(id,kb_id,source_id,external_key,filename,sha256) + VALUES($1,$2,$3,$4,$5,$5)", + ) + .bind(doc) + .bind(kb) + .bind(pushed) + .bind(format!("api:{identity}")) + .bind(identity) + .execute(&f.pool) + .await?; + sqlx::query( + "INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES($1,$2,$3,0,'harvest notes')", + ) + .bind(chunk) + .bind(kb) + .bind(doc) + .execute(&f.pool) + .await?; + f.search.reindex_document( + &kb.to_string(), + &doc.to_string(), + &[(chunk.to_string(), "harvest notes".into())], + )?; + } + Ok((pushed, custom, foreign)) +} + +fn found_external_ids(found: &serde_json::Value) -> serde_json::Value { + let mut ids: Vec<_> = found["body"]["results"] + .as_array() + .into_iter() + .flatten() + .map(|hit| hit["external_id"].clone()) + .collect(); + ids.sort_by_key(|id| id.to_string()); + json!({ "status": found["status"], "external_ids": ids }) +} + +#[tokio::test] +async fn a_search_by_source_and_external_id_prefix_docs_finds_only_docs_a_md() -> anyhow::Result<()> +{ + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let (pushed, _, _) = + seed_pushed_and_foreign_sources(&f, &["docs/a.md", "code/b.rs"]).await?; + let token = f.read_token(f.kb).await; + let all = f + .search(&token, json!({ "q": "harvest", "source_id": pushed })) + .await?; + let docs = f + .search( + &token, + json!({ "q": "harvest", "source_id": pushed, "external_id_prefix": "docs/" }), + ) + .await?; + anyhow::Ok((all, docs)) + } + .await; + f.cleanup().await?; + let (all, docs) = result?; + assert_eq!( + found_external_ids(&all), + json!({ "status": 200, "external_ids": ["code/b.rs", "docs/a.md"] }) + ); + assert_eq!( + found_external_ids(&docs), + json!({ "status": 200, "external_ids": ["docs/a.md"] }) + ); + Ok(()) +} + +#[tokio::test] +async fn a_search_by_another_bases_source_gets_404_and_a_prefix_without_a_push_source_gets_422( +) -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let (_, custom, foreign) = seed_pushed_and_foreign_sources(&f, &[]).await?; + let token = f.read_token(f.kb).await; + let mut codes = Vec::new(); + for body in [ + json!({ "q": "harvest", "source_id": foreign }), + json!({ "q": "harvest", "external_id_prefix": "docs/" }), + json!({ "q": "harvest", "source_id": custom, "external_id_prefix": "docs/" }), + ] { + let found = f.search(&token, body).await?; + codes.push(json!([found["status"], found["body"]["code"]])); + } + anyhow::Ok(codes) + } + .await; + f.cleanup().await?; + assert_eq!( + result?, + vec![ + json!([404, null]), + json!([422, "prefix_needs_push_source"]), + json!([422, "prefix_needs_push_source"]), + ] + ); + Ok(()) +} + +#[tokio::test] +async fn an_external_id_prefix_docs_underscore_finds_docs_a_md_but_not_docsxa_md( +) -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let (pushed, _, _) = seed_pushed_and_foreign_sources( + &f, + &["docs_a.md", "docsXa.md", "50%off.md", "50 percent off.md"], + ) + .await?; + let token = f.read_token(f.kb).await; + let mut found = Vec::new(); + for prefix in ["docs_", "50%"] { + let body = json!({ "q": "harvest", "source_id": pushed, "external_id_prefix": prefix }); + found.push(found_external_ids(&f.search(&token, body).await?)); + } + anyhow::Ok(found) + } + .await; + f.cleanup().await?; + assert_eq!( + result?, + [ + json!({ "status": 200, "external_ids": ["docs_a.md"] }), + json!({ "status": 200, "external_ids": ["50%off.md"] }), + ] + ); + Ok(()) +} diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs index df3141134..3c01813d4 100644 --- a/crates/utopia-server/src/main.rs +++ b/crates/utopia-server/src/main.rs @@ -33,6 +33,7 @@ mod owl_import; mod pack_alignment; mod phrase_alignment; mod pipeline; +mod pushed_keys; mod query_engine; mod rdf; mod readers; diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rs index d7252dc00..656a6a2d1 100644 --- a/crates/utopia-server/src/pipeline_tests.rs +++ b/crates/utopia-server/src/pipeline_tests.rs @@ -398,7 +398,10 @@ async fn a_large_statements_payload_is_searchable_without_embedding() -> anyhow: payload ); let chunks = utopia_store::documents::chunks_full(&f.pool, doc).await?; - let hits = f.state.search.search(&f.kb.to_string(), "sensor", 1)?; + let hits = f + .state + .search + .search(&f.kb.to_string(), "sensor", 1, None)?; assert_eq!(hits.len(), 1, "the payload remains searchable by keyword"); assert_eq!(hits[0].chunk_id, chunks[0].id.to_string()); f.cleanup().await diff --git a/crates/utopia-server/src/pushed_keys.rs b/crates/utopia-server/src/pushed_keys.rs new file mode 100644 index 000000000..a4c449d4c --- /dev/null +++ b/crates/utopia-server/src/pushed_keys.rs @@ -0,0 +1,23 @@ +//! 推送来源（`api`、`statements`）的文档怎么按推送方给的 identity 存 `external_key`、 +//! 怎么从它取回来。推送的处理器、按 `external_id` 读回和检索都用这一份（#1271） + +/// 有推送密钥的来源：`api` 推文档，`statements` 推陈述（0054）。两种的密钥同一套 +/// 生成、查看、轮换 +pub fn is_push_kind(kind: &str) -> bool { + matches!(kind, "api" | "statements") +} + +/// The `external_key` a push source files a pushed identity under (`api:…`, `statements:…`). +/// The push handlers write it and the read by `external_id` (#1276) looks it up, so the two +/// cannot drift apart. +pub fn pushed_key(source: &utopia_core::models::Source, identity: &str) -> String { + format!("{}:{identity}", source.kind) +} + +/// `pushed_key` 倒过来：推送来源的文档交回推送方给的 identity，别的来源没有（#1271） +pub fn pushed_identity<'k>(kind: &str, key: &'k str) -> Option<&'k str> { + if !is_push_kind(kind) { + return None; + } + key.strip_prefix(kind)?.strip_prefix(':') +} diff --git a/crates/utopia-server/src/retrieval.rs b/crates/utopia-server/src/retrieval.rs index 4a2938107..eb47147fd 100644 --- a/crates/utopia-server/src/retrieval.rs +++ b/crates/utopia-server/src/retrieval.rs @@ -31,6 +31,14 @@ pub struct SearchOpts { pub top_k: usize, pub offset: usize, pub as_of: Option>, + pub filter: Option, +} + +/// 只检索一个来源的文档，给了 `key_prefix` 就只要 `external_key` 以它开头的（#1271）。 +/// 两路各在召回**之内**过滤：融合完再筛，库里别的命中早占满了每路的名额，页就短了 +pub struct SourceFilter { + pub source_id: Uuid, + pub key_prefix: Option, } /// 一条命中连同它的分数（#1271）。`score` 是 RRF 总分，只在同一次查询里能排先后， @@ -39,6 +47,8 @@ pub struct SearchOpts { pub struct ScoredChunk { #[serde(flatten)] pub chunk: SourcedChunk, + /// 推送来源的文档才有：推送方给的那个 id，好让它拿回去对自己的记录 + pub external_id: Option, pub score: f64, pub bm25_rank: Option, pub vector_rank: Option, @@ -66,8 +76,11 @@ pub async fn hybrid( kb_id, workspace_id, query, - RECALL_PER_CHANNEL, - as_of, + Recall { + per_channel: RECALL_PER_CHANNEL, + as_of, + documents: None, + }, embeddings, ) .await?; @@ -89,13 +102,29 @@ pub async fn hybrid_scored( opts: SearchOpts, embeddings: Option<&mut llm_util::EmbedCache>, ) -> AppResult { + let documents = match &opts.filter { + Some(filter) => Some( + utopia_store::documents::ids_in_source( + &state.pool, + kb_id, + filter.source_id, + filter.key_prefix.as_deref(), + opts.as_of, + ) + .await?, + ), + None => None, + }; let (hits, vector_used) = fused_hits( state, kb_id, workspace_id, query, - MAX_DEPTH, - opts.as_of, + Recall { + per_channel: MAX_DEPTH, + as_of: opts.as_of, + documents: documents.as_deref(), + }, embeddings, ) .await?; @@ -113,35 +142,38 @@ pub async fn hybrid_scored( }) } -/// 两路各取 `per_channel` 条、RRF 融合、按记录轴过滤后的整张表，连同向量那一路跑了没有 +/// 每一路怎么召回：取几条、按哪一刻、只在哪些文档里（None 即全库） +#[derive(Clone, Copy)] +struct Recall<'a> { + per_channel: usize, + as_of: Option>, + documents: Option<&'a [Uuid]>, +} + +/// 两路按 `recall` 各取一份、RRF 融合、按记录轴过滤后的整张表，连同向量那一路跑了没有 async fn fused_hits( state: &AppState, kb_id: Uuid, workspace_id: Uuid, query: &str, - per_channel: usize, - as_of: Option>, + recall: Recall<'_>, embeddings: Option<&mut llm_util::EmbedCache>, ) -> AppResult<(Vec, bool)> { let bm25 = { let search = state.search.clone(); let kb = kb_id.to_string(); let query = query.to_string(); - tokio::task::spawn_blocking(move || search.search(&kb, &query, per_channel)) + let documents: Option> = recall + .documents + .map(|ids| ids.iter().map(Uuid::to_string).collect()); + let limit = recall.per_channel; + tokio::task::spawn_blocking(move || search.search(&kb, &query, limit, documents.as_deref())) }; // `join!` 不是 `try_join!`：向量那一路失败是降级不是错误（见 vector_channel）， // `try_join!` 会把模型故障变成检索报错 let (bm25, vector) = tokio::join!( bm25, - vector_channel( - state, - kb_id, - workspace_id, - query, - per_channel, - as_of, - embeddings - ) + vector_channel(state, kb_id, workspace_id, query, recall, embeddings) ); let bm25 = bm25 .map_err(|e| utopia_core::AppError::Other(anyhow::anyhow!("BM25 检索线程退出：{e}")))? @@ -159,14 +191,22 @@ async fn fused_hits( .filter_map(|f| Some((f.id.parse().ok()?, f))) .collect(); let ids: Vec = fused.iter().map(|(id, _)| *id).collect(); - let live = utopia_store::documents::chunks_by_ids(&state.pool, kb_id, &ids, as_of).await?; + let live = + utopia_store::documents::chunks_by_ids(&state.pool, kb_id, &ids, recall.as_of).await?; let mut fused: HashMap = fused.into_iter().collect(); let hits = live .into_iter() .filter_map(|chunk| { let f = fused.remove(&chunk.chunk.id)?; + let external_id = chunk + .source_kind + .as_deref() + .zip(chunk.external_key.as_deref()) + .and_then(|(kind, key)| crate::pushed_keys::pushed_identity(kind, key)) + .map(str::to_string); Some(ScoredChunk { chunk, + external_id, score: f.score, bm25_rank: position(&f.ranks, 0), vector_rank: position(&f.ranks, 1), @@ -196,8 +236,7 @@ async fn vector_channel( kb_id: Uuid, workspace_id: Uuid, query: &str, - per_channel: usize, - as_of: Option>, + recall: Recall<'_>, cache: Option<&mut llm_util::EmbedCache>, ) -> AppResult>> { let vector = match cache.as_deref().and_then(|c| c.get(query)).cloned() { @@ -228,8 +267,9 @@ async fn vector_channel( &state.pool, kb_id, &vector, - per_channel as i64, - as_of, + recall.per_channel as i64, + recall.as_of, + recall.documents, ) .await?; Ok(Some(ids.into_iter().map(|id| id.to_string()).collect())) diff --git a/crates/utopia-server/src/retrieval_tests.rs b/crates/utopia-server/src/retrieval_tests.rs index 5f7d9da64..24287b867 100644 --- a/crates/utopia-server/src/retrieval_tests.rs +++ b/crates/utopia-server/src/retrieval_tests.rs @@ -194,6 +194,7 @@ async fn both_channels_fuse_with_bm25_first() -> anyhow::Result<()> { top_k: 8, offset: 0, as_of: None, + filter: None, }; let page = super::hybrid_scored(&f.state, f.kb, f.ws, "apple harvest", opts, None).await?; let hits: Vec<_> = page @@ -205,7 +206,7 @@ async fn both_channels_fuse_with_bm25_first() -> anyhow::Result<()> { h.bm25_rank, h.vector_rank, h.chunk.source_id, - h.chunk.external_id.as_deref(), + h.external_id.as_deref(), ) }) .collect(); @@ -270,7 +271,7 @@ async fn historical_search_keeps_live_candidates_before_limiting_results() -> an let raw = f .state .search - .search(&f.kb.to_string(), "apple harvest", 24)?; + .search(&f.kb.to_string(), "apple harvest", 24, None)?; assert_eq!(raw.len(), 7); assert_eq!(raw.last().unwrap().chunk_id, f.b.to_string()); let current = super::hybrid(&f.state, f.kb, f.ws, "apple harvest", 6, None, None).await?; @@ -348,6 +349,7 @@ fn page(offset: usize, top_k: usize) -> super::SearchOpts { top_k, offset, as_of: None, + filter: None, } } @@ -394,3 +396,80 @@ async fn a_page_ending_at_depth_100_of_101_hits_has_no_more() -> anyhow::Result< assert_eq!((last.results.len(), last.has_more), (50, false)); Ok(()) } + +/// A source filter works inside both channels (#1271): a hundred better hits from +/// elsewhere fill each channel's candidates, so filtering after fusion would leave +/// nothing, and the six weak hits of the source must still page like any other list. +#[tokio::test] +async fn a_source_filter_pages_through_its_six_hits_under_a_hundred_better_ones( +) -> anyhow::Result<()> { + let Some(f) = fixture().await? else { + return Ok(()); + }; + let _server = seed_ranked_chunks(&f, 100).await?; + let (source, doc) = (Uuid::now_v7(), Uuid::now_v7()); + sqlx::raw_sql(&format!( + "INSERT INTO sources(id,kb_id,kind,name) VALUES ('{source}','{kb}','api','ci'); + INSERT INTO documents(id,kb_id,source_id,external_key,filename,sha256) + VALUES ('{doc}','{kb}','{source}','api:weak','weak.md','w')", + kb = f.kb + )) + .execute(&f.pool) + .await?; + let mut weak = Vec::new(); + for seq in 0..6 { + let id = Uuid::now_v7(); + let text = format!("apple {}", "pear ".repeat(200 + seq)); + sqlx::query( + "INSERT INTO chunks(id,kb_id,document_id,seq,text,embedding) + VALUES($1,$2,$3,$4,$5,$6::text::vector)", + ) + .bind(id) + .bind(f.kb) + .bind(doc) + .bind(seq as i32) + .bind(&text) + .bind(format!("[{},100,0,0]", seq + 1)) + .execute(&f.pool) + .await?; + weak.push((id.to_string(), text)); + } + f.state + .search + .reindex_document(&f.kb.to_string(), &doc.to_string(), &weak)?; + let walk = |offset, top_k| { + let opts = super::SearchOpts { + filter: Some(super::SourceFilter { + source_id: source, + key_prefix: None, + }), + ..page(offset, top_k) + }; + super::hybrid_scored(&f.state, f.kb, f.ws, "apple", opts, None) + }; + let (first, second, whole) = (walk(0, 3).await, walk(3, 3).await, walk(0, 10).await); + f.cleanup().await?; + let (first, second, whole) = (first?, second?, whole?); + let ids = + |hits: &[super::ScoredChunk]| hits.iter().map(|h| h.chunk.chunk.id).collect::>(); + let mut found: Vec<_> = ids(&whole.results).iter().map(Uuid::to_string).collect(); + found.sort(); + let mut expected: Vec<_> = weak.into_iter().map(|(id, _)| id).collect(); + expected.sort(); + assert_eq!(found, expected); + let channels: Vec<_> = whole + .results + .iter() + .map(|h| (h.bm25_rank.is_some(), h.vector_rank.is_some())) + .collect(); + assert_eq!(channels, [(true, true); 6]); + assert_eq!( + [ids(&first.results), ids(&second.results)].concat(), + ids(&whole.results) + ); + assert_eq!( + (first.has_more, second.has_more, whole.has_more), + (true, false, false) + ); + Ok(()) +} diff --git a/crates/utopia-server/src/search_recovery.rs b/crates/utopia-server/src/search_recovery.rs index 0316457ea..3c87e91cc 100644 --- a/crates/utopia-server/src/search_recovery.rs +++ b/crates/utopia-server/src/search_recovery.rs @@ -109,7 +109,12 @@ mod tests { assert!(marker(&index_dir).exists()); assert!(open(&index_dir).is_err(), "a second writer must be refused"); assert_eq!(search.len(), 2, "the live writer keeps its committed data"); - assert_eq!(search.search(&kb.to_string(), "firstneedle", 10)?.len(), 1); + assert_eq!( + search + .search(&kb.to_string(), "firstneedle", 10, None)? + .len(), + 1 + ); drop(search); let search = open(&index_dir)?; @@ -123,9 +128,14 @@ mod tests { rebuild(&index_dir, &search, live)?; assert_eq!(search.len(), 2); assert!(search - .search(&kb.to_string(), "firstneedle", 10)? + .search(&kb.to_string(), "firstneedle", 10, None)? .is_empty()); - assert_eq!(search.search(&kb.to_string(), "thirdneedle", 10)?.len(), 1); + assert_eq!( + search + .search(&kb.to_string(), "thirdneedle", 10, None)? + .len(), + 1 + ); assert!(!marker(&index_dir).exists()); } Ok(()) @@ -154,7 +164,12 @@ mod tests { drop(search); let search = open(&index_dir)?; assert_eq!(search.len(), 2); - assert_eq!(search.search(&kb.to_string(), "legacyneedle", 10)?.len(), 1); + assert_eq!( + search + .search(&kb.to_string(), "legacyneedle", 10, None)? + .len(), + 1 + ); // The database may become empty between attempts. Completing that // recovery must clear its marker instead of rebuilding on every boot. diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs index 6f1e30fe4..f974577d0 100644 --- a/crates/utopia-store/src/documents.rs +++ b/crates/utopia-store/src/documents.rs @@ -1890,12 +1890,16 @@ async fn set_embeddings_for_input( /// 次序在外层再排一遍（`vector_index::RESORT`）：`relaxed_order` 下索引给的次序 /// 只是大致按距离，而距离并列时精确路径和 HNSW 各排各的——同一问两种计划回的 /// 居首不同（#652）。并列由 id 定 +/// +/// `documents`：只在这些文档的块里找。条件在近邻 CTE 里面，`LIMIT` 数的就只是 +/// 允许的块；放到外层，库里别的块会先占满名额 pub async fn vector_search( pool: &PgPool, kb_id: Uuid, embedding: &[f32], limit: i64, as_of: Option>, + documents: Option<&[Uuid]>, ) -> AppResult> { let dims = embedding.len(); if dims == 0 { @@ -1908,12 +1912,17 @@ pub async fn vector_search( "WITH nearest AS MATERIALIZED ( SELECT id, {distance} AS distance FROM chunks c WHERE c.kb_id = $1 AND c.embedding IS NOT NULL AND {live} - AND {same_dims} + AND {same_dims}{only} ORDER BY {distance} LIMIT $3 ) SELECT id FROM nearest ORDER BY {resort}", live = crate::record_axis::chunk_live_at("c", as_of.map(|_| 4)), + only = if documents.is_some() { + " AND c.document_id = ANY($5)" + } else { + "" + }, same_dims = crate::vector_index::same_dims("c.embedding", dims), distance = crate::vector_index::distance("c.embedding", 2, dims), resort = crate::vector_index::RESORT, @@ -1922,12 +1931,37 @@ pub async fn vector_search( .bind(&query_vec) .bind(limit) .bind(as_of) + .bind(documents) .fetch_all(&mut *tx) .await?; tx.commit().await?; Ok(rows.into_iter().map(|(id,)| id).collect()) } +/// 检索按来源过滤时允许的文档：库里这个来源当时活着的文档，给了 `key_prefix` 就只要 +/// `external_key` 以它开头的。比较用 `starts_with`：路径里的 `_` 和 `%` 进了 LIKE 就是通配符 +pub async fn ids_in_source( + pool: &PgPool, + kb_id: Uuid, + source_id: Uuid, + key_prefix: Option<&str>, + as_of: Option>, +) -> AppResult> { + let rows: Vec<(Uuid,)> = sqlx::query_as(&format!( + "SELECT d.id FROM documents d + WHERE d.kb_id = $1 AND d.source_id = $2 AND {doc_live} + AND ($3::text IS NULL OR starts_with(d.external_key, $3))", + doc_live = crate::record_axis::document_live_at("d", as_of.map(|_| 4)), + )) + .bind(kb_id) + .bind(source_id) + .bind(key_prefix) + .bind(as_of) + .fetch_all(pool) + .await?; + Ok(rows.into_iter().map(|(id,)| id).collect()) +} + /// 按 id 集合取分块（带文档名与来源），保持传入顺序。 /// /// `as_of`：记录轴（0019）——那一刻还活着的块、还没被删的文档。文档的删除 @@ -1940,8 +1974,7 @@ pub async fn chunks_by_ids( ) -> AppResult> { let rows: Vec = sqlx::query_as(&format!( "SELECT c.id, c.document_id, c.seq, c.text, d.filename, d.source_id, - CASE WHEN s.kind IN ('api', 'statements') - THEN substr(d.external_key, length(s.kind) + 2) END AS external_id + s.kind AS source_kind, d.external_key FROM chunks c JOIN documents d ON d.id = c.document_id LEFT JOIN sources s ON s.id = d.source_id WHERE c.kb_id = $1 AND c.id = ANY($2) diff --git a/crates/utopia-store/tests/store/a_search_reads_the_base_as_it_was.rs b/crates/utopia-store/tests/store/a_search_reads_the_base_as_it_was.rs index 6baccb786..cf831ac42 100644 --- a/crates/utopia-store/tests/store/a_search_reads_the_base_as_it_was.rs +++ b/crates/utopia-store/tests/store/a_search_reads_the_base_as_it_was.rs @@ -158,13 +158,15 @@ async fn a_superseded_chunk_keeps_its_vector_and_answers_as_of_then() -> anyhow: // 2. 现在：向量召回给出新块，给不出被顶掉的旧块 let query = vec![1.0_f32, 0.0, 0.0]; - let now_hits = utopia_store::documents::vector_search(&pool, f.kb, &query, 10, None).await?; + let now_hits = + utopia_store::documents::vector_search(&pool, f.kb, &query, 10, None, None).await?; assert!(!now_hits.contains(&f.old_chunk), "现行检索不该翻出旧版本"); assert!(now_hits.contains(&f.new_chunk)); // 3. 三月：那时旧块是现行的，新块还不存在 let then = Some(t("2026-03-01T00:00:00Z")); - let then_hits = utopia_store::documents::vector_search(&pool, f.kb, &query, 10, then).await?; + let then_hits = + utopia_store::documents::vector_search(&pool, f.kb, &query, 10, then, None).await?; assert!( then_hits.contains(&f.old_chunk), "三月该搜得到当时那一版——这一条在丢向量的实现上不可能成立" diff --git a/crates/utopia-store/tests/store/the_nearest_chunk_is_found_however_it_is_reached.rs b/crates/utopia-store/tests/store/the_nearest_chunk_is_found_however_it_is_reached.rs index 8cf1de506..9027893c0 100644 --- a/crates/utopia-store/tests/store/the_nearest_chunk_is_found_however_it_is_reached.rs +++ b/crates/utopia-store/tests/store/the_nearest_chunk_is_found_however_it_is_reached.rs @@ -192,11 +192,18 @@ struct Answers { async fn ask(pool: &PgPool, f: &Fixture) -> anyhow::Result { use utopia_store::documents::vector_search; - let big_now = vector_search(pool, f.big, &query(), 10, None).await?; - let small_now = vector_search(pool, f.small, &query(), 10, None).await?; - let five = vector_search(pool, f.big, &[1.0, 0.0, 0.0, 0.0, 0.0], 10, None).await?; - let big_then = - vector_search(pool, f.big, &query(), 10, Some(t("2026-03-01T00:00:00Z"))).await?; + let big_now = vector_search(pool, f.big, &query(), 10, None, None).await?; + let small_now = vector_search(pool, f.small, &query(), 10, None, None).await?; + let five = vector_search(pool, f.big, &[1.0, 0.0, 0.0, 0.0, 0.0], 10, None, None).await?; + let big_then = vector_search( + pool, + f.big, + &query(), + 10, + Some(t("2026-03-01T00:00:00Z")), + None, + ) + .await?; Ok(Answers { nearest_in_big: big_now.first().copied(), small_limit_10: small_now, @@ -249,6 +256,7 @@ async fn the_answers_agree_with_and_without_the_index() -> anyhow::Result<()> { &query(), 2, Some(t("2026-03-01T00:00:00Z")), + None, ) .await?; assert!( diff --git a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md index 9825b075f..76aeb4e63 100644 --- a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md +++ b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md @@ -123,6 +123,11 @@ push documents in. Chunk search (`POST /kbs/{id}/search`, added 2026-10-24, #1271) takes a token on the same terms, so the service can ask the base about what it mirrored. Each hit carries its score, its BM25 and vector ranks and its source; `offset` pages through one fused list, at most 100 deep. + `source_id` narrows the search to one source of the base, and a source outside the base answers + 404 like a missing one. `external_id_prefix` narrows it further to a push source's documents + whose `external_id` starts with the prefix, taken literally; without the `source_id` of an + `api` or `statements` source it answers 422 `prefix_needs_push_source`. Both filters apply + inside each channel, before fusion, so other documents cannot fill the candidates first. ## Dead ends @@ -152,7 +157,8 @@ push documents in. and document content" now also counts them. 2026-10-19: the graph overview and neighbourhood join them, and so does reading a pushed document back by its `external_id` (#1276). The ontology and a class's instances follow (#1265), then a document's extractions and the - base's export (#1266). 2026-10-24: chunk search joins them (#1271). + base's export (#1266). 2026-10-24: chunk search joins them (#1271). 2026-10-25: it can be + narrowed to one source and, for a push source, to an `external_id` prefix (#1271). ## Open questions diff --git a/docs/design/chat-and-mcp.md b/docs/design/chat-and-mcp.md index eaa3fbe57..12acc42f7 100644 --- a/docs/design/chat-and-mcp.md +++ b/docs/design/chat-and-mcp.md @@ -79,7 +79,10 @@ RRF score ranks hits within one query only; it is not comparable across queries page fuses 100 candidates per channel where chat fuses 24, so the search page can rank the same query differently from chat and from before #1271. Pages join up only while the base is unchanged between requests, and each channel counts its ranks before the final `as_of` filter, so a -historical page can skip rank numbers. +historical page can skip rank numbers. A `source_id` of the base (another base's source is a 404) +and, for a push source, an `external_id_prefix` limit both channels to that source's documents +before fusion, so the base's other hits cannot take the candidates a filter after fusion would +lose; the prefix compares against the key `pushed_key` builds (#1271). **`remember`** writes a memory document at once; its statements go through open extraction and wait in `pending_facts`; the assistant says the sentence is recorded and its statements will be shown diff --git a/web/src/docs/ingest.md b/web/src/docs/ingest.md index b577f0251..df3a8fa2b 100644 --- a/web/src/docs/ingest.md +++ b/web/src/docs/ingest.md @@ -211,6 +211,8 @@ Content-Type: application/json `q` is required. `top_k` defaults to 10 and is capped at 50; `offset` defaults to 0. `as_of` (an RFC 3339 instant) reads the base as it was then; ranks are counted before `as_of` filters the hits. Any other field is a `422`. +To search one source, add its `source_id`; a source of another base is a `404`. To search part of what you pushed, add `external_id_prefix` as well, for example `"docs/"`: only documents whose `external_id` starts with it are searched, and `_` or `%` in it are plain characters. The prefix needs the `source_id` of an API or Statements source, otherwise it is a `422` (`prefix_needs_push_source`). For one exact `external_id`, read the document back as above. Both filters apply inside each channel before the hits are fused, so paging works the same with them as without. + ```json { "results": [ From 5e5b30af43d9b8b4943c1349f63cc2520950f3f0 Mon Sep 17 00:00:00 2001 From: Daniel <118035379+Floating-Y@users.noreply.github.com> Date: Mon, 26 Oct 2026 13:12:47 -0700 Subject: [PATCH 05/27] Saving one knowledge base settings card keeps unsaved edits in the other cards (#1452) Signed-off-by: Floating-Y <118035379+Floating-Y@users.noreply.github.com> Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- web/src/pages/KbSettings.test.ts | 220 +++++++++++++++++++++++++++++++ web/src/pages/KbSettings.tsx | 24 +++- 2 files changed, 243 insertions(+), 1 deletion(-) create mode 100644 web/src/pages/KbSettings.test.ts diff --git a/web/src/pages/KbSettings.test.ts b/web/src/pages/KbSettings.test.ts new file mode 100644 index 000000000..14acde6a1 --- /dev/null +++ b/web/src/pages/KbSettings.test.ts @@ -0,0 +1,220 @@ +import { + Children, isValidElement, useState, type ChangeEvent, type ComponentProps, + type JSXElementConstructor, type ReactNode, +} from "react"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { api, type Kb } from "../api"; +import { S } from "../i18n"; +import { Button, Checkbox, Input, Segmented, SettingsCard } from "../ui"; +import { KbSettings } from "./KbSettings"; + +// Node 中执行真实组件与保存回调；查询快照模拟刷新边界，不覆盖 TanStack 缓存集成。 +const hooks = vi.hoisted(() => ({ + kbId: "kb-a", + data: undefined as Kb | undefined, + query: undefined as { queryFn: () => Promise } | undefined, + invalidate: vi.fn(), + states: [] as unknown[], + dependencies: [] as Array, + effects: [] as Array<() => void>, + stateIndex: 0, + effectIndex: 0, + changed: false, + saving: undefined as Promise | undefined, +})); +vi.mock("react", async (original) => ({ + ...await original(), + useState: (initial: unknown) => { + const index = hooks.stateIndex++; + if (!(index in hooks.states)) hooks.states[index] = initial; + return [hooks.states[index], (value: unknown) => { + hooks.changed ||= !Object.is(hooks.states[index], value); + hooks.states[index] = value; + }]; + }, + useEffect: (effect: () => void, dependencies: readonly unknown[]) => { + const previous = hooks.dependencies[hooks.effectIndex]; + hooks.dependencies[hooks.effectIndex++] = dependencies; + if (!previous || dependencies.some((value, index) => !Object.is(value, previous[index]))) { + hooks.effects.push(effect); + } + }, +})); +vi.mock("@tanstack/react-router", () => ({ + useParams: () => ({ kbId: hooks.kbId }), + useNavigate: () => vi.fn(), +})); +vi.mock("@tanstack/react-query", () => ({ + useQueryClient: () => ({ invalidateQueries: hooks.invalidate }), + useQuery: (query: { queryKey: string[]; queryFn: () => Promise }) => { + if (query.queryKey[0] !== "kbOne") return { data: { failed: 0 } }; + hooks.query = query; + return { data: hooks.data, isPending: !hooks.data }; + }, + useMutation: (options: { + mutationFn: (patch: Record) => Promise; + onSuccess: (result: unknown) => void; + }) => { + const [status, setStatus] = useState({ isPending: false, isSuccess: false }); + return { + ...status, + mutate: (patch: Record) => { + setStatus({ isPending: true, isSuccess: false }); + hooks.saving = options.mutationFn(patch).then((result) => { + options.onSuccess(result); + setStatus({ isPending: false, isSuccess: true }); + }); + }, + }; + }, +})); + +const initial: Kb = { + id: "kb-a", workspace_id: "workspace", name: "Research", kind: "general", + description: "Original description", visibility: "open", is_default: false, + auto_extend_ontology: false, materialize_inferences: false, + auto_type_resolution: false, governance: false, governance_since: null, + inference_interval_minutes: 60, last_inference_at: null, ontology_lang: "en", +}; +let server: Kb; + +beforeEach(() => { + server = { ...initial }; + Object.assign(hooks, { + kbId: initial.id, data: undefined, query: undefined, + states: [], dependencies: [], effects: [], saving: undefined, + }); + hooks.invalidate.mockReset(); + vi.spyOn(api, "kbDetail").mockImplementation(async () => ({ ...server })); + vi.spyOn(api, "updateKb").mockImplementation(async (_id, patch) => { + server = { ...server, ...patch }; + return server; + }); +}); +afterEach(() => vi.restoreAllMocks()); + +async function refresh() { + render(); + expect(hooks.query).toBeDefined(); + hooks.data = await hooks.query!.queryFn(); +} + +function render(): ReactNode { + hooks.stateIndex = hooks.effectIndex = 0; + hooks.changed = false; + const tree = KbSettings(); + for (const effect of hooks.effects.splice(0)) effect(); + return hooks.changed ? render() : tree; +} + +function find

(node: ReactNode, type: JSXElementConstructor

| string): P[] { + if (!isValidElement<{ children?: ReactNode; action?: ReactNode }>(node)) return []; + // JSX 的 type 与组件一致时，props 就是该组件的公开属性。 + const matched = node.type === type ? [node.props as P] : []; + return [...matched, ...Children.toArray([node.props.children, node.props.action]) + .flatMap((child) => find(child, type))]; +} + +function card(title: string) { + const found = find(render(), SettingsCard).find((item) => item.title === title); + expect(found, title).toBeDefined(); + return found!; +} + +function editIdentity(name: string, description: string) { + const inputs = find(card(S.kbset.cardIdentity).children, Input); + for (const [index, value] of [name, description].entries()) { + inputs[index].onChange!({ target: { value } } as ChangeEvent); + } +} + +async function save(title: string) { + const button = find(card(title).action, Button)[0]; + expect(button.disabled).toBe(false); + button.onClick!({} as Parameters["onClick"]>>[0]); + await hooks.saving; + expect(hooks.invalidate.mock.calls).toEqual([ + [{ queryKey: ["kbOne", hooks.kbId] }], [{ queryKey: ["kbs"] }], + ]); + await refresh(); +} + +function expectSaved(title: string) { + const action = card(title).action; + expect(find(action, Button)[0].disabled).toBe(true); + expect(find<{ children?: ReactNode }>(action, "span") + .some((span) => span.children === S.kbset.saved)).toBe(true); +} + +it.each(["Original description", "", null])( + "keeps identity drafts when only visibility is saved (description: %s)", + async (description) => { + server.description = description; + await refresh(); + editIdentity("Draft name", "Draft description"); + find(card(S.settings.kbs.visibility).children, Segmented)[0].onChange("restricted"); + await save(S.settings.kbs.visibility); + + expect(api.updateKb).toHaveBeenCalledExactlyOnceWith("kb-a", { visibility: "restricted" }); + expect(api.kbDetail).toHaveBeenCalledTimes(2); + expect(find(card(S.kbset.cardIdentity).children, Input).map((input) => input.value)) + .toEqual(["Draft name", "Draft description"]); + expect(find(card(S.settings.kbs.visibility).children, Segmented)[0].value).toBe("restricted"); + expectSaved(S.settings.kbs.visibility); + }, +); + +it("synchronizes saved identity values and status while preserving other cards' drafts", async () => { + await refresh(); + editIdentity(" New name ", " New description "); + find(card(S.settings.kbs.visibility).children, Segmented)[0].onChange("restricted"); + find(card(S.kbset.cardAutomation).children, Checkbox)[0].onChange!(true); + find(card(S.kbset.ontologyLang).children, Segmented)[0].onChange("zh"); + await save(S.kbset.cardIdentity); + + expect(api.updateKb).toHaveBeenLastCalledWith("kb-a", { name: "New name", description: "New description" }); + expect(find(card(S.kbset.cardIdentity).children, Input).map((input) => input.value)) + .toEqual(["New name", "New description"]); + expectSaved(S.kbset.cardIdentity); + expect(find(card(S.settings.kbs.visibility).children, Segmented)[0].value).toBe("restricted"); + expect(find(card(S.kbset.cardAutomation).children, Checkbox)[0].checked).toBe(true); + expect(find(card(S.kbset.ontologyLang).children, Segmented)[0].value).toBe("zh"); +}); + +it("accepts server updates on unedited cards", async () => { + await refresh(); + render(); + server = { + ...server, name: "Renamed remotely", description: "Updated remotely", visibility: "restricted", + materialize_inferences: true, auto_type_resolution: true, governance: true, + inference_interval_minutes: 120, ontology_lang: "zh", + }; + await refresh(); + expect(find(card(S.kbset.cardIdentity).children, Input).map((input) => input.value)) + .toEqual([server.name, server.description]); + expect(find(card(S.settings.kbs.visibility).children, Segmented)[0].value).toBe("restricted"); + expect(find(card(S.kbset.cardAutomation).children, Checkbox).map((input) => input.checked)) + .toEqual([true, true, true]); + expect(find(card(S.kbset.cardAutomation).children, Input)[0].value).toBe(120); + expect(find(card(S.kbset.ontologyLang).children, Segmented)[0].value).toBe("zh"); +}); + +it.each([false, true])("resets drafts when switching KB with identical server values (preloaded: %s)", async (preloaded) => { + await refresh(); + editIdentity("Draft name", "Draft description"); + find(card(S.settings.kbs.visibility).children, Segmented)[0].onChange("restricted"); + find(card(S.kbset.cardAutomation).children, Checkbox)[0].onChange!(true); + find(card(S.kbset.ontologyLang).children, Segmented)[0].onChange("zh"); + render(); + hooks.kbId = "kb-b"; + server = { ...initial, id: "kb-b" }; + hooks.data = preloaded ? { ...server } : undefined; + render(); + await refresh(); + expect(find(card(S.kbset.cardIdentity).children, Input).map((input) => input.value)) + .toEqual([initial.name, initial.description]); + expect(find(card(S.settings.kbs.visibility).children, Segmented)[0].value).toBe("open"); + expect(find(card(S.kbset.cardAutomation).children, Checkbox).map((input) => input.checked)) + .toEqual([false, false, false]); + expect(find(card(S.kbset.ontologyLang).children, Segmented)[0].value).toBe("en"); +}); diff --git a/web/src/pages/KbSettings.tsx b/web/src/pages/KbSettings.tsx index 916652d45..8a30f5aad 100644 --- a/web/src/pages/KbSettings.tsx +++ b/web/src/pages/KbSettings.tsx @@ -125,18 +125,40 @@ export function KbSettings() { const [confirmingDelete, setConfirmingDelete] = useState(false); const [error, setError] = useState(null); + // 每张卡只跟随自己的服务端字段；保存别的卡触发重取时，保留这张卡的草稿。 useEffect(() => { if (kb.data) { setName(kb.data.name); setDesc(kb.data.description ?? ""); + } + }, [kbId, kb.data?.name, kb.data?.description]); + + useEffect(() => { + if (kb.data) { setVisibility(kb.data.visibility); + } + }, [kbId, kb.data?.visibility]); + + useEffect(() => { + if (kb.data) { setMaterialize(kb.data.materialize_inferences); setAutoResolve(kb.data.auto_type_resolution); setGovernance(kb.data.governance); setInferMins(kb.data.inference_interval_minutes); + } + }, [ + kbId, + kb.data?.materialize_inferences, + kb.data?.auto_type_resolution, + kb.data?.governance, + kb.data?.inference_interval_minutes, + ]); + + useEffect(() => { + if (kb.data) { setOntoLang(kb.data.ontology_lang); } - }, [kb.data]); + }, [kbId, kb.data?.ontology_lang]); // 四张卡，四个保存：改名字那一下不该把下面的开关一起送上去 const saveIdentity = useKbPatch(kbId!, setError); From 475bb6701d161356fc93b72f8fce7768eae5cac8 Mon Sep 17 00:00:00 2001 From: 0xvectr <57480598+0xvectr@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:26:56 +0800 Subject: [PATCH 06/27] A model card's Test button tests only that card (#1447) * A model card's Test button tests only that card Signed-off-by: 0xvectr <57480598+0xvectr@users.noreply.github.com> * Saving or editing a model card resets only that card's test Signed-off-by: 0xvectr <57480598+0xvectr@users.noreply.github.com> --------- Signed-off-by: 0xvectr <57480598+0xvectr@users.noreply.github.com> Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../utopia-server/src/api/settings_routes.rs | 151 +++++++++++----- web/src/api.ts | 9 +- web/src/pages/Settings.test.ts | 17 +- web/src/pages/Settings.tsx | 162 +++++++++--------- 4 files changed, 214 insertions(+), 125 deletions(-) diff --git a/crates/utopia-server/src/api/settings_routes.rs b/crates/utopia-server/src/api/settings_routes.rs index ff265d9ae..c1ca3834f 100644 --- a/crates/utopia-server/src/api/settings_routes.rs +++ b/crates/utopia-server/src/api/settings_routes.rs @@ -1,8 +1,8 @@ -use axum::extract::{Path, State}; +use axum::extract::{Path, Query, State}; use axum::Json; use serde::Deserialize; use serde_json::json; -use utopia_core::models::Role; +use utopia_core::models::{LlmSettings, Role}; use utopia_llm::ChatMessage; use uuid::Uuid; @@ -322,59 +322,86 @@ fn check(r: &anyhow::Result<()>) -> serde_json::Value { } } +/// 设置页的四张模型卡，各有一个「测试」 +#[derive(Deserialize, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum ModelCard { + Chat, + Embed, + Ocr, + Transcribe, +} + +#[derive(Deserialize)] +pub struct TestQuery { + /// 只测这一张卡。缺席 = 四张都测 + pub card: Option, +} + /// 连通性测试：对话按任务的三种请求形状各试一次（见 [`chat_checks`]）；embedding 试算一条并 -/// 返回维度；版面识别服务问一声健康。 +/// 返回维度；版面识别服务问一声健康。带 `?card=`（[`ModelCard`]）只测那一张卡，回来的也只有 +/// 那一项；不带就四张都测（#1424）。 pub async fn test( State(state): State, AuthUser(user): AuthUser, Path(workspace_id): Path, + Query(q): Query, ) -> ApiResult> { utopia_store::workspaces::require_role(&state.pool, user.id, workspace_id, Role::Admin).await?; - let Some(s) = utopia_store::settings::get(&state.pool, workspace_id).await? else { - return Ok(Json( - json!({ "chat": { "ok": false, "error": "Not configured" }, - "embed": { "ok": false, "error": "Not configured" }, - "ocr": { "ok": false, "error": "Not configured" }, - "transcribe": { "ok": false, "error": "Not configured" } }), - )); - }; + let s = utopia_store::settings::get(&state.pool, workspace_id).await?; + Ok(Json(probe(s.as_ref(), q.card).await)) +} - let chat_result = match llm_util::chat_probe_client(&s) { - None => json!({ "ok": false, "error": "Not configured" }), - Some(client) => chat_checks(&client).await, - }; +/// 只测点了的那张卡（#1424）。从前四张卡共用一次测试，按任意一张四套一起测；自 #1363 起 +/// 对话一测要发三问，测嵌入、读扫描件的卡也跟着白花这几问的 token。回来的只有测了的那几项 +async fn probe(s: Option<&LlmSettings>, only: Option) -> serde_json::Value { + let wants = |card| only.is_none_or(|c| c == card); + let not_configured = || json!({ "ok": false, "error": "Not configured" }); + let mut out = serde_json::Map::new(); - let embed_result = match llm_util::embed_client(&s) { - None => json!({ "ok": false, "error": "Not configured" }), - Some(client) => match client.embed(&["connectivity test".to_string()]).await { - Ok(v) if !v.is_empty() => json!({ "ok": true, "dim": v[0].len() }), - Ok(_) => json!({ "ok": false, "error": "Empty response" }), - Err(e) => json!({ "ok": false, "error": e.to_string() }), - }, - }; + if wants(ModelCard::Chat) { + let chat_result = match s.and_then(llm_util::chat_probe_client) { + None => not_configured(), + Some(client) => chat_checks(&client).await, + }; + out.insert("chat".into(), chat_result); + } - let ocr_result = match crate::readers::Ocr::from_settings(&s) { - None => json!({ "ok": false, "error": "Not configured" }), - Some(ocr) => match ocr.health().await { - Ok(h) => json!({ "ok": true, "version": h["version"] }), - Err(e) => json!({ "ok": false, "error": format!("{e:#}") }), - }, - }; + if wants(ModelCard::Embed) { + let embed_result = match s.and_then(llm_util::embed_client) { + None => not_configured(), + Some(client) => match client.embed(&["connectivity test".to_string()]).await { + Ok(v) if !v.is_empty() => json!({ "ok": true, "dim": v[0].len() }), + Ok(_) => json!({ "ok": false, "error": "Empty response" }), + Err(e) => json!({ "ok": false, "error": e.to_string() }), + }, + }; + out.insert("embed".into(), embed_result); + } - let transcribe_result = match crate::readers::Transcriber::from_settings(&s) { - None => json!({ "ok": false, "error": "Not configured" }), - Some(t) => match t.check().await { - Ok(()) => json!({ "ok": true }), - Err(e) => json!({ "ok": false, "error": format!("{e:#}") }), - }, - }; + if wants(ModelCard::Ocr) { + let ocr_result = match s.and_then(crate::readers::Ocr::from_settings) { + None => not_configured(), + Some(ocr) => match ocr.health().await { + Ok(h) => json!({ "ok": true, "version": h["version"] }), + Err(e) => json!({ "ok": false, "error": format!("{e:#}") }), + }, + }; + out.insert("ocr".into(), ocr_result); + } - Ok(Json(json!({ - "chat": chat_result, - "embed": embed_result, - "ocr": ocr_result, - "transcribe": transcribe_result, - }))) + if wants(ModelCard::Transcribe) { + let transcribe_result = match s.and_then(crate::readers::Transcriber::from_settings) { + None => not_configured(), + Some(t) => match t.check().await { + Ok(()) => json!({ "ok": true }), + Err(e) => json!({ "ok": false, "error": format!("{e:#}") }), + }, + }; + out.insert("transcribe".into(), transcribe_result); + } + + out.into() } #[cfg(test)] @@ -514,4 +541,42 @@ mod tests { "the streamed and tool shapes are not sent when the plain one fails" ); } + + /// #1424：按嵌入卡的「测试」，对话模型一问都收不到；不指定卡的旧调用照旧四项都回 + #[tokio::test] + async fn testing_one_card_sends_nothing_to_the_others() { + let chat = endpoint(false).await; + let s = utopia_core::models::LlmSettings { + workspace_id: uuid::Uuid::nil(), + chat_base_url: Some(chat.uri()), + chat_api_key: None, + chat_model: Some("scripted".into()), + embed_base_url: None, + embed_api_key: None, + embed_model: None, + embed_dim: None, + updated_at: chrono::Utc::now(), + ocr_base_url: None, + ocr_api_key: None, + ocr_backend: None, + ocr_provider: "mineru".into(), + ocr_model: None, + transcribe_base_url: None, + transcribe_api_key: None, + transcribe_model: None, + chat_reasoning_effort: None, + }; + + let out = super::probe(Some(&s), Some(super::ModelCard::Embed)).await; + assert_eq!( + out, + serde_json::json!({ "embed": { "ok": false, "error": "Not configured" } }) + ); + assert!(bodies(&chat).await.is_empty()); + + let all = super::probe(Some(&s), None).await; + let cards: Vec<_> = all.as_object().unwrap().keys().collect(); + assert_eq!(cards, ["chat", "embed", "ocr", "transcribe"], "{all}"); + assert_eq!(bodies(&chat).await.len(), 3); + } } diff --git a/web/src/api.ts b/web/src/api.ts index cd21e34c7..be54834d8 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -2808,19 +2808,20 @@ export const api = { }, ), - testSettings: (workspaceId: string) => + /** 只测 `card` 那一张卡，回来的也只有那一项（#1424） */ + testSettings: (workspaceId: string, card: "chat" | "embed" | "ocr" | "transcribe") => request<{ /** `ok` only when all three request shapes the jobs send pass; `checks` has each one (#1354) */ - chat: { + chat?: { ok: boolean; reply?: string; error?: string; checks?: Record<"plain" | "streaming" | "tools", { ok: boolean; timed_out?: boolean; error?: string }>; }; - embed: { ok: boolean; dim?: number; error?: string }; + embed?: { ok: boolean; dim?: number; error?: string }; ocr?: { ok: boolean; version?: string | null; error?: string }; transcribe?: { ok: boolean; error?: string }; - }>(`/api/v1/workspaces/${workspaceId}/settings/test`, { method: "POST" }), + }>(`/api/v1/workspaces/${workspaceId}/settings/test?card=${card}`, { method: "POST" }), /** 读扫描件的服务、转写模型各自一个保存：存它们不碰对话与嵌入那几列。 * `requeued`：因为缺它而等着的文件，这一存重新排进了处理队列几份 */ saveOcrSettings: ( diff --git a/web/src/pages/Settings.test.ts b/web/src/pages/Settings.test.ts index 0e73b4fbc..4f0949052 100644 --- a/web/src/pages/Settings.test.ts +++ b/web/src/pages/Settings.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest"; import { S } from "../i18n"; -import { chatTestMessage, modelCardStatus } from "./Settings"; +import { cardsOf, chatTestMessage, modelCardStatus } from "./Settings"; describe("model card status", () => { it("prefers a fresh test result over a stale save confirmation", () => { @@ -102,3 +102,18 @@ describe("chat test message", () => { expect(chatTestMessage({ ok: false, error: "Not configured" })).toBe("Not configured"); }); }); + +describe("cards an edit touches", () => { + it("is only the card whose field changed", () => { + // #1424：改读扫描件那张卡，对话卡刚测出的结论还作数 + expect(cardsOf(["ocr_base_url"])).toEqual(["ocr"]); + expect(cardsOf(["transcribe_api_key"])).toEqual(["transcribe"]); + }); + + it("is chat and embed for a preset", () => { + expect(cardsOf(["chat_base_url", "chat_model", "embed_base_url", "embed_model"])).toEqual([ + "chat", + "embed", + ]); + }); +}); diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index 466e2edce..d2e39e4ca 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -38,10 +38,10 @@ import { SsoAdmin } from "./Sso"; * 聊天卡的报错却是危险色。于是"Saved" 说的是哪次操作、这次测的又是哪张卡， * 全得靠猜。 * - * 三条规矩，显示层只管最后一条，前两条在调用处用 reset 保证： - * 1. 改任何一格、按任何一次保存，旧结论（测试结果、Saved、报错）全部过期； - * 2. "Testing…" 只亮在按下的那张卡上（`testCard`），因为测的虽然是两套， - * 人按的是这一张； + * 三条规矩，显示层只管最后一条，前两条在调用处保证： + * 1. 改任何一格、按任何一次保存，旧的 Saved、报错全部过期；测试结果只过期改了的那张卡的 + * （见 `cardsOf`），别的卡配置没动，测出的结论还作数（#1424）； + * 2. 一张卡一个测试（#1424）：测的、"Testing…"、测出的结论都只属于按下的那张卡； * 3. 新测到的结果（含测挂了）盖掉旧保存态——刚按了"测试"，卡上就该说测试的事。 * * 传纯数据、回纯结论（含用哪个色调），字串仍在调用处配 i18n； @@ -55,6 +55,15 @@ export type ModelCardStatus = | { kind: "saved" } | { kind: "idle" }; +export type ModelCard = "chat" | "embed" | "ocr" | "transcribe"; +/** 改了这几格，是哪几张卡的配置变了。字段名的前缀就是卡：chat_ / embed_ / ocr_ / transcribe_； + * 预设一次填对话与嵌入两张卡的格，两张都算（#1424） */ +export function cardsOf(fields: readonly string[]): ModelCard[] { + return (["chat", "embed", "ocr", "transcribe"] as const).filter((c) => + fields.some((f) => f.startsWith(`${c}_`)), + ); +} + /** 对话卡的测试结论：三种请求形状各一行，被拒的那一行带上端点的原话，于是一眼看得出 * 是哪个参数不收（#1354）。旧服务端不回 `checks` 时退回一整句 */ export function chatTestMessage(chat: { @@ -1027,16 +1036,21 @@ export function Settings() { saveTranscribe.reset(); }; - const test = useMutation({ - mutationFn: () => api.testSettings(workspace!.id), - // 测完谁按的就清掉：pending 的字样只在飞行中属于那张卡 - onSettled: () => setTestCard(null), - }); - /* 哪张卡按下的"测试"。测一次是两套一起测（一个接口），结果各自回卡； - 但两个按钮共用这一个 mutation，从前按任意一张两张一起转"Testing…"（#698） */ - const [testCard, setTestCard] = useState<"chat" | "embed" | "ocr" | "transcribe" | null>( - null, - ); + /* 四张卡各测各的，一张一个 mutation（#1424）：从前四张共用一次测试，按任意一张四套一起测， + 还得另记一个"是哪张按的"才能只让那一张转"Testing…"（#698）。分开以后测一张， + 别的卡上还作数的结论不受影响 */ + const useCardTest = (card: ModelCard) => + // eslint-disable-next-line react-hooks/rules-of-hooks + useMutation({ mutationFn: () => api.testSettings(workspace!.id, card) }); + const tests = { + chat: useCardTest("chat"), + embed: useCardTest("embed"), + ocr: useCardTest("ocr"), + transcribe: useCardTest("transcribe"), + }; + /* 改了哪几格，就只让那几张卡的测试结论过期 */ + const expireTests = (fields: readonly string[]) => + cardsOf(fields).forEach((c) => tests[c].reset()); /* 两张卡各自有没有改过、还没保存的格子。测试只测已保存的配置，所以有修改的那张卡 不让测，备注改说「先保存」；保存成功才清掉（#698） */ const [dirty, setDirty] = useState({ @@ -1045,11 +1059,12 @@ export function Settings() { ocr: false, transcribe: false, }); - /* 开测：上一轮的结论（两边卡的 Saved/报错、上一轮测试结果）全部让位给这一轮 */ - const startTest = () => { - test.reset(); - resetSaves(); - test.mutate(); + /* 开测：这张卡上一轮的结论（Saved/报错、上一轮测试结果）让位给这一轮；别的卡没测，不动 */ + const saves = { chat: saveChat, embed: saveEmbed, ocr: saveOcr, transcribe: saveTranscribe }; + const startTest = (card: keyof typeof tests) => { + tests[card].reset(); + saves[card].reset(); + tests[card].mutate(); }; if (!workspace) @@ -1057,9 +1072,9 @@ export function Settings() { const set = (k: keyof typeof form) => (e: React.ChangeEvent) => { - /* 改一格，旧结论全部过期：测试结果、Saved、报错说的都是改之前的那份配置（#698）。 - 测的永远是已存盘的那份，所以表单一动，卡上就不该再贴任何旧话 */ - test.reset(); + /* 改一格，旧结论过期：Saved、报错说的都是改之前的那份配置（#698）。测的永远是 + 已存盘的那份，这张卡的测试结果也一起过期；别的卡没动，测出的还作数（#1424） */ + expireTests([k]); resetSaves(); // 字段名的前缀就是卡：chat_ / embed_ / ocr_ / transcribe_ setDirty((d) => ({ ...d, [k.split("_")[0]]: true })); @@ -1070,15 +1085,16 @@ export function Settings() { /* 两张卡的备注：新测到的盖掉旧保存态，测挂了（连接口都没通）也不再 悄悄退回"Saved"或空白——从前请求本身失败时卡上什么都不说（#698） */ - const testTransportError = test.error ? (test.error as Error).message : null; + const testTransportError = (card: keyof typeof tests) => + tests[card].error ? (tests[card].error as Error).message : null; const chatStatus = modelCardStatus( - test.data + tests.chat.data?.chat ? { - ok: test.data.chat.ok, - message: chatTestMessage(test.data.chat), + ok: tests.chat.data.chat.ok, + message: chatTestMessage(tests.chat.data.chat), } : null, - testTransportError, + testTransportError("chat"), { error: saveChat.error ? (saveChat.error as Error).message : null, saved: saveChat.isSuccess, @@ -1086,15 +1102,15 @@ export function Settings() { }, ); const embedStatus = modelCardStatus( - test.data + tests.embed.data?.embed ? { - ok: test.data.embed.ok, - message: test.data.embed.ok - ? S.settings.okDim(test.data.embed.dim ?? 0) - : (test.data.embed.error ?? ""), + ok: tests.embed.data.embed.ok, + message: tests.embed.data.embed.ok + ? S.settings.okDim(tests.embed.data.embed.dim ?? 0) + : (tests.embed.data.embed.error ?? ""), } : null, - testTransportError, + testTransportError("embed"), { error: saveEmbed.error ? (saveEmbed.error as Error).message : null, saved: saveEmbed.isSuccess, @@ -1102,15 +1118,15 @@ export function Settings() { }, ); const ocrStatus = modelCardStatus( - test.data?.ocr + tests.ocr.data?.ocr ? { - ok: test.data.ocr.ok, - message: test.data.ocr.ok - ? S.settings.okVersion(test.data.ocr.version ?? "?") - : (test.data.ocr.error ?? ""), + ok: tests.ocr.data.ocr.ok, + message: tests.ocr.data.ocr.ok + ? S.settings.okVersion(tests.ocr.data.ocr.version ?? "?") + : (tests.ocr.data.ocr.error ?? ""), } : null, - testTransportError, + testTransportError("ocr"), { error: saveOcr.error ? (saveOcr.error as Error).message : null, saved: saveOcr.isSuccess, @@ -1118,15 +1134,15 @@ export function Settings() { }, ); const transcribeStatus = modelCardStatus( - test.data?.transcribe + tests.transcribe.data?.transcribe ? { - ok: test.data.transcribe.ok, - message: test.data.transcribe.ok + ok: tests.transcribe.data.transcribe.ok, + message: tests.transcribe.data.transcribe.ok ? S.settings.okReachable - : (test.data.transcribe.error ?? ""), + : (tests.transcribe.data.transcribe.error ?? ""), } : null, - testTransportError, + testTransportError("transcribe"), { error: saveTranscribe.error ? (saveTranscribe.error as Error).message : null, saved: saveTranscribe.isSuccess, @@ -1176,16 +1192,16 @@ export function Settings() { key={name} onClick={() => { /* 预设只是批量填格，和手输一样让旧结论过期 */ - test.reset(); - resetSaves(); - setDirty((d) => ({ ...d, chat: true, embed: true })); - setForm({ - ...form, + const filled = { chat_base_url: p.chat, chat_model: p.chatModel, embed_base_url: p.embed, embed_model: p.embedModel, - }); + }; + expireTests(Object.keys(filled)); + resetSaves(); + setDirty((d) => ({ ...d, chat: true, embed: true })); + setForm({ ...form, ...filled }); }} > {name} @@ -1198,21 +1214,18 @@ export function Settings() { note={cardNote(chatStatus)} action={ <> - {/* 试一次连的是两套模型（一个接口），结果各自回到各自那张卡； - 但"Testing…"只亮在按下的这一张上（`testCard`） */} - {needsDate && ( - setCloseAt(e.target.value)} - /> - )} + setCloseAt(e.target.value)} + /> +