diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index 898b474da..855c9f9f2 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -82,7 +82,7 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**，不是最大的编号（守卫 `schema_version_policy_compares_against_current` // 按个数比）：编号有空缺时两者不同——0071 由一个开放 PR 占着，0072 先落，个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 113; +const CURRENT_SCHEMA_VERSION: u32 = 114; fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs index b1c1f63e9..4653f8301 100644 --- a/crates/utopia-extract/src/lib.rs +++ b/crates/utopia-extract/src/lib.rs @@ -492,14 +492,18 @@ fn scan_quantity(s: &str, strict: bool) -> Option { } } } - // 2. 数字：前导的 [-+0-9.,_] + // 2. 数字：前导的 [-+0-9.,_]，负号也可以是排版用的 U+2212 let num_end = rest .char_indices() - .find(|(_, c)| !matches!(c, '0'..='9' | '.' | ',' | '_' | '-' | '+')) + .find(|(_, c)| !matches!(c, '0'..='9' | '.' | ',' | '_' | '-' | '+' | '−')) .map(|(i, _)| i) .unwrap_or(rest.len()); let (num, after) = rest.split_at(num_end); - let cleaned: String = num.chars().filter(|c| !matches!(c, ',' | '_')).collect(); + let cleaned: String = num + .chars() + .filter(|c| !matches!(c, ',' | '_')) + .map(|c| if c == '−' { '-' } else { c }) + .collect(); let mut n: f64 = cleaned.parse().ok()?; // 3. 数字后面：紧贴或空格隔开的量级词、货币词，逐个吃；吃不动的就是「其余」 let mut tail = after.trim_start(); @@ -758,10 +762,14 @@ pub fn attribute_value( return normalize_attr_value(datatype, raw); }; // 声明的单位拆成量级和其余：「万元」→ (4, 元)，「million dollars」→ (6, dollars) + // 币种写在前面的「USD million」「€ million」，量级在末尾 let (tok, rest) = next_token(declared); + let trailing = declared + .rsplit_once(char::is_whitespace) + .and_then(|(head, last)| Some((magnitude(&last.to_ascii_lowercase())?, head.trim()))); let (exponent, base) = match magnitude(&tok.to_ascii_lowercase()) { Some(m) => (m, rest.trim()), - None => (0, declared), + None => trailing.unwrap_or((0, declared)), }; let written = q.unit.as_deref().and_then(currency_unit); let wanted = currency_unit(base); @@ -1216,6 +1224,19 @@ mod tests { normalize_attr_value("number", &json!("52%")), Some(json!(52.0)) ); + // 排版过的负号是 U+2212（维基百科、排版出来的 PDF 都这么写），读法同 `-` + assert_eq!( + normalize_attr_value("number", &json!("−3.2%")), + Some(json!(-3.2)) + ); + assert_eq!( + normalize_attr_value("number", &json!("−1,250")), + Some(json!(-1250.0)) + ); + assert_eq!( + attribute_value("number", Some("万元"), &json!("−2.3亿元")), + Some(json!(-23000.0)) + ); // 原来就认的两种写法不受影响 assert_eq!( normalize_attr_value("number", &json!("35,000")), @@ -1514,6 +1535,24 @@ mod an_attribute_value_is_read_in_its_declared_unit { assert_eq!(value("date", None, "1982年"), Some(json!("1982"))); } + /// 财报的单位常把币种写在量级前面：「USD million」「€ million」，同「million dollars」 + #[test] + fn a_unit_with_its_currency_first_is_still_read_in_millions() { + assert_eq!( + value("number", Some("USD million"), "$5 billion"), + Some(json!(5000.0)) + ); + assert_eq!( + value("number", Some("€ million"), "€30 million"), + Some(json!(30.0)) + ); + assert_eq!(value("number", Some("USD million"), "€30 million"), None); + assert_eq!( + value("number", Some("million dollars"), "$5 billion"), + Some(json!(5000.0)) + ); + } + /// 换不了的不换：货币对不上、读不出数的，宁缺勿脏 #[test] fn what_cannot_be_read_is_not_written() { diff --git a/crates/utopia-ingest/src/ontology_rdf.rs b/crates/utopia-ingest/src/ontology_rdf.rs index 824e5a0d0..cb16586b4 100644 --- a/crates/utopia-ingest/src/ontology_rdf.rs +++ b/crates/utopia-ingest/src/ontology_rdf.rs @@ -840,13 +840,20 @@ fn is_opaque_local_name(local: &str) -> bool { } /// 派生 key：局部名是纯编号且词汇表给了 label 时用 label，否则用局部名。 +/// 局部名派生不出 key 的（`ex:公司` 一个 ASCII 字母都没有）也改用 label。 /// /// **IRI 始终是身份**，这里只换那个给模型读的标签，所以重导入、对齐、 /// `e_by_iri` 的认领都不受影响。 pub fn key_from_iri_or_label(iri: &str, label: Option<&str>) -> String { + let from_iri = key_from_iri(iri); match label { - Some(l) if is_opaque_local_name(local_name(iri)) && !l.trim().is_empty() => to_key(l), - _ => key_from_iri(iri), + Some(l) + if (from_iri.is_empty() || is_opaque_local_name(local_name(iri))) + && !l.trim().is_empty() => + { + to_key(l) + } + _ => from_iri, } } @@ -1292,6 +1299,29 @@ mod tests { assert_eq!(k("http://x#part_12", "part twelve"), "part_12"); } + /// 中文写的 IRI 派生不出 key：有英文 label 就用它，不然每个类都是空 key，导入整个被拒 + #[test] + fn a_chinese_iri_takes_its_key_from_the_label() { + let ttl = r#" +@prefix rdfs: . +@prefix owl: . +@prefix xsd: . +@prefix ex: . + +ex:公司 a owl:Class ; rdfs:label "Company"@en, "公司"@zh . +ex:人员 a owl:Class ; rdfs:label "Person"@en, "人员"@zh . +ex:注册资本 a owl:DatatypeProperty ; rdfs:label "registered capital"@en ; + rdfs:domain ex:公司 ; rdfs:range xsd:decimal . +"#; + let p = project(ttl.as_bytes(), RdfFormat::Turtle).unwrap(); + let mut keys: Vec<&str> = p.classes.iter().map(|c| c.key.as_str()).collect(); + keys.sort(); + assert_eq!(keys, ["company", "person"]); + assert_eq!(p.properties[0].key, "registered_capital"); + // 没有 label 的仍退回局部名 + assert_eq!(key_from_iri_or_label("http://x#Company", None), "company"); + } + /// schema.org 那一套的最小复刻：数据类型自报家门，属性用 /// domainIncludes / rangeIncludes，全部只声明成 rdf:Property。 const SCHEMA_ISH: &str = r#" diff --git a/crates/utopia-server/src/api/google_drive_routes.rs b/crates/utopia-server/src/api/google_drive_routes.rs index f8a995f79..12efe9740 100644 --- a/crates/utopia-server/src/api/google_drive_routes.rs +++ b/crates/utopia-server/src/api/google_drive_routes.rs @@ -42,6 +42,7 @@ const CONNECTION_KEYS: &[&str] = &[ "account_email", "connected_at", "connected_by", + "connected_by_name", "folders", ]; /// 一个来源最多选多少个文件夹。选得更多的，选它们共同的上级 @@ -98,7 +99,7 @@ pub(super) async fn grant_still_used( sqlx::query_scalar( "SELECT EXISTS (SELECT 1 FROM sources WHERE kind = 'google_drive' AND id <> $1 - AND config->>'account_email' = $2 AND config ? 'refresh_token')", + AND lower(config->>'account_email') = lower($2) AND config ? 'refresh_token')", ) .bind(source_id) .bind(account_email) @@ -106,6 +107,46 @@ pub(super) async fn grant_still_used( .await } +/// 重连之后还是不是同一个 Google 账号。邮箱不分大小写；有一边不知道就当换了： +/// 换了账号，上一个账号里选的文件夹不能照着新账号去读 +pub(super) fn same_account(previous: Option<&str>, now: Option<&str>) -> bool { + matches!((previous, now), (Some(a), Some(b)) if a.eq_ignore_ascii_case(b)) +} + +/// 重连之后作废不作废上一份授权：两边的账号都知道、而且不是同一个时才作废。 +/// 有一边不知道，就可能是同一个账号——作废会连刚换来的新令牌一起作废（0074 决定 5） +pub(super) fn revokes_previous(previous: Option<&str>, now: Option<&str>) -> bool { + previous.is_some() && now.is_some() && !same_account(previous, now) +} + +/// 请 Google 作废一枚刷新令牌，除非同一个账号还连着别的来源（[`grant_still_used`]）。 +/// 不知道是哪个账号就不作废。失败只记一笔：库里已经没有它了 +async fn revoke_unless_still_used( + pool: &sqlx::PgPool, + source_id: Uuid, + token: &str, + account: Option<&str>, +) -> Result<(), sqlx::Error> { + let Some(account) = account else { + return Ok(()); + }; + if grant_still_used(pool, source_id, account).await? { + return Ok(()); + } + let revoked = reqwest::Client::new() + .post("https://oauth2.googleapis.com/revoke") + .form(&[("token", token)]) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await; + match revoked { + Ok(resp) if resp.status().is_success() => {} + Ok(resp) => tracing::warn!(status = %resp.status(), "Google 拒绝作废刷新令牌，库里已删除"), + Err(e) => tracing::warn!(error = %e, "Google 刷新令牌作废失败，库里已删除"), + } + Ok(()) +} + /// 落一次连接流程。总数与这个人的在途数都在同一条插入里判，满了就不插，回 `None` pub(super) async fn record_flow( pool: &sqlx::PgPool, @@ -280,16 +321,30 @@ pub(super) async fn finish( ) })?; + let previous_token = source.config["refresh_token"].as_str(); + let previous_account = source.config["account_email"].as_str(); + // 连上的人换了一个 Google 账号：上一个账号里选的文件夹不留，它的授权在没别人用时作废。 + // 同一个账号绝不作废——Google 作废的是整份授权，刚换来的新令牌也会跟着失效 + let changed = !same_account(previous_account, grant.account_email.as_deref()); let grant_keys = json!({ "refresh_token": grant.refresh_token, "account_email": grant.account_email, // 列表里看不见令牌，界面凭这一项知道「连上了」：没验证过的邮箱不显示，不能拿它当标志 "connected_at": chrono::Utc::now().to_rfc3339(), "connected_by": user.id, + // 断开前问一句时说清是谁的 Drive；库的成员列表只给管理员，编辑者那边查不到名字 + "connected_by_name": user.display_name, }); - utopia_store::sources::patch_config(&state.pool, source_id, &grant_keys, &[]) + let dropped: &[&str] = if changed { &["folders"] } else { &[] }; + utopia_store::sources::patch_config(&state.pool, source_id, &grant_keys, dropped) .await .map_err(|e| ("unavailable", format!("saving the grant: {e}")))?; + let revoke = revokes_previous(previous_account, grant.account_email.as_deref()); + if let (true, Some(token)) = (revoke, previous_token) { + revoke_unless_still_used(&state.pool, source_id, token, previous_account) + .await + .map_err(|e| ("unavailable", format!("checking the previous grant: {e}")))?; + } state.emit_source(source.kb_id); // 审计不落令牌，只记连上的是哪个 Google 账号 let _ = utopia_store::audit::record( @@ -305,8 +360,7 @@ pub(super) async fn finish( Ok(()) } -/// 断开：删掉刷新令牌，并请 Google 把它作废——除非同一个账号还连着别的来源 -///（[`grant_still_used`]），或者不知道是哪个账号。作废失败不拦着：库里已经没有它了 +/// 断开：删掉刷新令牌与选的文件夹，再按 [`revoke_unless_still_used`] 请 Google 作废 pub async fn disconnect( State(state): State, AuthUser(user): AuthUser, @@ -314,32 +368,16 @@ pub async fn disconnect( ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; let source = drive_source(&state, kb_id, source_id).await?; - let token = source.config["refresh_token"].as_str().map(String::from); - let account = source.config["account_email"].as_str().map(String::from); utopia_store::sources::patch_config(&state.pool, source_id, &json!({}), CONNECTION_KEYS) .await?; - let revoke = match (token, account) { - (Some(token), Some(account)) - if !grant_still_used(&state.pool, source_id, &account).await? => - { - Some(token) - } - _ => None, - }; - if let Some(token) = revoke { - let revoked = reqwest::Client::new() - .post("https://oauth2.googleapis.com/revoke") - .form(&[("token", token.as_str())]) - .timeout(std::time::Duration::from_secs(10)) - .send() - .await; - match revoked { - Ok(resp) if resp.status().is_success() => {} - Ok(resp) => { - tracing::warn!(status = %resp.status(), "Google 拒绝作废刷新令牌，库里已删除") - } - Err(e) => tracing::warn!(error = %e, "Google 刷新令牌作废失败，库里已删除"), - } + if let Some(token) = source.config["refresh_token"].as_str() { + revoke_unless_still_used( + &state.pool, + source_id, + token, + source.config["account_email"].as_str(), + ) + .await?; } state.emit_source(kb_id); let _ = utopia_store::audit::record( diff --git a/crates/utopia-server/src/api/google_drive_routes_tests.rs b/crates/utopia-server/src/api/google_drive_routes_tests.rs index 29ab75a1b..bd79c4640 100644 --- a/crates/utopia-server/src/api/google_drive_routes_tests.rs +++ b/crates/utopia-server/src/api/google_drive_routes_tests.rs @@ -342,8 +342,12 @@ async fn a_grant_is_kept_while_another_source_uses_the_same_account() -> anyhow: ) .await? .id; - for id in [fx.source, other] { - let grant = json!({ "refresh_token": "1//grant", "account_email": account }); + // 同一个邮箱，大小写不同：Google 账号不分大小写 + for (id, email) in [ + (fx.source, account.clone()), + (other, account.to_uppercase()), + ] { + let grant = json!({ "refresh_token": "1//grant", "account_email": email }); utopia_store::sources::patch_config(&fx.pool, id, &grant, &[]).await?; } let while_other_connected = grant_still_used(&fx.pool, fx.source, &account).await?; @@ -381,3 +385,29 @@ async fn a_drive_source_cannot_be_created_without_an_oauth_client() -> anyhow::R assert_eq!(denied_code(created), Some("google_drive_not_configured")); Ok(()) } + +#[test] +fn a_reconnect_keeps_the_grant_only_for_the_same_account() { + assert_eq!( + [ + same_account(Some("Bruno@re-cinq.com"), Some("bruno@re-cinq.com")), + same_account(Some("bruno@re-cinq.com"), Some("ana@re-cinq.com")), + same_account(None, Some("bruno@re-cinq.com")), + same_account(None, None), + ], + [true, false, false, false] + ); +} + +#[test] +fn a_reconnect_revokes_the_previous_grant_only_when_both_accounts_are_known_and_differ() { + assert_eq!( + [ + revokes_previous(Some("bruno@re-cinq.com"), Some("ana@re-cinq.com")), + revokes_previous(Some("bruno@re-cinq.com"), Some("Bruno@re-cinq.com")), + revokes_previous(Some("bruno@re-cinq.com"), None), + revokes_previous(None, Some("ana@re-cinq.com")), + ], + [true, false, false, false] + ); +} diff --git a/crates/utopia-server/src/api/search_routes.rs b/crates/utopia-server/src/api/search_routes.rs index 29c89f0eb..b7eb2a673 100644 --- a/crates/utopia-server/src/api/search_routes.rs +++ b/crates/utopia-server/src/api/search_routes.rs @@ -10,13 +10,13 @@ use crate::error::ApiResult; use crate::retrieval; use crate::state::AppState; -/// 翻页最深到第几条：每路召回随页深加长，再深就是在拿检索当导出用（#1271） -const MAX_DEPTH: usize = 100; /// 前缀最多几条、每条多长：它们进的是一条 SQL 的数组参数，不是让人塞一份清单 const MAX_PREFIXES: usize = 8; const MAX_PREFIX_BYTES: usize = 256; +/// 不认识的字段答 422：过滤条件还没做，悄悄忽略一个 `filters` 会让调用方以为它生效了 #[derive(Deserialize)] +#[serde(deny_unknown_fields)] pub struct SearchReq { pub q: String, #[serde(default)] @@ -81,14 +81,13 @@ pub async fn search( if q.is_empty() { return Err(AppError::invalid("empty_query", "Search query cannot be empty").into()); } - super::documents_routes::require_reader_kb(&state, &reader, kb_id).await?; - let kb = utopia_store::kbs::get(&state.pool, kb_id).await?; + let kb = super::documents_routes::require_reader_kb(&state, &reader, kb_id).await?; let top_k = req.top_k.unwrap_or(10).min(50); - if req.offset.saturating_add(top_k) > MAX_DEPTH { + if req.offset.saturating_add(top_k) > retrieval::MAX_DEPTH { return Err(AppError::invalid( "page_too_deep", - format!("offset + top_k may not exceed {MAX_DEPTH}"), + format!("offset + top_k may not exceed {}", retrieval::MAX_DEPTH), ) .into()); } diff --git a/crates/utopia-server/src/api/token_read_tests.rs b/crates/utopia-server/src/api/token_read_tests.rs index 22887e4fe..339e7654f 100644 --- a/crates/utopia-server/src/api/token_read_tests.rs +++ b/crates/utopia-server/src/api/token_read_tests.rs @@ -1,8 +1,8 @@ //! 个人令牌读图（0014 决定 7）：搜实体、实体详情、认知历史、事实证据、图总览、邻域、本体、 -//! 某类的实例、文档的抽取结果和整库导出这些读路由收任何一枚活着的令牌。角色照旧是上限，`kb_ids` 排除了的库答 404；改实体仍只认会话。 +//! 某类的实例、文档的抽取结果、整库导出和分块检索这些读路由收任何一枚活着的令牌。角色照旧是上限，`kb_ids` 排除了的库答 404；改实体仍只认会话。 use crate::state::AppState; use axum::body::Body; -use axum::http::{Request, StatusCode}; +use axum::http::{Method, Request, StatusCode}; use serde_json::json; use std::sync::Arc; use tower::ServiceExt; @@ -87,28 +87,49 @@ impl Fixture { .1 } - fn read_paths(&self) -> [String; 10] { + fn read_paths(&self) -> [(Method, String); 11] { let (kb, person, entity, fact, doc) = (self.kb, self.person, self.entity, self.fact, self.doc); [ - format!("/kbs/{kb}/entities?q=Zhang"), - format!("/kbs/{kb}/entities/{entity}"), - format!("/kbs/{kb}/entities/{entity}/history"), - format!("/kbs/{kb}/facts/{fact}/evidence"), - format!("/kbs/{kb}/graph/overview"), - format!("/kbs/{kb}/graph/neighborhood?entity={entity}"), - format!("/kbs/{kb}/ontology"), - format!("/kbs/{kb}/ontology/entity-types/{person}/entities"), - format!("/documents/{doc}/extractions"), - format!("/kbs/{kb}/export?format=turtle"), + (Method::GET, format!("/kbs/{kb}/entities?q=Zhang")), + (Method::GET, format!("/kbs/{kb}/entities/{entity}")), + (Method::GET, format!("/kbs/{kb}/entities/{entity}/history")), + (Method::GET, format!("/kbs/{kb}/facts/{fact}/evidence")), + (Method::GET, format!("/kbs/{kb}/graph/overview")), + ( + Method::GET, + format!("/kbs/{kb}/graph/neighborhood?entity={entity}"), + ), + (Method::GET, format!("/kbs/{kb}/ontology")), + ( + Method::GET, + format!("/kbs/{kb}/ontology/entity-types/{person}/entities"), + ), + (Method::GET, format!("/documents/{doc}/extractions")), + (Method::GET, format!("/kbs/{kb}/export?format=turtle")), + (Method::POST, format!("/kbs/{kb}/search")), ] } - async fn get(&self, path: &str, token: &str) -> anyhow::Result { + /// 每个请求都带同一个检索体；GET 路由不读它 + async fn send(&self, method: Method, path: &str, token: &str) -> anyhow::Result { + self.send_json(method, path, token, r#"{"q":"Zhang"}"#) + .await + } + + async fn send_json( + &self, + method: Method, + path: &str, + token: &str, + body: &str, + ) -> anyhow::Result { let request = Request::builder() + .method(method) .uri(format!("/api/v1{path}")) .header("authorization", format!("Bearer {token}")) - .body(Body::empty())?; + .header("content-type", "application/json") + .body(Body::from(body.to_string()))?; Ok(self.app.clone().oneshot(request).await?.status()) } @@ -128,8 +149,8 @@ async fn a_read_token_of_a_base_viewer_gets_200_on_every_graph_read() -> anyhow: }; let result = async { let token = f.read_token(f.kb).await; - for path in f.read_paths() { - let status = f.get(&path, &token).await?; + for (method, path) in f.read_paths() { + let status = f.send(method, &path, &token).await?; anyhow::ensure!(status == StatusCode::OK, "{path}: {status}"); } Ok(()) @@ -146,8 +167,8 @@ async fn a_token_limited_to_another_base_gets_404_on_every_graph_read() -> anyho }; let result = async { let token = f.read_token(f.other_kb).await; - for path in f.read_paths() { - let status = f.get(&path, &token).await?; + for (method, path) in f.read_paths() { + let status = f.send(method, &path, &token).await?; anyhow::ensure!(status == StatusCode::NOT_FOUND, "{path}: {status}"); } Ok(()) @@ -168,7 +189,7 @@ async fn an_export_by_token_is_audited_under_its_owner_with_the_token_id() -> an .await? .token_id; let path = format!("/kbs/{}/export?format=turtle", f.kb); - let status = f.get(&path, &token).await?; + let status = f.send(Method::GET, &path, &token).await?; anyhow::ensure!(status == StatusCode::OK, "{path}: {status}"); let audited: (Option, serde_json::Value) = sqlx::query_as( @@ -188,3 +209,23 @@ async fn an_export_by_token_is_audited_under_its_owner_with_the_token_id() -> an f.cleanup().await?; result } + +#[tokio::test] +async fn a_search_past_depth_100_or_with_an_unknown_filters_field_gets_422() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let token = f.read_token(f.kb).await; + let path = format!("/kbs/{}/search", f.kb); + let mut statuses = Vec::new(); + for body in [ + r#"{"q":"Zhang","top_k":50,"offset":51}"#, + r#"{"q":"Zhang","filters":{"source_id":"x"}}"#, + ] { + statuses.push(f.send_json(Method::POST, &path, &token, body).await); + } + f.cleanup().await?; + let statuses = statuses.into_iter().collect::>>()?; + assert_eq!(statuses, [StatusCode::UNPROCESSABLE_ENTITY; 2]); + Ok(()) +} diff --git a/crates/utopia-server/src/auth.rs b/crates/utopia-server/src/auth.rs index 0e4a966f9..0d4e79b5b 100644 --- a/crates/utopia-server/src/auth.rs +++ b/crates/utopia-server/src/auth.rs @@ -181,6 +181,7 @@ impl FromRequestParts for AuthUser { /// 整库导出（决定 7）。 /// /// 按推送时的 `external_id` 读回一篇文档（#1276）也是文档原件，开给令牌的理由写在决定 7。 +/// 分块检索（#1271）同样按决定 7 开给令牌。 /// /// 来源关上自己不再陈述的事实（`close-unconfirmed`，fork 的 RC-004）：推陈述的客户端 /// 只有令牌。 diff --git a/crates/utopia-server/src/google_drive.rs b/crates/utopia-server/src/google_drive.rs index 1c02f7243..c48994e8a 100644 --- a/crates/utopia-server/src/google_drive.rs +++ b/crates/utopia-server/src/google_drive.rs @@ -272,6 +272,12 @@ fn id_token_email(id_token: &str, client_id: &str) -> anyhow::Result Utc::now().timestamp()), + "google id_token has expired" + ); Ok(match claims["email_verified"].as_bool() { Some(true) => claims["email"].as_str().map(String::from), _ => None, @@ -610,6 +616,11 @@ impl Drive { else { continue; }; + // Drive 回来的 id 之后要拼进查询和地址：与选中的文件夹同一个字符集，不合的不收 + if !valid_id(id) { + listing.skipped += 1; + continue; + } if mime == FOLDER_MIME { if depth + 1 > MAX_DEPTH { listing.truncated = true; @@ -703,6 +714,11 @@ impl Drive { /// 一个文件的字节：Google 格式导出，其余原样下载 pub async fn content(&self, file: &DriveFile) -> anyhow::Result { + anyhow::ensure!( + valid_id(&file.id), + "google drive: file id {:?} is not a Drive id", + file.id + ); if let Some((mime, ext)) = export_as(&file.mime) { let url = format!("{}/files/{}/export", self.api_root, file.id); let resp = self.get("export", &url, &[("mimeType", mime)]).await?; diff --git a/crates/utopia-server/src/google_drive_tests.rs b/crates/utopia-server/src/google_drive_tests.rs index 179ed0d52..c248057d8 100644 --- a/crates/utopia-server/src/google_drive_tests.rs +++ b/crates/utopia-server/src/google_drive_tests.rs @@ -50,7 +50,11 @@ async fn mount_children(server: &MockServer, parent: &str, files: serde_json::Va .await; } -fn id_token(claims: serde_json::Value) -> String { +/// Google 签的 ID token 一小时后过期；没写 `exp` 的按刚签发算 +fn id_token(mut claims: serde_json::Value) -> String { + if claims.get("exp").is_none() { + claims["exp"] = json!(Utc::now().timestamp() + 3600); + } format!( "{}.{}.sig", URL_SAFE_NO_PAD.encode(br#"{"alg":"RS256"}"#), @@ -113,6 +117,30 @@ async fn a_walk_descends_skips_what_cannot_be_read_and_counts_each_file_once() - Ok(()) } +/// Drive 回来的 id 要拼进查询与地址：不合字符表的文件夹不进、文件不收 +#[tokio::test] +async fn ids_from_drive_outside_the_id_alphabet_are_skipped() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "top", false).await; + mount_children( + &server, + "top", + json!([ + {"id": "x' or name contains '", "name": "Sneaky", "mimeType": FOLDER_MIME}, + {"id": "../../about", "name": "notes.pdf", "mimeType": "application/pdf", + "size": "10", "md5Checksum": "a"}, + {"id": "ok1", "name": "fine.pdf", "mimeType": "application/pdf", + "size": "10", "md5Checksum": "b"} + ]), + ) + .await; + + let listing = drive(&server).walk(&["top".to_string()]).await?; + let ids: Vec<&str> = listing.files.iter().map(|f| f.id.as_str()).collect(); + assert_eq!((ids, listing.skipped), (vec!["ok1"], 2)); + Ok(()) +} + /// 别人组织的会议：文件夹里只有快捷方式，纪要住在组织者的 Drive 里。 /// 指向文件的按目标收（目标自己的标记与时间），指向文件夹的往里走； /// 同一个目标既被直接选到、又经快捷方式到达，只收一次 @@ -551,6 +579,11 @@ fn an_id_token_must_be_googles_and_ours() { let other_issuer = id_token(json!({"iss": "https://evil.example", "aud": ours, "email": "a@b.c", "email_verified": true})); assert!(id_token_email(&other_issuer, &ours).is_err()); + + let expired = id_token(json!({"iss": "accounts.google.com", "aud": ours, + "email": "a@b.c", "email_verified": true, + "exp": Utc::now().timestamp() - 60})); + assert!(id_token_email(&expired, &ours).is_err()); } /// 配置里的文件夹：对象或字符串都认，id 不合字符表的丢掉 diff --git a/crates/utopia-server/src/query_engine/mysql.rs b/crates/utopia-server/src/query_engine/mysql.rs index cf2df7086..df25f7d42 100644 --- a/crates/utopia-server/src/query_engine/mysql.rs +++ b/crates/utopia-server/src/query_engine/mysql.rs @@ -88,6 +88,8 @@ fn cell_kind(type_name: &str) -> Cell { match ty.as_str() { "BOOLEAN" | "BOOL" => Cell::Bool, "TINYINT" | "SMALLINT" | "MEDIUMINT" | "INT" | "BIGINT" => Cell::Int, + // 名字里没有 UNSIGNED，驱动却只认 u64：i64 与 String 都读不出，整列变 null + "YEAR" | "BIT" => Cell::UnsignedInt, "FLOAT" | "DOUBLE" => Cell::Float, "DECIMAL" => Cell::Decimal, "JSON" => Cell::Json, @@ -469,6 +471,24 @@ mod tests { assert!(super::super::guard_sql_for("mysql", "DELETE FROM sales.orders").is_err()); } + /// YEAR 与 BIT 只有无符号整数读得出：字符串、字节都不收它们，落进兜底档就整列 null + #[tokio::test] + async fn a_year_or_bit_column_reads_as_its_number_mysql() { + let Some(fx) = Fx::new(1).await else { return }; + let a = fx.schemas[0].clone(); + fx.exec(&format!( + "CREATE TABLE {a}.budget (fiscal YEAR, active BIT(1), mask BIT(8)); + INSERT INTO {a}.budget VALUES (2024, b'1', b'00000101');" + )) + .await; + let r = MysqlEngine::new(&fx.url) + .execute(&format!("SELECT fiscal, active, mask FROM {a}.budget")) + .await + .expect("execute"); + assert_eq!(r.rows, vec![r#"{"fiscal":2024,"active":1,"mask":5}"#]); + fx.cleanup().await; + } + /// 每个测试自己的库，名字带随机后缀：并行跑不撞，也不碰服务器上已有的库。 /// 测完逐个 `DROP DATABASE`。**这是 MySQL 的「schema」，跟 PG 的命名空间 /// 不同——MySQL 的 schema 就是 database**（见 fetch_schema 顶上的注释） @@ -730,6 +750,10 @@ mod tests { ] { assert_eq!(cell_kind(t), Cell::UnsignedInt, "{t}"); } + // YEAR 与 BIT 的类型名不带 UNSIGNED，驱动却只肯用无符号整数读它们 + for t in ["YEAR", "BIT"] { + assert_eq!(cell_kind(t), Cell::UnsignedInt, "{t}"); + } // DECIMAL 被驱动同时挡在 f64 与 String 之外，只有 BigDecimal 读得出。 // 金额列几乎都是这个类型，掉档的代价最大 assert_eq!(cell_kind("DECIMAL"), Cell::Decimal); diff --git a/crates/utopia-server/src/retrieval.rs b/crates/utopia-server/src/retrieval.rs index fac382749..51c377fe6 100644 --- a/crates/utopia-server/src/retrieval.rs +++ b/crates/utopia-server/src/retrieval.rs @@ -24,6 +24,9 @@ use uuid::Uuid; const RECALL_PER_CHANNEL: usize = 24; +/// 翻页最深到第几条（#1271）：再深就是在拿检索当导出用 +pub const MAX_DEPTH: usize = 100; + pub struct SearchOpts { pub top_k: usize, pub offset: usize, @@ -71,14 +74,26 @@ pub async fn hybrid( as_of, documents: None, }; - let page = hybrid_scored(state, kb_id, workspace_id, query, opts, embeddings).await?; - Ok(page - .results + let (hits, _) = fused_hits( + state, + kb_id, + workspace_id, + query, + RECALL_PER_CHANNEL, + &opts, + embeddings, + ) + .await?; + Ok(hits .into_iter() + .take(top_k) .map(|hit| hit.chunk.chunk) .collect()) } +/// 一页命中（#1271）。每一页都向每路要同样的 `MAX_DEPTH` 条，所以各页切的是 +/// 同一张融合表：页与页不重不漏。到了 `MAX_DEPTH` 的那一页没有下一页（再翻就是 422）， +/// `has_more` 是 false pub async fn hybrid_scored( state: &AppState, kb_id: Uuid, @@ -94,9 +109,40 @@ pub async fn hybrid_scored( vector_used: false, }); } - // 每路多取一条：页尾之后还有没有，得在召回里看得见 - let depth = opts.offset + opts.top_k; - let recall = (depth + 1).max(RECALL_PER_CHANNEL); + let (hits, vector_used) = fused_hits( + state, + kb_id, + workspace_id, + query, + MAX_DEPTH, + &opts, + embeddings, + ) + .await?; + let end = opts.offset + opts.top_k; + let has_more = end < MAX_DEPTH && hits.len() > end; + let results = hits + .into_iter() + .skip(opts.offset) + .take(opts.top_k) + .collect(); + Ok(SearchPage { + results, + has_more, + vector_used, + }) +} + +/// 两路各取 `per_channel` 条、RRF 融合、按记录轴过滤、陈旧的降过分的整张表，连同向量那一路跑了没有 +async fn fused_hits( + state: &AppState, + kb_id: Uuid, + workspace_id: Uuid, + query: &str, + per_channel: usize, + opts: &SearchOpts, + embeddings: Option<&mut llm_util::EmbedCache>, +) -> AppResult<(Vec, bool)> { let bm25 = { let search = state.search.clone(); let kb = kb_id.to_string(); @@ -106,14 +152,22 @@ pub async fn hybrid_scored( .as_ref() .map(|ids| ids.iter().map(Uuid::to_string).collect()); tokio::task::spawn_blocking(move || { - search.search(&kb, &query, recall, documents.as_deref()) + search.search(&kb, &query, per_channel, documents.as_deref()) }) }; // `join!` 不是 `try_join!`：向量那一路失败是降级不是错误（见 vector_channel）， // `try_join!` 会把模型故障变成检索报错 let (bm25, vector) = tokio::join!( bm25, - vector_channel(state, kb_id, workspace_id, query, recall, &opts, embeddings) + vector_channel( + state, + kb_id, + workspace_id, + query, + per_channel, + opts, + embeddings + ) ); let bm25 = bm25 .map_err(|e| utopia_core::AppError::Other(anyhow::anyhow!("BM25 检索线程退出：{e}")))? @@ -132,9 +186,8 @@ pub async fn hybrid_scored( .collect(); let ids: Vec = fused.iter().map(|(id, _)| *id).collect(); let live = utopia_store::documents::chunks_by_ids(&state.pool, kb_id, &ids, opts.as_of).await?; - let has_more = live.len() > depth; let mut fused: HashMap = fused.into_iter().collect(); - let mut scored: Vec = live + let mut hits: Vec = live .into_iter() .filter_map(|chunk| { let f = fused.remove(&chunk.chunk.id)?; @@ -148,22 +201,10 @@ pub async fn hybrid_scored( }) .collect(); // 陈旧的先降分再翻页：久没人读的文档才可能被挤出这一页，而不是只在页内挪位置 - let document_ids: Vec = scored - .iter() - .map(|hit| hit.chunk.chunk.document_id) - .collect(); + let document_ids: Vec = hits.iter().map(|hit| hit.chunk.chunk.document_id).collect(); let stale = utopia_store::decay::stale_documents(&state.pool, &document_ids).await?; - demote_stale(&mut scored, &stale); - let results = scored - .into_iter() - .skip(opts.offset) - .take(opts.top_k) - .collect(); - Ok(SearchPage { - results, - has_more, - vector_used, - }) + demote_stale(&mut hits, &stale); + Ok((hits, vector_used)) } /// 一次检索交出去的文档记一笔（RC-003）。放到后台写：失败只记日志，不拖慢也不打断这次读。 @@ -214,7 +255,7 @@ async fn vector_channel( kb_id: Uuid, workspace_id: Uuid, query: &str, - recall: usize, + per_channel: usize, opts: &SearchOpts, cache: Option<&mut llm_util::EmbedCache>, ) -> AppResult>> { @@ -246,7 +287,7 @@ async fn vector_channel( &state.pool, kb_id, &vector, - recall as i64, + per_channel as i64, opts.as_of, opts.documents.as_deref(), ) diff --git a/crates/utopia-server/src/retrieval_tests.rs b/crates/utopia-server/src/retrieval_tests.rs index 58bc9dfd7..7e71f8317 100644 --- a/crates/utopia-server/src/retrieval_tests.rs +++ b/crates/utopia-server/src/retrieval_tests.rs @@ -195,6 +195,42 @@ async fn both_channels_fuse_with_bm25_first() -> anyhow::Result<()> { .mount(&server) .await; f.point_embeddings_at(&server).await?; + let source = Uuid::now_v7(); + sqlx::raw_sql(&format!( + "INSERT INTO sources(id,kb_id,kind,name) VALUES ('{source}','{}','api','ci'); + UPDATE documents SET source_id='{source}', external_key='api:obs-7' WHERE id='{}'", + f.kb, f.doc + )) + .execute(&f.pool) + .await?; + let opts = super::SearchOpts { + top_k: 8, + offset: 0, + as_of: None, + documents: None, + }; + let page = super::hybrid_scored(&f.state, f.kb, f.ws, "apple harvest", opts, None).await?; + let hits: Vec<_> = page + .results + .iter() + .map(|h| { + ( + h.score, + h.bm25_rank, + h.vector_rank, + h.chunk.source_id, + h.chunk.external_id.as_deref(), + ) + }) + .collect(); + assert!(page.vector_used); + assert_eq!( + hits, + vec![ + (2.0 / 61.0, Some(1), Some(1), Some(source), Some("obs-7")), + (1.0 / 62.0, None, Some(2), Some(source), Some("obs-7")), + ] + ); let ids = f.search("apple harvest").await?; assert_eq!( ids, @@ -354,3 +390,99 @@ async fn a_search_scores_a_stale_document_of_a_decaying_source_at_sixty_percent( assert!((is - was * 0.6).abs() < 1e-12, "{is} is not 0.6 of {was}"); Ok(()) } + +/// `count` chunks replace the fixture's two (#1271): BM25 ranks them c0 first (more +/// "apple" in the same length), the vector channel ranks them the other way round. +async fn seed_ranked_chunks(f: &Fixture, count: usize) -> anyhow::Result { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "data": [{ "embedding": [1.0, 0.0, 0.0, 0.0] }] + }))) + .mount(&server) + .await; + f.point_embeddings_at(&server).await?; + sqlx::query("DELETE FROM chunks WHERE document_id=$1") + .bind(f.doc) + .execute(&f.pool) + .await?; + let mut indexed = Vec::new(); + for seq in 0..count { + let id = Uuid::now_v7(); + let text = std::iter::repeat_n("apple", count - seq) + .chain(std::iter::repeat_n("pear", seq)) + .collect::>() + .join(" "); + sqlx::query( + "INSERT INTO chunks(id,kb_id,document_id,seq,text,embedding) + VALUES($1,$2,$3,$4,$5,$6::text::vector)", + ) + .bind(id) + .bind(f.kb) + .bind(f.doc) + .bind(seq as i32) + .bind(&text) + .bind(format!("[{},1,0,0]", seq + 1)) + .execute(&f.pool) + .await?; + indexed.push((id.to_string(), text)); + } + f.state + .search + .reindex_document(&f.kb.to_string(), &f.doc.to_string(), &indexed)?; + Ok(server) +} + +fn page(offset: usize, top_k: usize) -> super::SearchOpts { + super::SearchOpts { + top_k, + offset, + as_of: None, + documents: None, + } +} + +/// Pages slice one fused list (#1271). A page asking each channel for fewer +/// candidates than a deeper page fuses a different list, so two pages of fifteen +/// would not join into one of thirty. +#[tokio::test] +async fn two_pages_of_fifteen_join_into_the_first_thirty_hits() -> anyhow::Result<()> { + let Some(f) = fixture().await? else { + return Ok(()); + }; + let _server = seed_ranked_chunks(&f, 30).await?; + let walk = |opts| super::hybrid_scored(&f.state, f.kb, f.ws, "apple", opts, None); + let first = walk(page(0, 15)).await?; + let second = walk(page(15, 15)).await?; + let whole = walk(page(0, 30)).await?; + let ids = + |hits: &[super::ScoredChunk]| hits.iter().map(|h| h.chunk.chunk.id).collect::>(); + let joined = [ids(&first.results), ids(&second.results)].concat(); + f.cleanup().await?; + assert_eq!(whole.results.len(), 30); + assert_eq!( + joined, + ids(&whole.results), + "no hit repeats and none is skipped" + ); + assert_eq!( + (first.has_more, second.has_more, whole.has_more), + (true, false, false) + ); + Ok(()) +} + +/// The next page past depth 100 is a 422, so the page ending there has no more +#[tokio::test] +async fn a_page_ending_at_depth_100_of_101_hits_has_no_more() -> anyhow::Result<()> { + let Some(f) = fixture().await? else { + return Ok(()); + }; + let _server = seed_ranked_chunks(&f, 101).await?; + let last = super::hybrid_scored(&f.state, f.kb, f.ws, "apple", page(50, 50), None).await; + f.cleanup().await?; + let last = last?; + assert_eq!((last.results.len(), last.has_more), (50, false)); + Ok(()) +} diff --git a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md index 43d4da629..d23d12794 100644 --- a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md +++ b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md @@ -1,6 +1,6 @@ # 0014 · Identity from the person, scope from the token -- **Status**: Implemented · 2026-10-16 · #180, #318, #1243, #1262 · open: tokens across workspaces +- **Status**: Implemented · 2026-10-16 · #180, #318, #1243, #1262, #1271 · open: tokens across workspaces - **Written**: 2026-09-01 · condensed into English 2026-09-03 - **Related**: migration `0014_data_source_grants` gave data sources a grant layer; this is the same question where a machine knocks. [0004](0004-language-and-localization.md) has the server speak English only, MCP error codes included. [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) removes the main objection to opening `remember` over MCP. @@ -126,6 +126,10 @@ push documents in. mirror to hold a write token for a read-only job. The control is the `kb.exported` audit row, which names the owner and, through `via_token`, the token. + Chunk search (`POST /kbs/{id}/search`, added 2026-10-24, #1271) takes a token on the same + terms, so the service can ask the base about what it mirrored. Each hit carries its score, its + BM25 and vector ranks and its source; `offset` pages through one fused list, at most 100 deep. + ## Dead ends - **Base-level machine tokens**, one per base and unrelated to a person. It would be a third @@ -157,7 +161,7 @@ push documents in. and document content" now also counts them. 2026-10-19: the graph overview and neighbourhood join them, and so does reading a pushed document back by its `external_id` (#1276). The ontology and a class's instances follow (#1265), then a document's extractions and the - base's export (#1266). + base's export (#1266). 2026-10-24: chunk search joins them (#1271). ## Open questions diff --git a/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md b/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md index f6ae08abd..0ae245121 100644 --- a/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md +++ b/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md @@ -18,9 +18,9 @@ Every existing connector is configured with a key an administrator pastes in. Dr 3. **The flow is bound to the person who started it.** `POST …/google-drive/connect` records a single-use `state`, a PKCE verifier, the user and the source in `google_drive_flows` (ten minutes, at most 5 pending per person and 1000 in all, both checked in the insert itself), and returns Google's URL. The callback deletes the row before using it, requires the browser's session to be that same user, and checks again that the user is still an editor. Without the session check, someone could start a flow for their own source and get another person to click through consent, and the other person's Drive would be connected to their source. `prompt=consent` and `access_type=offline` are always sent, so a reconnect after revocation gets a new refresh token. The access token a sync works with lasts an hour and a run can last longer, so a 401 mid-run gets a new access token from the refresh token and the request is tried once more. -4. **The ID token names the account, and that is all it does.** `openid email` exists so the settings can say "connected as bruno@…". The ID token comes straight from Google's token endpoint over TLS, so issuer and audience are checked and the signature is not (OIDC Core 3.1.3.7). An unverified email is not shown. The email never identifies a Utopia account and never links one; that stays as 0056 decided. `connected_at` in the config is what the interface reads as "connected", because the email may legitimately be absent. +4. **The ID token names the account, and that is all it does.** `openid email` exists so the settings can say "connected as bruno@…". The ID token comes straight from Google's token endpoint over TLS, so issuer, audience and expiry are checked and the signature is not (OIDC Core 3.1.3.7). An unverified email is not shown. The email never identifies a Utopia account and never links one; that stays as 0056 decided. `connected_at` in the config is what the interface reads as "connected", because the email may legitimately be absent. -5. **The person who connected chooses the folders, and subfolders come with them.** The callback records `connected_by`. Only that person may browse (`GET …/google-drive/folders`) and save a selection (`PUT …/google-drive/folders`); another editor gets 403. Everyone in the base can read what is synced, so a colleague who could browse would be choosing what leaves someone else's Drive. For the same reason a Drive source's config is written only by its connect, select and disconnect routes: the general source create and update leave it alone. Only the person who connected may reconnect: anyone else gets 403 from `connect`, and a callback that finds someone else connected in the meantime is refused. Otherwise one click by a colleague would replace that person's Drive with their own. To change who reads, an editor disconnects first, which removes the selection with the grant (decision 11), and then connects. `config.folders` is `[{id, name}]`. The name is only for the interface. The browser lists My Drive, Shared with me and every shared drive. Saving the selection queues a sync at once. Ids are checked against `[A-Za-z0-9_-]` before they go into a Drive query, which is simpler to trust than escaping. Each of these routes changes only its own keys in one statement (`patch_config`), so a selection saved during a callback cannot drop the token the callback just wrote. +5. **The person who connected chooses the folders, and subfolders come with them.** The callback records `connected_by`. Only that person may browse (`GET …/google-drive/folders`) and save a selection (`PUT …/google-drive/folders`); another editor gets 403. Everyone in the base can read what is synced, so a colleague who could browse would be choosing what leaves someone else's Drive. For the same reason a Drive source's config is written only by its connect, select and disconnect routes: the general source create and update leave it alone. Only the person who connected may reconnect: anyone else gets 403 from `connect`, and a callback that finds someone else connected in the meantime is refused. Otherwise one click by a colleague would replace that person's Drive with their own. To change who reads, an editor disconnects first, which removes the selection with the grant (decision 11), and then connects. When the person who connected reconnects with a different Google account, the selection made in the old account is dropped and the old grant is revoked as in decision 11. A reconnect with the same account never revokes, because revoking ends the whole grant and would take the new token with it. Accounts compare by email ignoring case; when either email is unknown the account counts as changed, so the folders are dropped, but nothing is revoked. `config.folders` is `[{id, name}]`. The name is only for the interface. The browser lists My Drive, Shared with me and every shared drive. Saving the selection queues a sync at once. Ids are checked against `[A-Za-z0-9_-]` before they go into a Drive query or URL, which is simpler to trust than escaping. That holds for the ids Drive itself returns too: a child folder, file or shortcut target outside that alphabet is skipped. Each of these routes changes only its own keys in one statement (`patch_config`), so a selection saved during a callback cannot drop the token the callback just wrote. 6. **The grant reads only while its person is an editor.** A sync first checks that `connected_by` is still an active editor of the base. When they have left, been made a viewer or been deactivated, the run fails with "reconnect it in the source settings" and reads nothing. Their Google grant would otherwise keep pulling their files into a base they no longer belong to. @@ -32,7 +32,7 @@ Every existing connector is configured with a key an administrator pastes in. Dr 10. **Retraction follows a complete walk only.** Files not seen are marked "not in source" (`reconcile_missing`) only when every selected folder opened and the listing was complete. The download cap does not stop it, because the listing still saw every file. A folder that returns 404, because it was deleted or is no longer shared with the connected person, fails the run by name and marks nothing. Not being able to look is not the same as finding it empty. A file that cannot be read (Google refuses exports over 10 MB) keeps its stored document, the rest of the run continues, and the run is reported as failed with the names, as Notion does. -11. **Disconnecting forgets the grant, and revokes it at Google when nothing else uses it.** `DELETE …/google-drive/connection` removes the token and the selection from the config first. Revoking a refresh token at Google ends the person's whole grant to our OAuth client, every token issued under it, so it would also stop their other Drive sources. The revoke is therefore sent only when no other Google Drive source holds a token for the same account (`account_email`). When the account is unknown because the email was unverified, nothing is revoked: the token is gone from Utopia, and the person can still remove the grant in their Google account. A failed revoke, by network or by a non-2xx status, is logged and does not undo the disconnect. Any editor may disconnect, since stopping a read never exposes anything. Connecting and disconnecting are audited with the account email and never the token. +11. **Disconnecting forgets the grant, and revokes it at Google when nothing else uses it.** `DELETE …/google-drive/connection` removes the token and the selection from the config first. Revoking a refresh token at Google ends the person's whole grant to our OAuth client, every token issued under it, so it would also stop their other Drive sources. The revoke is therefore sent only when no other Google Drive source holds a token for the same account (`account_email`). When the account is unknown because the email was unverified, nothing is revoked: the token is gone from Utopia, and the person can still remove the grant in their Google account. A failed revoke, by network or by a non-2xx status, is logged and does not undo the disconnect. Any editor may disconnect, since stopping a read never exposes anything. Deleting a source does not revoke: its token goes with the row, and the grant stays at Google until the person removes it there or disconnects another source of the same account. Connecting and disconnecting are audited with the account email and never the token. ## Dead ends diff --git a/docs/design/access-and-audit.md b/docs/design/access-and-audit.md index 2f533da5a..6353d9a34 100644 --- a/docs/design/access-and-audit.md +++ b/docs/design/access-and-audit.md @@ -19,10 +19,10 @@ writes an audit row `mcp.tool_called`; the tokens page shows the plaintext once snippet [0014]. A `write` token also sets up a base as its owner: creating a base and a `statements` or `api` source, reading the source's push key, importing an ontology and deciding a kind word, with the owner's role unchanged as the ceiling; any token lists a workspace's bases; a token limited by `kb_ids` -creates no base. These routes, document content and eight graph reads (entity search, detail, +creates no base. These routes, document content, eight graph reads (entity search, detail, history, fact evidence, overview, neighbourhood, ontology, a class's instances; Viewer suffices, -`kb_ids` answers 404), a document's extractions and the base's export (its `kb.exported` row -names the token) take `SessionOrToken`; apart from MCP, every other route is session-only [0014 d6, d7]. `ingest_token` on a source is a different kind: plaintext, push-only [0014]. +`kb_ids` answers 404), a document's extractions, the base's export (its `kb.exported` row +names the token) and chunk search take `SessionOrToken`; apart from MCP, every other route is session-only [0014 d6, d7]. `ingest_token` on a source is a different kind: plaintext, push-only [0014]. **A client reads a pushed document back by its `external_id`.** `GET /kbs/{id}/sources/{sid}/documents/by-external-id?external_id=…` finds an `api` or `statements` diff --git a/docs/design/chat-and-mcp.md b/docs/design/chat-and-mcp.md index ebc21d6b2..9c63cbe07 100644 --- a/docs/design/chat-and-mcp.md +++ b/docs/design/chat-and-mcp.md @@ -75,12 +75,15 @@ the prompt through retrieval; a conventions document works today where a rule wo graph lookups share the turn's embedding cache (#971). The search route answers with each hit's RRF score, its 1-based rank in each channel, `vector_used` (false when the vector channel was skipped) and `offset`/`has_more` paging to a depth of 100; a pushed source's hit carries its `external_id`. An -RRF score ranks hits within one query only; it is not comparable across queries (#1271). The body -may name a `source_id` of the base (another base's source is a 404) and, with it, up to 8 -`external_id_prefixes` of at most 256 bytes; a prefix without a `source_id` is refused. Both channels -then recall only from that source's documents, live at `as_of`, whose `external_id` starts with one -of the prefixes, so the base's other documents cannot take the recall a filter after fusion would -lose (2026-10-19). +RRF score ranks hits within one query only; it is not comparable across queries (#1271). Every +page fuses 100 candidates per channel where chat fuses 24, so the search page can rank the same +query differently from chat and from before #1271. Pages join up only while the base is unchanged +between requests, and each channel counts its ranks before the final `as_of` filter, so a +historical page can skip rank numbers. The body may name a `source_id` of the base (another base's +source is a 404) and, with it, up to 8 `external_id_prefixes` of at most 256 bytes; a prefix without a +`source_id` is refused. Both channels then recall only from that source's documents, live at `as_of`, +whose `external_id` starts with one of the prefixes, so the base's other documents cannot take the +recall a filter after fusion would lose (2026-10-19). A search that returns a document of a source that decays counts as a use of it, in the background and never failing the read ([RC-003](../decisions/re-cinq/RC-003-a-source-may-decay.md)): `search_chunks` diff --git a/migrations/0120_a_google_drive_source_is_connected_by_a_person.sql b/migrations/0120_a_google_drive_source_is_connected_by_a_person.sql new file mode 100644 index 000000000..986a01318 --- /dev/null +++ b/migrations/0120_a_google_drive_source_is_connected_by_a_person.sql @@ -0,0 +1,25 @@ +-- Google Drive 来源（0074）：由一个人授权连上，按他选的文件夹同步。 +-- +-- 一次连接尝试的临时状态：授权码换令牌之前，state 与 PKCE verifier 得先落着， +-- 还要记下是谁、替哪个来源发起的——回调时这个浏览器里的会话必须还是他。 +-- 短命、一次性：回调一到就删，半途弃单的过期扫走。与 oidc_flows 同一个形状。 +CREATE TABLE IF NOT EXISTS google_drive_flows ( + state TEXT PRIMARY KEY, + verifier TEXT NOT NULL, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS google_drive_flows_expiry_idx ON google_drive_flows (expires_at); + +-- 每个文件上一次摄入时 Drive 给的变更标记：二进制文件是 md5Checksum，Google 文档 +-- 没有 md5，用 modifiedTime。**不能靠内容哈希判「没变」**：Docs/Sheets/Slides 每次 +-- 导出成 Office 格式，压缩包里的时间戳都不同，哈希每轮都变，每轮都会重新抽取。 +-- 标记没变、文档还在，就连下载都省了。 +CREATE TABLE IF NOT EXISTS google_drive_files ( + source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + file_id TEXT NOT NULL, + marker TEXT NOT NULL, + PRIMARY KEY (source_id, file_id) +); diff --git a/web/src/api.ts b/web/src/api.ts index dd9b6929d..1b91f3e99 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -253,6 +253,8 @@ export interface SourceView { connected_at?: string; /** google_drive：连上的那个 Utopia 用户；浏览与选文件夹只给他 */ connected_by?: string; + /** google_drive：连上的那个人连接时的名字，断开前的确认里用 */ + connected_by_name?: string; /** false = 这个来源下的文档只检索、不抽取（schema 文档，0035 决定 7）；缺省抽取 */ extract?: boolean; } | null; diff --git a/web/src/docs/ingest.md b/web/src/docs/ingest.md index d4b9a6d9e..a4db0cd30 100644 --- a/web/src/docs/ingest.md +++ b/web/src/docs/ingest.md @@ -211,6 +211,37 @@ The credential is a personal access token (or a signed-in session) of someone wh `content` is the current version's text; older versions are at `GET /api/v1/documents/{id}/content?version=N`. An identity marked "Not in source", a document in the trash or purged, and an identity this source never pushed are all `404`. +## Searching what you pushed + +A service can search the base's chunks with the same personal access token. The push token is not accepted here either. + +``` +POST {your-utopia-base}/api/v1/kbs/{kb_id}/search +Authorization: Bearer utp_pat_… +Content-Type: application/json + +{ "q": "harvest report", "top_k": 20, "offset": 0 } +``` + +`q` is required. `top_k` defaults to 10 and is capped at 50; `offset` defaults to 0. `as_of` (an RFC 3339 instant) reads the base as it was then; ranks are counted before `as_of` filters the hits. Any other field is a `422`. + +```json +{ + "results": [ + { + "id": "0199…", "document_id": "0199…", "seq": 3, "text": "…", "filename": "obs-000412.json", + "source_id": "0199…", "external_id": "obs-000412", + "score": 0.0328, "bm25_rank": 1, "vector_rank": 2 + } + ], + "offset": 0, "top_k": 20, "has_more": true, "vector_used": true +} +``` + +`external_id` is the identity you pushed the document under, so you can match a hit to your own record; it is `null` for documents that did not come from an API or Statements push. `score` orders hits within this one query and means nothing across queries. `bm25_rank` and `vector_rank` count from 1 and are `null` when the hit was not in that channel. `vector_used` is `false` when no embedding model answered and the hits are keyword-only. + +Page by raising `offset` by `top_k` while `has_more` is `true`. Pages stop at a depth of 100: a request with `offset + top_k` above 100 is a `422` (`page_too_deep`), and the page that reaches 100 has `has_more: false`. Pages join without repeats only while the base does not change between requests. + --- ## Shared semantics diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index e86771e53..ddadca68f 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -808,11 +808,12 @@ export const en = { remove: "Remove", saveFolders: "Save folders and sync", saved: "Folders saved. A sync has started.", - disconnectConfirm: (account: string, mine: boolean) => + disconnectConfirm: (account: string, mine: boolean, connector: string) => mine ? `Stop reading your Google Drive (${account})? Its access and the chosen folders are removed here, and connecting again starts from scratch.` - : `Stop reading ${account}'s Google Drive? Someone else connected it, and only they chose its folders. Its access and those folders are removed here.`, + : `Stop reading the Google Drive ${connector} connected (${account})? Only ${connector} chose its folders. Its access and those folders are removed here.`, unnamedAccount: "the connected account", + someoneElse: "someone else", cancel: "Cancel", disconnected: "Google Drive disconnected.", }, diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index c1350b171..d7d531dbb 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -737,11 +737,12 @@ export const zh: Strings = { remove: "移除", saveFolders: "保存并同步", saved: "文件夹已保存，同步已开始。", - disconnectConfirm: (account: string, mine: boolean) => + disconnectConfirm: (account: string, mine: boolean, connector: string) => mine ? `停止读取你的 Google Drive（${account}）？这里会删掉它的授权和选好的文件夹，再连接要从头选。` - : `停止读取 ${account} 的 Google Drive？这是别人连上的，文件夹也是他选的。这里会删掉它的授权和那些文件夹。`, + : `停止读取 ${connector} 连上的 Google Drive（${account}）？文件夹只有 ${connector} 选过。这里会删掉它的授权和那些文件夹。`, unnamedAccount: "已连接的账号", + someoneElse: "别人", cancel: "取消", disconnected: "Google Drive 已断开。", }, diff --git a/web/src/pages/GoogleDrivePanel.tsx b/web/src/pages/GoogleDrivePanel.tsx index 2d5511c57..4c79c34c6 100644 --- a/web/src/pages/GoogleDrivePanel.tsx +++ b/web/src/pages/GoogleDrivePanel.tsx @@ -166,7 +166,11 @@ export function GoogleDrivePanel({ kbId, source }: { kbId: string; source: Sourc {confirmingDisconnect && (