From 5ceb99d066a1a1becad893ff2e9e8834eade3613 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Sun, 25 Oct 2026 08:21:14 +0100 Subject: [PATCH 1/7] A person can connect Google Drive, and only they choose which of their folders a base reads (#1386) Signed-off-by: Bogdan Szabo Co-authored-by: muemich --- .env.example | 7 + crates/utopia-cli/src/main.rs | 2 +- crates/utopia-core/src/models.rs | 4 + .../src/api/google_drive_routes.rs | 506 +++++++++++ .../src/api/google_drive_routes_tests.rs | 412 +++++++++ crates/utopia-server/src/api/mod.rs | 16 + .../utopia-server/src/api/sources_routes.rs | 24 +- crates/utopia-server/src/google_drive.rs | 846 ++++++++++++++++++ .../utopia-server/src/google_drive_tests.rs | 630 +++++++++++++ crates/utopia-server/src/ingest_sources.rs | 170 +++- crates/utopia-server/src/main.rs | 1 + crates/utopia-store/src/sources.rs | 24 + .../tests/store/a_secret_is_sealed_at_rest.rs | 26 + ...e-drive-source-is-connected-by-a-person.md | 48 + docs/decisions/README.md | 1 + docs/design/sources.md | 6 +- ..._drive_source_is_connected_by_a_person.sql | 25 + web/src/i18n/en.ts | 5 + web/src/i18n/zh.ts | 5 + web/src/pages/SourcesRail.tsx | 2 + web/src/sourceKinds.ts | 1 + 21 files changed, 2754 insertions(+), 7 deletions(-) create mode 100644 crates/utopia-server/src/api/google_drive_routes.rs create mode 100644 crates/utopia-server/src/api/google_drive_routes_tests.rs create mode 100644 crates/utopia-server/src/google_drive.rs create mode 100644 crates/utopia-server/src/google_drive_tests.rs create mode 100644 docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md create mode 100644 migrations/0120_a_google_drive_source_is_connected_by_a_person.sql diff --git a/.env.example b/.env.example index 7536fb174..62da0f4bd 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,13 @@ UTOPIA_OPEN_REGISTRATION=true # 没配单点登录时这个 false 不生效。单点登录开的账号没有密码、也没法重设，改回 true 也进不去—— # 包括经 SSO 引导出来的首个管理员。务必留一个有密码的管理员账号当后门。 # UTOPIA_PASSWORD_LOGIN=true +# Google Drive 来源（0074）：一个人用自己的 Google 账号连接，按他选的文件夹同步。 +# 在 Google Cloud 建一个「Web 应用」类型的 OAuth 客户端，回调地址填下面这个，并启用 Drive API。 +# 三项都填才开放；与上面 SSO 的客户端是两回事（权限范围不同）。drive.readonly 是受限范围： +# Workspace 内部应用不需要 Google 审核，对外发布的需要。 +# UTOPIA_GOOGLE_DRIVE_CLIENT_ID= +# UTOPIA_GOOGLE_DRIVE_CLIENT_SECRET= +# UTOPIA_GOOGLE_DRIVE_REDIRECT_URI=https://your-deployment/api/v1/google-drive/callback # 数据库连接池上限。缺省 32，与 worker 并发的缺省对齐。 # 池子小于并发时症状是请求变慢、而不是任何一处说"池子不够"——调高 worker # 并发时一并考虑这里，也别超过 Postgres 自己的 max_connections（默认 100）。 diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index 8f0fe0364..f3683aabd 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -82,7 +82,7 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**，不是最大的编号（守卫 `schema_version_policy_compares_against_current` // 按个数比）：编号有空缺时两者不同——0071 由一个开放 PR 占着，0072 先落，个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 105; +const CURRENT_SCHEMA_VERSION: u32 = 106; fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 8eab5f642..890493356 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -259,6 +259,8 @@ pub const SOURCE_SECRET_KEYS: &[&str] = &[ "secret_access_key", "account_key", "service_account_key", + // Google Drive 的 OAuth 刷新令牌（0074）：能换出那个人整个 Drive 的只读权限 + "refresh_token", ]; impl Source { @@ -306,6 +308,8 @@ pub enum SourceKind { Gcs, Webdav, Notion, + /// 一个人授权、按他选的文件夹同步（0074） + GoogleDrive, Api, /// 推送的不是文档而是陈述本身（0054）：请求体就是开放抽取契约，抽取不问模型 Statements, diff --git a/crates/utopia-server/src/api/google_drive_routes.rs b/crates/utopia-server/src/api/google_drive_routes.rs new file mode 100644 index 000000000..12efe9740 --- /dev/null +++ b/crates/utopia-server/src/api/google_drive_routes.rs @@ -0,0 +1,506 @@ +//! Google Drive 来源的连接与选文件夹（0074）。 +//! +//! **连接由来源的编辑者本人完成。** 他在来源设置里点「连接」，我们记下一次流程 +//! （state、PKCE verifier、是谁、哪个来源），把他送到 Google 的同意页；回调时这个 +//! 浏览器里的会话必须还是他，而且他对那个库还得是 Editor。不核对的话，有人可以 +//! 自己发起一次流程，再诱使别人点完同意——别人的 Drive 就接进了他的来源。 +//! +//! **连上之后，文件夹只由连上的那个人选。** 读的是他的 Drive，库里任何人都看得见同步进来的 +//! 东西：别的编辑者能浏览、能选，就等于能把他没打算交出来的文件夹拉进库里。所以回调记下 +//! `connected_by`，浏览与选择只认这个人；他不再是编辑者时，同步停下来让人重连。 +//! +//! 刷新令牌只进不出：存进来源配置的 `refresh_token`（封印），文件夹浏览器的请求 +//! 在服务器上拿它换访问令牌，浏览器从头到尾拿不到任何一枚 Google 令牌。 + +use axum::extract::{Path, Query, State}; +use axum::response::Html; +use axum::Json; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use serde::Deserialize; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use utopia_core::models::{Role, Source, SourceKind}; +use utopia_core::AppError; +use uuid::Uuid; + +use super::graph_routes::require_kb; +use crate::auth::{self, AuthUser}; +use crate::error::{ApiErr, ApiResult}; +use crate::google_drive::{self, Drive, FolderEntry, OAuthConfig}; +use crate::state::AppState; + +/// 一次连接流程能活多久：同意页上停十分钟还没点完，就当放弃了 +const FLOW_TTL_SECS: u64 = 600; +/// 同时在途的连接流程上限，理由同 SSO：每一次 `connect` 都落一行 +const MAX_PENDING_FLOWS: i64 = 1000; +/// 一个人同时在途的上限：没有它，一个编辑者连点就能占满上面那个总数，十分钟里谁都连不上 +const MAX_PENDING_FLOWS_PER_USER: i64 = 5; +/// 连接这件事落在配置里的几个键。只由连接、断开两条路由写；断开时连同文件夹一起删， +/// 下一个连上的人从头选 +const CONNECTION_KEYS: &[&str] = &[ + "refresh_token", + "account_email", + "connected_at", + "connected_by", + "connected_by_name", + "folders", +]; +/// 一个来源最多选多少个文件夹。选得更多的，选它们共同的上级 +const MAX_FOLDERS: usize = 50; + +fn configured() -> Result { + google_drive::oauth_config().ok_or_else(|| { + AppError::invalid( + "google_drive_not_configured", + "Google Drive is not configured on this server", + ) + }) +} + +/// 取一条 Google Drive 来源，确认它属于路径上的这个库 +async fn drive_source(state: &AppState, kb_id: Uuid, source_id: Uuid) -> Result { + let source = utopia_store::sources::get(&state.pool, source_id).await?; + if source.kb_id != kb_id || SourceKind::parse(&source.kind) != Some(SourceKind::GoogleDrive) { + return Err(AppError::NotFound); + } + Ok(source) +} + +/// 这个来源连着的 Drive 是不是这个人的：浏览与选文件夹只认他（0074 决定 5） +fn require_connector(source: &Source, user_id: Uuid) -> Result<(), AppError> { + if google_drive::connected_by(&source.config) == Some(user_id) { + return Ok(()); + } + Err(AppError::CodedForbidden { + code: "google_drive_not_connector", + message: "Only the person who connected this Google Drive can choose its folders".into(), + }) +} + +/// 别人连着的来源不能直接重连：一点就会换掉他的 Drive、丢掉他选的文件夹。 +/// 要换人，先断开（断开会问一句，说清是谁的 Drive），再由新的人来连 +fn refuse_someone_elses_connection(source: &Source, user_id: Uuid) -> Result<(), AppError> { + match google_drive::connected_by(&source.config) { + Some(connector) if connector != user_id => Err(AppError::CodedForbidden { + code: "google_drive_connected_by_someone_else", + message: "This source reads someone else's Google Drive; disconnect it first".into(), + }), + _ => Ok(()), + } +} + +/// 同一个 Google 账号是不是还连着别的来源。Google 撤销一枚刷新令牌，撤的是这个人对 +/// 我们这个客户端的整份授权，他别的 Drive 来源会一起失效；还有别的在用就不撤 +pub(super) async fn grant_still_used( + pool: &sqlx::PgPool, + source_id: Uuid, + account_email: &str, +) -> Result { + sqlx::query_scalar( + "SELECT EXISTS (SELECT 1 FROM sources + WHERE kind = 'google_drive' AND id <> $1 + AND lower(config->>'account_email') = lower($2) AND config ? 'refresh_token')", + ) + .bind(source_id) + .bind(account_email) + .fetch_one(pool) + .await +} + +/// 重连之后还是不是同一个 Google 账号。邮箱不分大小写；有一边不知道就当换了： +/// 换了账号，上一个账号里选的文件夹不能照着新账号去读 +pub(super) fn same_account(previous: Option<&str>, now: Option<&str>) -> bool { + matches!((previous, now), (Some(a), Some(b)) if a.eq_ignore_ascii_case(b)) +} + +/// 重连之后作废不作废上一份授权：两边的账号都知道、而且不是同一个时才作废。 +/// 有一边不知道，就可能是同一个账号——作废会连刚换来的新令牌一起作废（0074 决定 5） +pub(super) fn revokes_previous(previous: Option<&str>, now: Option<&str>) -> bool { + previous.is_some() && now.is_some() && !same_account(previous, now) +} + +/// 请 Google 作废一枚刷新令牌，除非同一个账号还连着别的来源（[`grant_still_used`]）。 +/// 不知道是哪个账号就不作废。失败只记一笔：库里已经没有它了 +async fn revoke_unless_still_used( + pool: &sqlx::PgPool, + source_id: Uuid, + token: &str, + account: Option<&str>, +) -> Result<(), sqlx::Error> { + let Some(account) = account else { + return Ok(()); + }; + if grant_still_used(pool, source_id, account).await? { + return Ok(()); + } + let revoked = reqwest::Client::new() + .post("https://oauth2.googleapis.com/revoke") + .form(&[("token", token)]) + .timeout(std::time::Duration::from_secs(10)) + .send() + .await; + match revoked { + Ok(resp) if resp.status().is_success() => {} + Ok(resp) => tracing::warn!(status = %resp.status(), "Google 拒绝作废刷新令牌，库里已删除"), + Err(e) => tracing::warn!(error = %e, "Google 刷新令牌作废失败，库里已删除"), + } + Ok(()) +} + +/// 落一次连接流程。总数与这个人的在途数都在同一条插入里判，满了就不插，回 `None` +pub(super) async fn record_flow( + pool: &sqlx::PgPool, + user_id: Uuid, + source_id: Uuid, +) -> Result, sqlx::Error> { + sqlx::query("DELETE FROM google_drive_flows WHERE expires_at < now()") + .execute(pool) + .await?; + let flow_state = auth::generate_jwt_secret(); + let verifier = auth::generate_jwt_secret(); + let inserted = sqlx::query( + "INSERT INTO google_drive_flows (state, verifier, user_id, source_id, expires_at) + SELECT $1, $2, $3, $4, now() + make_interval(secs => $5) + WHERE (SELECT count(*) FROM google_drive_flows WHERE user_id = $3) < $6 + AND (SELECT count(*) FROM google_drive_flows) < $7", + ) + .bind(&flow_state) + .bind(&verifier) + .bind(user_id) + .bind(source_id) + .bind(FLOW_TTL_SECS as f64) + .bind(MAX_PENDING_FLOWS_PER_USER) + .bind(MAX_PENDING_FLOWS) + .execute(pool) + .await?; + Ok((inserted.rows_affected() == 1).then_some((flow_state, verifier))) +} + +/// 界面据此决定要不要提供这种来源的连接按钮 +pub async fn status(AuthUser(_): AuthUser) -> Json { + Json(json!({ "enabled": google_drive::oauth_config().is_some() })) +} + +/// 开始一次连接：回给界面一个 Google 同意页的地址，界面在弹窗里打开它 +pub async fn connect( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + refuse_someone_elses_connection(&source, user.id)?; + let cfg = configured()?; + let (flow_state, verifier) = record_flow(&state.pool, user.id, source_id) + .await? + .ok_or_else(|| { + AppError::invalid( + "google_drive_busy", + "Too many Google Drive connections are in progress; try again in a few minutes", + ) + })?; + let challenge = URL_SAFE_NO_PAD.encode(Sha256::digest(verifier.as_bytes())); + + let mut url = + reqwest::Url::parse(google_drive::AUTHORIZE_URL).map_err(|e| AppError::Other(e.into()))?; + url.query_pairs_mut().extend_pairs([ + ("response_type", "code"), + ("client_id", cfg.client_id.as_str()), + ("redirect_uri", cfg.redirect.as_str()), + ("scope", google_drive::SCOPES), + ("state", flow_state.as_str()), + ("code_challenge", challenge.as_str()), + ("code_challenge_method", "S256"), + // 刷新令牌只在 offline + 每次都请同意时才保证发：重连一个撤过授权的来源也一样 + ("access_type", "offline"), + ("prompt", "consent"), + ("include_granted_scopes", "false"), + ]); + Ok(Json(json!({ "url": url.as_str() }))) +} + +#[derive(Deserialize)] +pub struct Callback { + code: Option, + state: Option, + /// 人在同意页点了取消 + error: Option, +} + +/// 回调落在弹窗里：成功与失败都回一张不带脚本的小页面。界面那头在轮询来源， +/// 看见 `account_email` 出现就关掉弹窗 +pub async fn callback( + State(state): State, + user: Result, + Query(q): Query, +) -> Html { + match finish(&state, user.ok().map(|AuthUser(u)| u), q).await { + Ok(()) => page( + "Google Drive connected", + "You can close this window and choose the folders to sync.", + ), + Err((code, reason)) => { + tracing::warn!(code, reason = %reason, "Google Drive 连接被拒"); + page( + "Google Drive was not connected", + match code { + "cancelled" => "The consent screen was cancelled. Close this window and try again.", + "session" => "This window is not signed in to Utopia as the person who started the connection. Close it and start again from the source settings.", + "expired" => "This connection attempt has expired. Close this window and start again.", + "scope" => "Google did not grant read access to Drive. Close this window, connect again and allow it.", + "taken" => "Someone else connected this source while you were on Google's page. Close this window.", + _ => "Google could not be reached or refused the request. Close this window and try again.", + }, + ) + } + } +} + +pub(super) async fn finish( + state: &AppState, + user: Option, + q: Callback, +) -> Result<(), (&'static str, String)> { + let db = |e: sqlx::Error| ("unavailable", format!("database: {e}")); + let (Some(code), Some(flow_state)) = (q.code, q.state) else { + return Err(match q.error { + Some(e) => ("cancelled", format!("google returned {e}")), + None => ("invalid", "callback without code or state".into()), + }); + }; + // 一次性：先删后用，同一个 state 第二次到就什么都取不出来 + let flow: Option<(String, Uuid, Uuid)> = sqlx::query_as( + "DELETE FROM google_drive_flows WHERE state = $1 AND expires_at > now() + RETURNING verifier, user_id, source_id", + ) + .bind(&flow_state) + .fetch_optional(&state.pool) + .await + .map_err(db)?; + let (verifier, flow_user, source_id) = + flow.ok_or(("expired", "unknown or expired state".to_string()))?; + if let Some(e) = q.error { + return Err(("cancelled", format!("google returned {e}"))); + } + let user = user.filter(|u| u.id == flow_user).ok_or(( + "session", + "callback session is not the user who started it".to_string(), + ))?; + let source = utopia_store::sources::get(&state.pool, source_id) + .await + .map_err(|e| ("invalid", format!("source: {e}")))?; + // 发起之后的这几分钟里权限可能被收回了 + require_kb(state, &user, source.kb_id, Role::Editor) + .await + .map_err(|_| { + ( + "session", + "user is no longer an editor of the knowledge base".to_string(), + ) + })?; + // 同意页上的这几分钟里，别人可能先连上了 + refuse_someone_elses_connection(&source, user.id).map_err(|_| { + ( + "taken", + "someone else connected this source in the meantime".to_string(), + ) + })?; + + let cfg = google_drive::oauth_config().ok_or(("unavailable", "not configured".to_string()))?; + let grant = google_drive::exchange_code(&cfg, &code, &verifier) + .await + .map_err(|e| { + let msg = format!("{e:#}"); + ( + if msg.contains("read access to Drive") { + "scope" + } else { + "unavailable" + }, + msg, + ) + })?; + + let previous_token = source.config["refresh_token"].as_str(); + let previous_account = source.config["account_email"].as_str(); + // 连上的人换了一个 Google 账号：上一个账号里选的文件夹不留，它的授权在没别人用时作废。 + // 同一个账号绝不作废——Google 作废的是整份授权，刚换来的新令牌也会跟着失效 + let changed = !same_account(previous_account, grant.account_email.as_deref()); + let grant_keys = json!({ + "refresh_token": grant.refresh_token, + "account_email": grant.account_email, + // 列表里看不见令牌，界面凭这一项知道「连上了」：没验证过的邮箱不显示，不能拿它当标志 + "connected_at": chrono::Utc::now().to_rfc3339(), + "connected_by": user.id, + // 断开前问一句时说清是谁的 Drive；库的成员列表只给管理员，编辑者那边查不到名字 + "connected_by_name": user.display_name, + }); + let dropped: &[&str] = if changed { &["folders"] } else { &[] }; + utopia_store::sources::patch_config(&state.pool, source_id, &grant_keys, dropped) + .await + .map_err(|e| ("unavailable", format!("saving the grant: {e}")))?; + let revoke = revokes_previous(previous_account, grant.account_email.as_deref()); + if let (true, Some(token)) = (revoke, previous_token) { + revoke_unless_still_used(&state.pool, source_id, token, previous_account) + .await + .map_err(|e| ("unavailable", format!("checking the previous grant: {e}")))?; + } + state.emit_source(source.kb_id); + // 审计不落令牌，只记连上的是哪个 Google 账号 + let _ = utopia_store::audit::record( + &state.pool, + Some(source.kb_id), + user.id, + "source.google_drive_connected", + "source", + Some(source_id), + json!({ "account_email": grant.account_email }), + ) + .await; + Ok(()) +} + +/// 断开：删掉刷新令牌与选的文件夹，再按 [`revoke_unless_still_used`] 请 Google 作废 +pub async fn disconnect( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + utopia_store::sources::patch_config(&state.pool, source_id, &json!({}), CONNECTION_KEYS) + .await?; + if let Some(token) = source.config["refresh_token"].as_str() { + revoke_unless_still_used( + &state.pool, + source_id, + token, + source.config["account_email"].as_str(), + ) + .await?; + } + state.emit_source(kb_id); + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "source.google_drive_disconnected", + "source", + Some(source_id), + json!({}), + ) + .await; + Ok(Json(json!({ "ok": true }))) +} + +#[derive(Deserialize)] +pub struct FoldersQuery { + /// 不给 = 第一层（我的云端硬盘、与我共享、各个共享云端硬盘） + parent: Option, +} + +/// 文件夹浏览器：以连上的那个人的身份列子文件夹，只给他本人看 +pub async fn folders( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, + Query(q): Query, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + require_connector(&source, user.id)?; + let cfg = configured()?; + let refresh = source.config["refresh_token"] + .as_str() + .filter(|t| !t.is_empty()) + .ok_or_else(|| { + AppError::invalid( + "google_drive_not_connected", + "Connect a Google account first", + ) + })?; + let unavailable = |e: anyhow::Error| { + AppError::invalid_detail( + "google_drive_unavailable", + "Google Drive could not be read", + format!("{e:#}"), + ) + }; + let drive = Drive::connect(&cfg, refresh).await.map_err(unavailable)?; + let folders: Vec = match q.parent.as_deref().filter(|p| !p.is_empty()) { + None => drive.roots().await, + Some(parent) => drive.child_folders(parent).await, + } + .map_err(unavailable)?; + Ok(Json(json!({ "folders": folders }))) +} + +#[derive(Deserialize)] +pub struct SelectBody { + folders: Vec, +} + +/// 存下选中的文件夹，并立即同步一次。只有连上的那个人能选。换了文件夹，上一批里 +/// 不在新范围内的文档会在这一轮被标成「不在来源中」，删不删由人决定，与别的来源一样 +pub async fn select( + State(state): State, + AuthUser(user): AuthUser, + Path((kb_id, source_id)): Path<(Uuid, Uuid)>, + Json(body): Json, +) -> ApiResult> { + require_kb(&state, &user, kb_id, Role::Editor).await?; + let source = drive_source(&state, kb_id, source_id).await?; + require_connector(&source, user.id)?; + if body.folders.len() > MAX_FOLDERS { + return Err(AppError::invalid( + "google_drive_too_many_folders", + format!("Choose at most {MAX_FOLDERS} folders"), + ) + .into()); + } + if body.folders.iter().any(|f| !google_drive::valid_id(&f.id)) { + return Err(AppError::invalid("google_drive_invalid_folder", "Invalid folder id").into()); + } + let updated = utopia_store::sources::patch_config( + &state.pool, + source_id, + &json!({ "folders": body.folders }), + &[], + ) + .await?; + if !body.folders.is_empty() { + let _ = utopia_store::sources::mark_queued(&state.pool, source_id).await; + utopia_store::jobs::enqueue( + &state.pool, + "sync_source", + json!({ "source_id": source_id }), + ) + .await?; + } + state.emit_source(kb_id); + let _ = utopia_store::audit::record( + &state.pool, + Some(kb_id), + user.id, + "source.updated", + "source", + Some(source_id), + json!({ "google_drive_folders": body.folders.len() }), + ) + .await; + Ok(Json(json!({ "source": updated.without_secrets() }))) +} + +/// 一张小页面，不带脚本。字符串都是这里写死的，不拼任何请求里来的东西 +fn page(title: &str, body: &str) -> Html { + Html(format!( + "{title}\ + \ + \ +

{title}

{body}

" + )) +} + +#[cfg(test)] +#[path = "google_drive_routes_tests.rs"] +mod tests; diff --git a/crates/utopia-server/src/api/google_drive_routes_tests.rs b/crates/utopia-server/src/api/google_drive_routes_tests.rs new file mode 100644 index 000000000..06174863a --- /dev/null +++ b/crates/utopia-server/src/api/google_drive_routes_tests.rs @@ -0,0 +1,412 @@ +//! 连接流程的几道关卡与「谁连上的谁来选」（0074 决定 3、5），连真库跑。 +//! 测试环境没配 OAuth 客户端：过了所有关卡的回调停在「没配置」，不会去敲 Google +use super::*; +use std::sync::Arc; +use utopia_core::models::User; + +struct Fixture { + pool: sqlx::PgPool, + state: AppState, + kb: Uuid, + source: Uuid, + connector: User, + colleague: User, + _dir: tempfile::TempDir, +} + +impl Fixture { + async fn new() -> anyhow::Result> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(None); + }; + let pool = sqlx::PgPool::connect(&url).await?; + utopia_store::db::migrate(&pool).await?; + let id = Uuid::now_v7; + let (org, ws, kb, connector, colleague) = (id(), id(), id(), id(), id()); + // 插进去的只有本地生成的 UUID + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','google-drive-test'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','google-drive-test'); + INSERT INTO users(id,org_id,email,display_name,password_hash) VALUES + ('{connector}','{org}','{connector}@drive.test','Connector','unused'), + ('{colleague}','{org}','{colleague}@drive.test','Colleague','unused'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','drive'); + INSERT INTO kb_members(kb_id,user_id,role) VALUES + ('{kb}','{connector}','editor'), ('{kb}','{colleague}','editor');" + )) + .execute(&pool) + .await?; + let source = utopia_store::sources::create( + &pool, + kb, + "google_drive", + "Drive", + &json!({}), + None, + None, + None, + ) + .await? + .id; + utopia_store::sources::patch_config( + &pool, + source, + &json!({ + "refresh_token": "1//grant", + "connected_by": connector, + "folders": [{"id": "chosen", "name": "Chosen"}], + }), + &[], + ) + .await?; + let dir = tempfile::tempdir()?; + let cfg = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open( + &dir.path().join("search"), + )?); + let state = AppState::new(pool.clone(), &cfg, search, "test-only".into()); + let user = |id| utopia_store::accounts::find_user_by_id(&pool, id); + let (connector, colleague) = (user(connector).await?, user(colleague).await?); + Ok(Some(Self { + state, + kb, + source, + connector: connector.expect("connector exists"), + colleague: colleague.expect("colleague exists"), + pool, + _dir: dir, + })) + } + + async fn flow(&self) -> anyhow::Result { + let (flow_state, _) = record_flow(&self.pool, self.connector.id, self.source) + .await? + .expect("room for a flow"); + Ok(flow_state) + } + + async fn callback(&self, session: Option<&User>, flow_state: &str) -> &'static str { + let q = Callback { + code: Some("code".into()), + state: Some(flow_state.into()), + error: None, + }; + match finish(&self.state, session.cloned(), q).await { + Ok(()) => "connected", + Err((code, _)) => code, + } + } + + async fn demote_connector(&self) -> anyhow::Result<()> { + sqlx::query("UPDATE kb_members SET role = 'viewer' WHERE kb_id = $1 AND user_id = $2") + .bind(self.kb) + .bind(self.connector.id) + .execute(&self.pool) + .await?; + Ok(()) + } + + async fn config(&self) -> anyhow::Result { + Ok(utopia_store::sources::get(&self.pool, self.source) + .await? + .config) + } +} + +fn denied_code(result: ApiResult) -> Option<&'static str> { + match result { + Err(ApiErr(AppError::CodedForbidden { code, .. } | AppError::Invalid { code, .. })) => { + Some(code) + } + _ => None, + } +} + +#[tokio::test] +async fn a_state_used_a_second_time_is_expired() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let flow_state = fx.flow().await?; + assert_eq!( + fx.callback(Some(&fx.connector), &flow_state).await, + "unavailable" + ); + assert_eq!( + fx.callback(Some(&fx.connector), &flow_state).await, + "expired" + ); + Ok(()) +} + +#[tokio::test] +async fn a_callback_in_another_session_is_refused() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let flow_state = fx.flow().await?; + assert_eq!( + fx.callback(Some(&fx.colleague), &flow_state).await, + "session" + ); + let flow_state = fx.flow().await?; + assert_eq!(fx.callback(None, &flow_state).await, "session"); + Ok(()) +} + +#[tokio::test] +async fn a_callback_after_the_person_stopped_being_an_editor_is_refused() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let flow_state = fx.flow().await?; + fx.demote_connector().await?; + assert_eq!( + fx.callback(Some(&fx.connector), &flow_state).await, + "session" + ); + Ok(()) +} + +#[tokio::test] +async fn a_callback_after_ten_minutes_is_expired() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let flow_state = fx.flow().await?; + sqlx::query( + "UPDATE google_drive_flows SET expires_at = now() - interval '1 second' WHERE state = $1", + ) + .bind(&flow_state) + .execute(&fx.pool) + .await?; + assert_eq!( + fx.callback(Some(&fx.connector), &flow_state).await, + "expired" + ); + Ok(()) +} + +#[tokio::test] +async fn a_person_has_at_most_five_connections_in_progress() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + for _ in 0..5 { + fx.flow().await?; + } + let sixth = record_flow(&fx.pool, fx.connector.id, fx.source).await?; + let colleagues_first = record_flow(&fx.pool, fx.colleague.id, fx.source).await?; + assert_eq!((sixth.is_some(), colleagues_first.is_some()), (false, true)); + Ok(()) +} + +#[tokio::test] +async fn a_colleague_cannot_browse_or_choose_the_connected_persons_folders() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let path = || Path((fx.kb, fx.source)); + let browsed = folders( + State(fx.state.clone()), + AuthUser(fx.colleague.clone()), + path(), + Query(FoldersQuery { parent: None }), + ) + .await; + let chosen = select( + State(fx.state.clone()), + AuthUser(fx.colleague.clone()), + path(), + Json(SelectBody { + folders: vec![FolderEntry { + id: "theirs".into(), + name: "Theirs".into(), + }], + }), + ) + .await; + assert_eq!( + (denied_code(browsed), denied_code(chosen)), + ( + Some("google_drive_not_connector"), + Some("google_drive_not_connector") + ) + ); + assert_eq!( + fx.config().await?["folders"], + json!([{"id": "chosen", "name": "Chosen"}]) + ); + Ok(()) +} + +#[tokio::test] +async fn the_source_settings_cannot_rewrite_a_drive_connection() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let before = fx.config().await?; + let _ = super::super::sources_routes::update( + State(fx.state.clone()), + AuthUser(fx.colleague.clone()), + Path((fx.kb, fx.source)), + Json(super::super::sources_routes::UpdateBody { + name: None, + config: Some(json!({ + "connected_by": fx.colleague.id, + "folders": [{"id": "theirs", "name": "Theirs"}], + })), + icon: None, + schedule: None, + }), + ) + .await + .map_err(|ApiErr(e)| e)?; + assert_eq!(fx.config().await?, before); + Ok(()) +} + +#[tokio::test] +async fn a_sync_stops_once_the_connected_person_is_no_longer_an_editor() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let reason = |e: anyhow::Error| format!("{e:#}"); + let while_editor = crate::ingest_sources::sync_source(&fx.state, fx.source) + .await + .map_err(reason); + fx.demote_connector().await?; + let after = crate::ingest_sources::sync_source(&fx.state, fx.source) + .await + .map_err(reason); + let says_reconnect = |r: &Result<(), String>| { + r.as_ref() + .is_err_and(|e| e.contains("no longer an editor") && e.contains("reconnect")) + }; + assert_eq!( + (says_reconnect(&while_editor), says_reconnect(&after)), + (false, true) + ); + Ok(()) +} + +#[tokio::test] +async fn a_colleague_cannot_reconnect_the_connected_persons_drive() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let started = connect( + State(fx.state.clone()), + AuthUser(fx.colleague.clone()), + Path((fx.kb, fx.source)), + ) + .await; + assert_eq!( + denied_code(started), + Some("google_drive_connected_by_someone_else") + ); + Ok(()) +} + +#[tokio::test] +async fn a_callback_after_someone_else_connected_is_refused() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let (flow_state, _) = record_flow(&fx.pool, fx.colleague.id, fx.source) + .await? + .expect("room for a flow"); + assert_eq!(fx.callback(Some(&fx.colleague), &flow_state).await, "taken"); + Ok(()) +} + +#[tokio::test] +async fn a_grant_is_kept_while_another_source_uses_the_same_account() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let account = format!("{}@drive.test", Uuid::now_v7()); + let other = utopia_store::sources::create( + &fx.pool, + fx.kb, + "google_drive", + "Other Drive", + &json!({}), + None, + None, + None, + ) + .await? + .id; + // 同一个邮箱，大小写不同：Google 账号不分大小写 + for (id, email) in [ + (fx.source, account.clone()), + (other, account.to_uppercase()), + ] { + let grant = json!({ "refresh_token": "1//grant", "account_email": email }); + utopia_store::sources::patch_config(&fx.pool, id, &grant, &[]).await?; + } + let while_other_connected = grant_still_used(&fx.pool, fx.source, &account).await?; + utopia_store::sources::patch_config(&fx.pool, other, &json!({}), CONNECTION_KEYS).await?; + let after_other_disconnected = grant_still_used(&fx.pool, fx.source, &account).await?; + assert_eq!( + (while_other_connected, after_other_disconnected), + (true, false) + ); + Ok(()) +} + +#[tokio::test] +async fn a_drive_source_cannot_be_created_without_an_oauth_client() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let created = super::super::sources_routes::create( + State(fx.state.clone()), + crate::auth::SessionOrToken { + user: fx.connector.clone(), + pat: None, + }, + Path(fx.kb), + Json(super::super::sources_routes::CreateBody { + kind: "google_drive".into(), + name: "Drive".into(), + config: json!({}), + icon: None, + sync_interval_minutes: None, + sync_cron: None, + }), + ) + .await; + assert_eq!(denied_code(created), Some("google_drive_not_configured")); + Ok(()) +} + +#[test] +fn a_reconnect_keeps_the_grant_only_for_the_same_account() { + assert_eq!( + [ + same_account(Some("Bruno@re-cinq.com"), Some("bruno@re-cinq.com")), + same_account(Some("bruno@re-cinq.com"), Some("ana@re-cinq.com")), + same_account(None, Some("bruno@re-cinq.com")), + same_account(None, None), + ], + [true, false, false, false] + ); +} + +#[test] +fn a_reconnect_revokes_the_previous_grant_only_when_both_accounts_are_known_and_differ() { + assert_eq!( + [ + revokes_previous(Some("bruno@re-cinq.com"), Some("ana@re-cinq.com")), + revokes_previous(Some("bruno@re-cinq.com"), Some("Bruno@re-cinq.com")), + revokes_previous(Some("bruno@re-cinq.com"), None), + revokes_previous(None, Some("ana@re-cinq.com")), + ], + [true, false, false, false] + ); +} diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs index 7d488e3f1..2b01f5426 100644 --- a/crates/utopia-server/src/api/mod.rs +++ b/crates/utopia-server/src/api/mod.rs @@ -8,6 +8,7 @@ mod datasource_routes; pub(crate) mod documents_routes; mod events_routes; mod export_routes; +mod google_drive_routes; mod graph_routes; mod jobs_routes; mod kbs; @@ -122,6 +123,9 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router { .route("/auth/oidc/callback", get(oidc_routes::callback)) // 受信的另一个应用拿它用户刚到手的 ID token 换一个会话（0066） .route("/auth/oidc/exchange", post(oidc_routes::exchange)) + // Google Drive 来源（0074）：部署配了 OAuth 客户端才有；回调落在连接弹窗里 + .route("/google-drive/status", get(google_drive_routes::status)) + .route("/google-drive/callback", get(google_drive_routes::callback)) // 我自己的绑定：看、解绑。绑定走 `/auth/oidc/start?link=1`，由本人完成 .route( "/auth/oidc/me", @@ -505,6 +509,18 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router { 4 * sources_routes::STATEMENTS_MAX_BYTES, )), ) + .route( + "/kbs/{id}/sources/{source_id}/google-drive/connect", + post(google_drive_routes::connect), + ) + .route( + "/kbs/{id}/sources/{source_id}/google-drive/connection", + axum::routing::delete(google_drive_routes::disconnect), + ) + .route( + "/kbs/{id}/sources/{source_id}/google-drive/folders", + get(google_drive_routes::folders).put(google_drive_routes::select), + ) .route( "/kbs/{id}/sources/{source_id}/token", get(sources_routes::get_token), diff --git a/crates/utopia-server/src/api/sources_routes.rs b/crates/utopia-server/src/api/sources_routes.rs index 7a4e62eb8..ddd859dd8 100644 --- a/crates/utopia-server/src/api/sources_routes.rs +++ b/crates/utopia-server/src/api/sources_routes.rs @@ -6,7 +6,7 @@ use axum::Json; use chrono::{DateTime, Utc}; use serde::Deserialize; use serde_json::json; -use utopia_core::models::{Role, SOURCE_SECRET_KEYS}; +use utopia_core::models::{Role, SourceKind, SOURCE_SECRET_KEYS}; use uuid::Uuid; use super::graph_routes::require_kb; @@ -109,12 +109,22 @@ pub async fn create( let user = caller.user; require_kb(&state, &user, kb_id, Role::Editor).await?; validate_rss_config(&body.kind, &body.config)?; + let drive = config_is_drive_owned(&body.kind); + // 没配 OAuth 客户端的部署连不上 Google，建出来也只会一直失败（0074 决定 2） + if drive && crate::google_drive::oauth_config().is_none() { + return Err(utopia_core::AppError::invalid( + "google_drive_not_configured", + "Google Drive is not configured on this server", + ) + .into()); + } + let config = if drive { json!({}) } else { body.config }; let source = utopia_store::sources::create( &state.pool, kb_id, &body.kind, &body.name, - &body.config, + &config, body.icon.as_deref(), body.sync_interval_minutes, body.sync_cron.as_deref(), @@ -214,6 +224,12 @@ fn validate_rss_config(kind: &str, config: &serde_json::Value) -> utopia_core::A Ok(()) } +/// Google Drive 来源的配置只由它自己的连接、选文件夹、断开三条路由写（0074）。 +/// 通用的建、改要是也能写，任何编辑者都能绕过「只有连上的那个人能选文件夹」 +fn config_is_drive_owned(kind: &str) -> bool { + SourceKind::parse(kind) == Some(SourceKind::GoogleDrive) +} + /// 更新时凭据的合并规则，每个 `SOURCE_SECRET_KEYS` 里的键一样：新配置里**没有**这个键 /// 或值是空串 → 保留库里的原值（表单留空就是「别动」）；显式 `null` → 删掉； /// 其余照新值。响应从不回显，所以客户端没有办法把旧值原样送回来，规则只能长在这里 @@ -266,7 +282,9 @@ pub async fn update( require_kb(&state, &user, kb_id, Role::Editor).await?; let existing = source_in_kb(&state, kb_id, source_id).await?; // 凭据只进不出：响应从不回显，表单留空 / 没传 = 保留库里原值 - let mut config = body.config; + let mut config = body + .config + .filter(|_| !config_is_drive_owned(&existing.kind)); if let Some(cfg) = config.as_mut() { keep_secrets(cfg, &existing.config); } diff --git a/crates/utopia-server/src/google_drive.rs b/crates/utopia-server/src/google_drive.rs new file mode 100644 index 000000000..c48994e8a --- /dev/null +++ b/crates/utopia-server/src/google_drive.rs @@ -0,0 +1,846 @@ +//! Google Drive 来源：一个人用自己的 Google 账号授权，按他选的文件夹同步（0074）。 +//! +//! 按 [0013](../../docs/decisions/0013-a-source-should-hand-over-its-history.md) 的四条判据： +//! +//! | 判据 | Google Drive | +//! |---|---| +//! | 真实时间戳 | `modifiedTime`，文件内容最后一次改动的时刻 | +//! | 会不会自我推翻 | 方案、纪要、合同的新版本覆盖旧版本 | +//! | 稳定身份 | 文件 id，改名、挪目录都不变 | +//! | 企业知识住不住在那儿 | 大多数团队的文档就住在这儿 | +//! +//! 与 Notion 一样**只交出现状**：修订历史在 API 里有，但 Google 文档的旧修订只能 +//! 导出成当时的全文，抽样哪几个是另一个决定，这一版不做。 +//! +//! ## 授权是一个人的，不是部署的 +//! +//! Notion、对象存储是管理员贴一把密钥；这里是**这个来源的编辑者**点「连接 Google +//! Drive」，在 Google 那边同意只读授权，我们存下刷新令牌（封印，`SOURCE_SECRET_KEYS`）。 +//! OAuth 客户端是部署级的（`UTOPIA_GOOGLE_DRIVE_CLIENT_ID` 等），没配就没有这个按钮。 +//! 授权顺带要 `openid email`，只为在界面上说清「这是以谁的身份在读」。 +//! +//! ## 几个容易踩的 +//! +//! **没有递归查询。** `q` 只认 `'{id}' in parents`，一层一层走；两个选中的文件夹 +//! 互相嵌套时同一个文件会出现两次，按 id 去重。 +//! +//! **共享云端硬盘里的东西默认看不见。** 每个请求都得带 `supportsAllDrives=true`， +//! 列表还得带 `includeItemsFromAllDrives=true`，少了不报错，只是结果是空的。 +//! +//! **Google 自己的格式要导出。** Docs/Sheets/Slides 没有字节，`alt=media` 回 403； +//! 走 `export` 换成 DOCX/XLSX/PPTX，交给已有的解析器（标题层级、表格、多张工作表都在）。 +//! 导出上限 10 MB，超了是 403 `exportSizeLimitExceeded`，按读不出来的那一份处理。 +//! +//! **导出的字节每次都不同**（压缩包里带时间戳），所以「变没变」不能靠内容哈希， +//! 靠 Drive 给的标记：二进制用 `md5Checksum`，Google 格式用 `modifiedTime`。 +//! +//! 错误文案是英文：它会原样落到来源的同步状态里给用户看。 + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::time::Duration; + +use anyhow::Context as _; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use chrono::{DateTime, Utc}; +use reqwest::StatusCode; +use serde_json::Value; +use uuid::Uuid; + +pub const AUTHORIZE_URL: &str = "https://accounts.google.com/o/oauth2/v2/auth"; +const TOKEN_URL: &str = "https://oauth2.googleapis.com/token"; +const API_ROOT: &str = "https://www.googleapis.com/drive/v3"; + +/// 只读整个 Drive。`drive.file` 不够：它只给应用自己建的、或经 Picker 逐个选中的 +/// 文件，选一个文件夹并不连带它里面的东西 +pub const DRIVE_SCOPE: &str = "https://www.googleapis.com/auth/drive.readonly"; +pub const SCOPES: &str = "openid email https://www.googleapis.com/auth/drive.readonly"; + +const FOLDER_MIME: &str = "application/vnd.google-apps.folder"; +const SHORTCUT_MIME: &str = "application/vnd.google-apps.shortcut"; +/// 一个文件要用到的元数据，列表与快捷方式的目标都取这些 +const FILE_FIELDS: &str = "id,name,mimeType,modifiedTime,md5Checksum,size"; + +/// 一次遍历最多列多少个文件。列表只是元数据，标记没变的文件不下载，所以这里放得宽； +/// 它只防一棵大得离谱的树把内存撑爆。到了这个数，这一轮说出来，不报成功 +const MAX_FILES_LISTED: usize = 50_000; +/// 一次同步最多下载（或导出）多少个文件，与 WebDAV、对象存储同一个量级：摄入不可逆。 +/// 上限卡在下载上而不是列表上：列表每轮都是全的，没变的不花钱，到了上限的这一轮 +/// 说出来，下一轮从还没下的接着下，不会每次都停在同一个地方（#1212 那类缺口） +pub const MAX_DOWNLOADS_PER_SYNC: usize = 2_000; +/// 单个文件的上限，同对象存储 +const MAX_FILE_BYTES: u64 = 32 * 1024 * 1024; +/// 选中文件夹往下走几层。再深的留着不走，比一轮同步卡在一棵树上好 +const MAX_DEPTH: usize = 16; +/// 文件夹浏览器一次最多回多少个子文件夹 +const MAX_FOLDERS_LISTED: usize = 1_000; +/// 429 / 5xx 最多再试几回 +const MAX_RETRIES: u32 = 3; +const JSON_LIMIT: usize = 4 * 1024 * 1024; + +/// 部署级的 OAuth 客户端。三项都得有，缺一项就当没配 +#[derive(Clone)] +pub struct OAuthConfig { + pub client_id: String, + pub client_secret: String, + pub redirect: String, +} + +pub fn oauth_config() -> Option { + let get = |key: &str| { + std::env::var(key) + .ok() + .map(|v| v.trim().to_string()) + .filter(|v| !v.is_empty()) + }; + Some(OAuthConfig { + client_id: get("UTOPIA_GOOGLE_DRIVE_CLIENT_ID")?, + client_secret: get("UTOPIA_GOOGLE_DRIVE_CLIENT_SECRET")?, + redirect: get("UTOPIA_GOOGLE_DRIVE_REDIRECT_URI")?, + }) +} + +/// 文件、文件夹、共享盘的 id 只有这些字符。它们要拼进 `q` 的单引号里， +/// 不在这张表上的一律不收——比转义可靠 +pub fn valid_id(id: &str) -> bool { + !id.is_empty() + && id.len() <= 128 + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} + +/// 来源配置里选中的文件夹：`folders: [{id, name}]`。名字只给界面看，同步只认 id +pub fn selected_folders(config: &Value) -> Vec { + config["folders"] + .as_array() + .map(|a| { + a.iter() + .filter_map(|f| f["id"].as_str().or_else(|| f.as_str())) + .map(str::trim) + .filter(|id| valid_id(id)) + .map(String::from) + .collect() + }) + .unwrap_or_default() +} + +/// 谁连上的这个来源：回调时记下的 Utopia 用户。没有 = 还没连，或是这条规则之前连的 +pub fn connected_by(config: &Value) -> Option { + config["connected_by"].as_str()?.parse().ok() +} + +/// 授权码换来的东西 +#[derive(Debug)] +pub struct Grant { + pub refresh_token: String, + /// 授权的那个 Google 账号，ID token 里核验过的邮箱；只用来在界面上显示 + pub account_email: Option, +} + +fn http() -> anyhow::Result { + Ok(reqwest::Client::builder() + .timeout(Duration::from_secs(60)) + .build()?) +} + +/// 授权码 + PKCE verifier 换令牌。要的是刷新令牌：访问令牌一小时就过期， +/// 同步是在没人看着的时候跑的 +pub async fn exchange_code(cfg: &OAuthConfig, code: &str, verifier: &str) -> anyhow::Result { + exchange_code_at(&http()?, TOKEN_URL, cfg, code, verifier).await +} + +async fn exchange_code_at( + http: &reqwest::Client, + token_url: &str, + cfg: &OAuthConfig, + code: &str, + verifier: &str, +) -> anyhow::Result { + let resp = http + .post(token_url) + .form(&[ + ("grant_type", "authorization_code"), + ("code", code), + ("code_verifier", verifier), + ("client_id", &cfg.client_id), + ("client_secret", &cfg.client_secret), + ("redirect_uri", &cfg.redirect), + ]) + .send() + .await + .context("google token")?; + let v = token_response(resp).await?; + // 同意页上 Google 允许人把某一项权限的勾去掉；去掉了 Drive 的那一项， + // 令牌照发，只是什么都读不到。在这里说清楚，比让第一次同步报 403 好 + let scopes = v["scope"].as_str().unwrap_or(""); + anyhow::ensure!( + scopes.split_whitespace().any(|s| s == DRIVE_SCOPE), + "Google did not grant read access to Drive; connect again and allow it" + ); + // 只有第一次同意（或 `prompt=consent`）才发刷新令牌；`begin` 每次都带 consent + let refresh_token = v["refresh_token"] + .as_str() + .filter(|t| !t.is_empty()) + .context("Google returned no refresh token; connect again")? + .to_string(); + let account_email = match v["id_token"].as_str() { + Some(t) => id_token_email(t, &cfg.client_id)?, + None => None, + }; + Ok(Grant { + refresh_token, + account_email, + }) +} + +/// 刷新令牌换一个访问令牌 +async fn access_token_at( + http: &reqwest::Client, + token_url: &str, + cfg: &OAuthConfig, + refresh_token: &str, +) -> anyhow::Result { + let resp = http + .post(token_url) + .form(&[ + ("grant_type", "refresh_token"), + ("refresh_token", refresh_token), + ("client_id", &cfg.client_id), + ("client_secret", &cfg.client_secret), + ]) + .send() + .await + .context("google token")?; + let v = token_response(resp).await?; + Ok(v["access_token"] + .as_str() + .filter(|t| !t.is_empty()) + .context("google token: response has no access_token")? + .to_string()) +} + +/// 令牌端点的响应。`invalid_grant` 单独说：那是人撤了授权、改了密码，或者 +/// 刷新令牌半年没用过期了——重连能好，重试不能 +async fn token_response(resp: reqwest::Response) -> anyhow::Result { + let status = resp.status(); + let v: Value = resp + .json() + .await + .context("google token: reading the response")?; + if status.is_success() { + return Ok(v); + } + if v["error"] == "invalid_grant" { + anyhow::bail!( + "Google Drive access was revoked or has expired; reconnect the source in its settings" + ); + } + anyhow::bail!( + "google token returned {status}: {}", + v["error_description"] + .as_str() + .or(v["error"].as_str()) + .unwrap_or("no message") + ) +} + +/// ID token 里的邮箱。 +/// +/// **不验签名**：这枚令牌是我们自己拿授权码、经 TLS 直接从 Google 的令牌端点换回来的， +/// OIDC Core 3.1.3.7 允许这时以 TLS 代替签名校验。issuer 与 audience 照查——查的是 +/// 「这是 Google 签给本部署的」。邮箱只拿来显示，不认身份、不开账号； +/// 没验证过的邮箱不显示，免得界面上写着一个那人并不拥有的地址 +fn id_token_email(id_token: &str, client_id: &str) -> anyhow::Result> { + let payload = id_token + .split('.') + .nth(1) + .context("google id_token is not a JWT")?; + let claims: Value = serde_json::from_slice( + &URL_SAFE_NO_PAD + .decode(payload.trim_end_matches('=')) + .context("google id_token payload is not base64url")?, + ) + .context("google id_token payload is not JSON")?; + anyhow::ensure!( + matches!( + claims["iss"].as_str(), + Some("https://accounts.google.com" | "accounts.google.com") + ), + "google id_token has an unexpected issuer" + ); + anyhow::ensure!( + claims["aud"].as_str() == Some(client_id), + "google id_token was issued to a different client" + ); + anyhow::ensure!( + claims["exp"] + .as_i64() + .is_some_and(|exp| exp > Utc::now().timestamp()), + "google id_token has expired" + ); + Ok(match claims["email_verified"].as_bool() { + Some(true) => claims["email"].as_str().map(String::from), + _ => None, + }) +} + +/// 文件夹浏览器里的一项 +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub struct FolderEntry { + pub id: String, + pub name: String, +} + +/// 选中的文件夹里的一个文件 +#[derive(Debug, Clone)] +pub struct DriveFile { + pub id: String, + pub name: String, + pub mime: String, + pub modified: Option>, + md5: Option, + modified_raw: Option, +} + +impl DriveFile { + /// `gdrive://{file_id}`——文件 id 是它最稳的身份 + pub fn external_key(&self) -> String { + format!("gdrive://{}", self.id) + } + + /// 「变没变」的标记，见模块注释 + pub fn marker(&self) -> String { + self.md5 + .clone() + .or_else(|| self.modified_raw.clone()) + .unwrap_or_default() + } +} + +/// 一轮遍历的结果 +#[derive(Debug, Default)] +pub struct Listing { + pub files: Vec, + /// 到了文件数上限，或有文件夹深过 `MAX_DEPTH` + pub truncated: bool, + /// 选中却打不开的文件夹（删了、或不再共享给这个人）。有它时不对账： + /// 打不开不等于里面的东西都没了 + pub unreachable: Vec, + /// 跳过的文件，每个一句原因：表单、超大的 + pub skipped: usize, + /// 指向这个人打不开的文件的快捷方式（没共享给他、删了），按快捷方式的名字 + pub broken_shortcuts: Vec, +} + +/// 读一个文件拿到的字节 +pub struct Content { + pub bytes: Vec, + pub filename: String, + pub mime: &'static str, +} + +/// Google 自己的格式导出成什么。不在这张表上的 Google 格式（表单、绘图、 +/// 快捷方式、站点……）没有能读的正文，跳过 +fn export_as(mime: &str) -> Option<(&'static str, &'static str)> { + Some(match mime { + "application/vnd.google-apps.document" => ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "docx", + ), + "application/vnd.google-apps.spreadsheet" => ( + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xlsx", + ), + "application/vnd.google-apps.presentation" => ( + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "pptx", + ), + _ => return None, + }) +} + +/// 拿着一个访问令牌的 Drive 客户端 +pub struct Drive { + http: reqwest::Client, + token: std::sync::Mutex, + /// 访问令牌一小时就过期，一轮同步可以更长：遇到 401 凭刷新令牌换一枚新的，再试一次 + renewal: Option, + // 私有地址入口让 HTTP 测试走真实的遍历逻辑，不增加用户配置 + api_root: String, +} + +struct Renewal { + cfg: OAuthConfig, + refresh_token: String, + token_url: String, +} + +impl Drive { + /// 用刷新令牌换访问令牌，得到一个能用的客户端 + pub async fn connect(cfg: &OAuthConfig, refresh_token: &str) -> anyhow::Result { + let http = http()?; + let token = access_token_at(&http, TOKEN_URL, cfg, refresh_token).await?; + Ok(Self { + http, + token: std::sync::Mutex::new(token), + renewal: Some(Renewal { + cfg: cfg.clone(), + refresh_token: refresh_token.into(), + token_url: TOKEN_URL.into(), + }), + api_root: API_ROOT.into(), + }) + } + + /// 发一次 GET。429 与 5xx 退避重试；401 换一枚访问令牌再试一次（只一次：换来的也被拒， + /// 就是授权本身出了问题）；其余非 2xx 带上 Google 自己的 message 报错， + /// 状态码与 `reason` 留在错误里给调用方分辨（404 = 打不开） + async fn get( + &self, + what: &str, + url: &str, + query: &[(&str, &str)], + ) -> anyhow::Result { + let url = reqwest::Url::parse_with_params(url, query) + .with_context(|| format!("google drive {what}: building the URL"))?; + let mut attempt = 0u32; + let mut renewed = false; + loop { + let token = self.token.lock().expect("token lock").clone(); + let resp = self + .http + .get(url.clone()) + .bearer_auth(&token) + .send() + .await + .with_context(|| format!("google drive {what}"))?; + let status = resp.status(); + if let (StatusCode::UNAUTHORIZED, false, Some(r)) = (status, renewed, &self.renewal) { + renewed = true; + let fresh = + access_token_at(&self.http, &r.token_url, &r.cfg, &r.refresh_token).await?; + *self.token.lock().expect("token lock") = fresh; + continue; + } + if (status == StatusCode::TOO_MANY_REQUESTS || status.is_server_error()) + && attempt < MAX_RETRIES + { + attempt += 1; + let wait = Duration::from_millis(500 * 2u64.pow(attempt)); + tracing::info!(what, attempt, %status, "google drive busy, waiting before the next attempt"); + tokio::time::sleep(wait).await; + continue; + } + if status.is_success() { + return Ok(resp); + } + let v: Value = resp.json().await.unwrap_or(Value::Null); + let reason = v["error"]["errors"][0]["reason"].as_str().unwrap_or(""); + return Err(DriveError { + status, + message: format!( + "google drive {what} returned {status}{}: {}", + if reason.is_empty() { + String::new() + } else { + format!(" ({reason})") + }, + v["error"]["message"].as_str().unwrap_or("no message") + ), + } + .into()); + } + } + + async fn get_json( + &self, + what: &str, + url: &str, + query: &[(&str, &str)], + ) -> anyhow::Result { + let resp = self.get(what, url, query).await?; + let bytes = read_capped(resp, JSON_LIMIT as u64) + .await + .with_context(|| format!("google drive {what}"))?; + serde_json::from_slice(&bytes).with_context(|| format!("google drive {what}: not JSON")) + } + + /// 按 `q` 列文件，翻完所有页或到 `limit` 为止。第二项：是不是因为到了上限才停的 + async fn list( + &self, + q: &str, + fields: &str, + limit: usize, + ) -> anyhow::Result<(Vec, bool)> { + let url = format!("{}/files", self.api_root); + let fields = format!("nextPageToken,files({fields})"); + let mut out = Vec::new(); + let mut page_token: Option = None; + loop { + let mut query = vec![ + ("q", q), + ("fields", fields.as_str()), + ("pageSize", "1000"), + ("orderBy", "name"), + ("supportsAllDrives", "true"), + ("includeItemsFromAllDrives", "true"), + ]; + if let Some(t) = &page_token { + query.push(("pageToken", t)); + } + let v = self.get_json("files.list", &url, &query).await?; + for f in v["files"].as_array().into_iter().flatten() { + if out.len() >= limit { + return Ok((out, true)); + } + out.push(f.clone()); + } + page_token = v["nextPageToken"].as_str().map(String::from); + if page_token.is_none() { + return Ok((out, false)); + } + } + } + + /// 文件夹浏览器的第一层：我的云端硬盘、与我共享的、每个看得见的共享云端硬盘。 + /// 前两项的名字由界面按 id 换成它的语言 + pub async fn roots(&self) -> anyhow::Result> { + let mut out = vec![ + FolderEntry { + id: "root".into(), + name: "My Drive".into(), + }, + FolderEntry { + id: SHARED_WITH_ME.into(), + name: "Shared with me".into(), + }, + ]; + let url = format!("{}/drives", self.api_root); + let mut page_token: Option = None; + loop { + let mut query = vec![ + ("pageSize", "100"), + ("fields", "nextPageToken,drives(id,name)"), + ]; + if let Some(t) = &page_token { + query.push(("pageToken", t)); + } + let v = self.get_json("drives.list", &url, &query).await?; + for d in v["drives"].as_array().into_iter().flatten() { + if let (Some(id), Some(name)) = (d["id"].as_str(), d["name"].as_str()) { + out.push(FolderEntry { + id: id.into(), + name: name.into(), + }); + } + } + page_token = v["nextPageToken"].as_str().map(String::from); + if page_token.is_none() || out.len() >= MAX_FOLDERS_LISTED { + return Ok(out); + } + } + } + + /// 一个文件夹（或共享盘、或「与我共享」）下面的子文件夹 + pub async fn child_folders(&self, parent: &str) -> anyhow::Result> { + let q = if parent == SHARED_WITH_ME { + format!("sharedWithMe = true and mimeType = '{FOLDER_MIME}' and trashed = false") + } else { + anyhow::ensure!(valid_id(parent), "invalid folder id"); + format!("'{parent}' in parents and mimeType = '{FOLDER_MIME}' and trashed = false") + }; + let (items, _) = self.list(&q, "id,name", MAX_FOLDERS_LISTED).await?; + Ok(items + .iter() + .filter_map(|f| { + Some(FolderEntry { + id: f["id"].as_str()?.into(), + name: f["name"].as_str()?.into(), + }) + }) + .collect()) + } + + /// 走遍选中的文件夹，收集文件。 + /// + /// 广度优先，访问过的文件夹不再进（两个选中的文件夹互相嵌套时），文件按 id 去重 + pub async fn walk(&self, folders: &[String]) -> anyhow::Result { + let mut listing = Listing::default(); + let mut visited: HashSet = HashSet::new(); + let mut seen_files: HashSet = HashSet::new(); + let mut queue: VecDeque<(String, usize)> = VecDeque::new(); + + for id in folders { + match self.folder_is_open(id).await { + Ok(true) => queue.push_back((id.clone(), 0)), + // 进了回收站：里面的东西对这个人已经不算数了，不进去，让对账把它们标出来 + Ok(false) => {} + Err(e) if is_not_found(&e) => listing.unreachable.push(id.clone()), + Err(e) => return Err(e), + } + } + + while let Some((folder, depth)) = queue.pop_front() { + if !visited.insert(folder.clone()) { + continue; + } + let q = format!("'{folder}' in parents and trashed = false"); + let room = MAX_FILES_LISTED.saturating_sub(listing.files.len()) + MAX_FOLDERS_LISTED; + let fields = format!("{FILE_FIELDS},shortcutDetails(targetId,targetMimeType)"); + let (items, cut) = self.list(&q, &fields, room).await?; + if cut { + listing.truncated = true; + } + for item in items { + // 别人组织的会议，纪要与转写住在他的 Drive 里，这边的文件夹里只有快捷方式： + // 换成它指向的那一个，之后与普通文件一样 + let f = if item["mimeType"] == SHORTCUT_MIME { + match self.shortcut_target(&item).await { + Ok(Some(target)) => target, + Ok(None) => { + listing.skipped += 1; + continue; + } + Err(e) if is_inaccessible(&e) => { + let name = item["name"].as_str().unwrap_or("a shortcut"); + listing.broken_shortcuts.push(name.to_string()); + continue; + } + Err(e) => return Err(e), + } + } else { + item + }; + let (Some(id), Some(name), Some(mime)) = + (f["id"].as_str(), f["name"].as_str(), f["mimeType"].as_str()) + else { + continue; + }; + // Drive 回来的 id 之后要拼进查询和地址：与选中的文件夹同一个字符集，不合的不收 + if !valid_id(id) { + listing.skipped += 1; + continue; + } + if mime == FOLDER_MIME { + if depth + 1 > MAX_DEPTH { + listing.truncated = true; + } else { + queue.push_back((id.to_string(), depth + 1)); + } + continue; + } + let google_format = mime.starts_with("application/vnd.google-apps."); + if google_format && export_as(mime).is_none() { + listing.skipped += 1; + continue; + } + // Drive 的 size 是字符串里的整数；Google 格式没有 size + let size = f["size"].as_str().and_then(|s| s.parse::().ok()); + if !google_format && size.is_none_or(|n| n == 0 || n > MAX_FILE_BYTES) { + listing.skipped += 1; + continue; + } + if !seen_files.insert(id.to_string()) { + continue; + } + if listing.files.len() >= MAX_FILES_LISTED { + listing.truncated = true; + return Ok(listing); + } + let modified_raw = f["modifiedTime"].as_str().map(String::from); + listing.files.push(DriveFile { + id: id.into(), + name: name.into(), + mime: mime.into(), + modified: modified_raw + .as_deref() + .and_then(|s| DateTime::parse_from_rfc3339(s).ok()) + .map(|d| d.with_timezone(&Utc)), + md5: f["md5Checksum"].as_str().map(String::from), + modified_raw, + }); + } + } + Ok(listing) + } + + /// 快捷方式指向的那一个。指向文件夹的照文件夹走；指向文件的取目标自己的元数据， + /// 变更标记、大小、导出格式都按目标算，文件也按目标的 id 认（直接选到与经快捷方式 + /// 到达的是同一份）。目标在回收站里 → `None`；打不开 → 403/404 的错误 + async fn shortcut_target(&self, shortcut: &Value) -> anyhow::Result> { + let details = &shortcut["shortcutDetails"]; + let Some(target) = details["targetId"].as_str().filter(|id| valid_id(id)) else { + return Ok(None); + }; + if details["targetMimeType"] == FOLDER_MIME { + return Ok(Some(serde_json::json!({ + "id": target, + "name": shortcut["name"], + "mimeType": FOLDER_MIME, + }))); + } + let url = format!("{}/files/{target}", self.api_root); + let fields = format!("{FILE_FIELDS},trashed"); + let v = self + .get_json( + "files.get", + &url, + &[("fields", fields.as_str()), ("supportsAllDrives", "true")], + ) + .await?; + Ok((v["trashed"].as_bool() != Some(true)).then_some(v)) + } + + /// 选中的这个还在不在：在且没进回收站 → true；进了回收站 → false； + /// 打不开 → 404 的错误。`root` 是「我的云端硬盘」，共享盘的 id 也能当文件夹用 + async fn folder_is_open(&self, id: &str) -> anyhow::Result { + let url = format!("{}/files/{id}", self.api_root); + let v = self + .get_json( + "files.get", + &url, + &[ + ("fields", "id,mimeType,trashed"), + ("supportsAllDrives", "true"), + ], + ) + .await?; + anyhow::ensure!( + v["mimeType"] == FOLDER_MIME, + "google drive: selected item {id} is not a folder" + ); + Ok(v["trashed"].as_bool() != Some(true)) + } + + /// 一个文件的字节：Google 格式导出，其余原样下载 + pub async fn content(&self, file: &DriveFile) -> anyhow::Result { + anyhow::ensure!( + valid_id(&file.id), + "google drive: file id {:?} is not a Drive id", + file.id + ); + if let Some((mime, ext)) = export_as(&file.mime) { + let url = format!("{}/files/{}/export", self.api_root, file.id); + let resp = self.get("export", &url, &[("mimeType", mime)]).await?; + let bytes = read_capped(resp, MAX_FILE_BYTES).await?; + let filename = if file.name.to_lowercase().ends_with(&format!(".{ext}")) { + file.name.clone() + } else { + format!("{}.{ext}", file.name) + }; + return Ok(Content { + bytes, + filename, + mime, + }); + } + let url = format!("{}/files/{}", self.api_root, file.id); + let resp = self + .get( + "download", + &url, + &[("alt", "media"), ("supportsAllDrives", "true")], + ) + .await?; + // 不猜 mime，理由同对象存储：解析先看魔数、再看扩展名 + Ok(Content { + bytes: read_capped(resp, MAX_FILE_BYTES).await?, + filename: file.name.clone(), + mime: "application/octet-stream", + }) + } +} + +/// 文件夹浏览器里「与我共享」的占位 id。真的 id 里没有冒号，撞不上 +pub const SHARED_WITH_ME: &str = "shared:with-me"; + +/// Drive 的一个非 2xx 响应 +#[derive(Debug)] +struct DriveError { + status: StatusCode, + message: String, +} + +impl std::fmt::Display for DriveError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for DriveError {} + +fn is_not_found(e: &anyhow::Error) -> bool { + e.downcast_ref::() + .is_some_and(|d| d.status == StatusCode::NOT_FOUND) +} + +/// 打不开：Drive 对没共享给这个人的文件有时回 404，有时回 403 +fn is_inaccessible(e: &anyhow::Error) -> bool { + e.downcast_ref::() + .is_some_and(|d| d.status == StatusCode::NOT_FOUND || d.status == StatusCode::FORBIDDEN) +} + +/// 读一个限了体积的响应体。先看 Content-Length，再边读边数 +async fn read_capped(mut resp: reqwest::Response, limit: u64) -> anyhow::Result> { + if resp.content_length().is_some_and(|n| n > limit) { + anyhow::bail!("response is larger than {limit} bytes"); + } + let mut body = Vec::new(); + while let Some(chunk) = resp.chunk().await? { + body.extend_from_slice(&chunk); + if body.len() as u64 > limit { + anyhow::bail!("response is larger than {limit} bytes"); + } + } + Ok(body) +} + +/// 上一轮摄入时每个文件的标记 +pub async fn markers( + pool: &sqlx::PgPool, + source_id: Uuid, +) -> anyhow::Result> { + let rows: Vec<(String, String)> = + sqlx::query_as("SELECT file_id, marker FROM google_drive_files WHERE source_id = $1") + .bind(source_id) + .fetch_all(pool) + .await?; + Ok(rows.into_iter().collect()) +} + +pub async fn remember( + pool: &sqlx::PgPool, + source_id: Uuid, + file_id: &str, + marker: &str, +) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO google_drive_files (source_id, file_id, marker) VALUES ($1, $2, $3) + ON CONFLICT (source_id, file_id) DO UPDATE SET marker = EXCLUDED.marker", + ) + .bind(source_id) + .bind(file_id) + .bind(marker) + .execute(pool) + .await?; + Ok(()) +} + +/// 这一轮没出现的文件忘掉标记：它再出现时得重新读一遍，不能凭旧标记跳过。 +/// 只在遍历完整的轮次调用，与对账同一个条件 +pub async fn forget_others( + pool: &sqlx::PgPool, + source_id: Uuid, + seen: &[String], +) -> anyhow::Result<()> { + sqlx::query("DELETE FROM google_drive_files WHERE source_id = $1 AND NOT (file_id = ANY($2))") + .bind(source_id) + .bind(seen) + .execute(pool) + .await?; + Ok(()) +} + +#[cfg(test)] +#[path = "google_drive_tests.rs"] +mod tests; diff --git a/crates/utopia-server/src/google_drive_tests.rs b/crates/utopia-server/src/google_drive_tests.rs new file mode 100644 index 000000000..c248057d8 --- /dev/null +++ b/crates/utopia-server/src/google_drive_tests.rs @@ -0,0 +1,630 @@ +//! Google Drive 的遍历、导出与授权。响应的形状照 Drive v3 与 Google 令牌端点的文档写， +//! **没有对一个真实的 Drive 跑过**（0013 决定 4 要求说出来）：真连的那条在最下面， +//! 给了令牌才跑。 +use super::*; +use serde_json::json; +use wiremock::matchers::{body_string_contains, header, method, path, query_param}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +fn drive(server: &MockServer) -> Drive { + Drive { + // 本地 mock 不走开发机的系统代理 + http: reqwest::Client::builder().no_proxy().build().unwrap(), + token: std::sync::Mutex::new("ya29.test".into()), + renewal: None, + api_root: server.uri(), + } +} + +fn cfg() -> OAuthConfig { + OAuthConfig { + client_id: "client-1.apps.googleusercontent.com".into(), + client_secret: "secret".into(), + redirect: "https://utopia.example/api/v1/google-drive/callback".into(), + } +} + +fn folder_meta(id: &str, trashed: bool) -> serde_json::Value { + json!({"id": id, "mimeType": FOLDER_MIME, "trashed": trashed}) +} + +async fn mount_folder(server: &MockServer, id: &str, trashed: bool) { + Mock::given(method("GET")) + .and(path(format!("/files/{id}"))) + .respond_with(ResponseTemplate::new(200).set_body_json(folder_meta(id, trashed))) + .mount(server) + .await; +} + +async fn mount_children(server: &MockServer, parent: &str, files: serde_json::Value) { + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param( + "q", + format!("'{parent}' in parents and trashed = false"), + )) + .and(query_param("supportsAllDrives", "true")) + .and(query_param("includeItemsFromAllDrives", "true")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ "files": files }))) + .mount(server) + .await; +} + +/// Google 签的 ID token 一小时后过期；没写 `exp` 的按刚签发算 +fn id_token(mut claims: serde_json::Value) -> String { + if claims.get("exp").is_none() { + claims["exp"] = json!(Utc::now().timestamp() + 3600); + } + format!( + "{}.{}.sig", + URL_SAFE_NO_PAD.encode(br#"{"alg":"RS256"}"#), + URL_SAFE_NO_PAD.encode(claims.to_string()) + ) +} + +/// 子文件夹往下走；Google 表单这类没有正文的格式与超大文件跳过； +/// 两个选中的文件夹互相嵌套时同一个文件只收一次 +#[tokio::test] +async fn a_walk_descends_skips_what_cannot_be_read_and_counts_each_file_once() -> anyhow::Result<()> +{ + let server = MockServer::start().await; + mount_folder(&server, "top", false).await; + mount_folder(&server, "sub", false).await; + mount_children( + &server, + "top", + json!([ + {"id": "sub", "name": "Sub", "mimeType": FOLDER_MIME}, + {"id": "doc1", "name": "Plan", "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-10-01T09:30:00.000Z"}, + {"id": "form1", "name": "Survey", "mimeType": "application/vnd.google-apps.form"}, + {"id": "big", "name": "video.mp4", "mimeType": "video/mp4", "size": "999999999"} + ]), + ) + .await; + mount_children( + &server, + "sub", + json!([ + {"id": "pdf1", "name": "contract.pdf", "mimeType": "application/pdf", + "size": "1024", "md5Checksum": "abc", "modifiedTime": "2026-09-01T00:00:00Z"} + ]), + ) + .await; + + // `sub` 也被单独选中：它的文件不该出现两次 + let listing = drive(&server) + .walk(&["top".to_string(), "sub".to_string()]) + .await?; + let ids: Vec<&str> = listing.files.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(ids, ["doc1", "pdf1"]); + assert_eq!(listing.skipped, 2, "表单与超大文件"); + assert!(!listing.truncated); + assert!(listing.unreachable.is_empty()); + + let doc = &listing.files[0]; + assert_eq!(doc.external_key(), "gdrive://doc1"); + assert_eq!( + doc.marker(), + "2026-10-01T09:30:00.000Z", + "Google 格式没有 md5，用修改时刻" + ); + assert_eq!( + doc.modified.map(|d| d.to_rfc3339()), + Some("2026-10-01T09:30:00+00:00".into()) + ); + assert_eq!(listing.files[1].marker(), "abc", "二进制文件用 md5"); + Ok(()) +} + +/// Drive 回来的 id 要拼进查询与地址：不合字符表的文件夹不进、文件不收 +#[tokio::test] +async fn ids_from_drive_outside_the_id_alphabet_are_skipped() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "top", false).await; + mount_children( + &server, + "top", + json!([ + {"id": "x' or name contains '", "name": "Sneaky", "mimeType": FOLDER_MIME}, + {"id": "../../about", "name": "notes.pdf", "mimeType": "application/pdf", + "size": "10", "md5Checksum": "a"}, + {"id": "ok1", "name": "fine.pdf", "mimeType": "application/pdf", + "size": "10", "md5Checksum": "b"} + ]), + ) + .await; + + let listing = drive(&server).walk(&["top".to_string()]).await?; + let ids: Vec<&str> = listing.files.iter().map(|f| f.id.as_str()).collect(); + assert_eq!((ids, listing.skipped), (vec!["ok1"], 2)); + Ok(()) +} + +/// 别人组织的会议：文件夹里只有快捷方式，纪要住在组织者的 Drive 里。 +/// 指向文件的按目标收（目标自己的标记与时间），指向文件夹的往里走； +/// 同一个目标既被直接选到、又经快捷方式到达，只收一次 +#[tokio::test] +async fn a_shortcut_brings_in_what_it_points_to() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "meeting", false).await; + mount_folder(&server, "theirs", false).await; + mount_children( + &server, + "meeting", + json!([ + {"id": "s1", "name": "Notes by Gemini", "mimeType": SHORTCUT_MIME, + "shortcutDetails": {"targetId": "notes1", + "targetMimeType": "application/vnd.google-apps.document"}}, + {"id": "s2", "name": "Recordings", "mimeType": SHORTCUT_MIME, + "shortcutDetails": {"targetId": "theirs", "targetMimeType": FOLDER_MIME}} + ]), + ) + .await; + mount_children( + &server, + "theirs", + json!([ + {"id": "notes1", "name": "Notes by Gemini", "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-10-20T10:00:00.000Z"}, + {"id": "t1", "name": "Transcript", "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-10-20T10:05:00.000Z"} + ]), + ) + .await; + Mock::given(method("GET")) + .and(path("/files/notes1")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "notes1", "name": "Notes by Gemini", + "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-10-20T10:00:00.000Z", "trashed": false + }))) + .mount(&server) + .await; + + let listing = drive(&server).walk(&["meeting".to_string()]).await?; + let files: Vec<(&str, String)> = listing + .files + .iter() + .map(|f| (f.id.as_str(), f.marker())) + .collect(); + assert_eq!( + files, + [ + ("notes1", "2026-10-20T10:00:00.000Z".to_string()), + ("t1", "2026-10-20T10:05:00.000Z".to_string()) + ] + ); + Ok(()) +} + +/// 快捷方式指向的文件没共享给这个人：点名，不当作「没东西」 +#[tokio::test] +async fn a_shortcut_to_a_file_the_person_cannot_open_is_named() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "meeting", false).await; + mount_children( + &server, + "meeting", + json!([ + {"id": "s1", "name": "Transcript", "mimeType": SHORTCUT_MIME, + "shortcutDetails": {"targetId": "private1", + "targetMimeType": "application/vnd.google-apps.document"}} + ]), + ) + .await; + Mock::given(method("GET")) + .and(path("/files/private1")) + .respond_with(ResponseTemplate::new(404).set_body_json(json!({ + "error": {"message": "File not found: private1."} + }))) + .mount(&server) + .await; + + let listing = drive(&server).walk(&["meeting".to_string()]).await?; + assert_eq!( + (listing.files.len(), listing.broken_shortcuts), + (0, vec!["Transcript".to_string()]) + ); + Ok(()) +} + +/// 选中的文件夹打不开（删了、或不再共享）不是错误，但要点名——同步据此不对账 +#[tokio::test] +async fn an_unreachable_folder_is_named_not_fatal() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/gone")) + .respond_with(ResponseTemplate::new(404).set_body_json(json!({ + "error": {"code": 404, "message": "File not found: gone.", + "errors": [{"reason": "notFound"}]} + }))) + .mount(&server) + .await; + mount_folder(&server, "here", false).await; + mount_children(&server, "here", json!([])).await; + + let listing = drive(&server) + .walk(&["gone".to_string(), "here".to_string()]) + .await?; + assert_eq!(listing.unreachable, ["gone"]); + Ok(()) +} + +/// 进了回收站的文件夹不进去，也不算打不开：里面的东西对这个人已经不算数了 +#[tokio::test] +async fn a_trashed_folder_is_not_walked() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "binned", true).await; + let listing = drive(&server).walk(&["binned".to_string()]).await?; + assert!(listing.files.is_empty()); + assert!(listing.unreachable.is_empty()); + // 没有任何 files.list 请求 + let lists = server + .received_requests() + .await + .unwrap() + .iter() + .filter(|r| r.url.path() == "/files") + .count(); + assert_eq!(lists, 0); + Ok(()) +} + +/// 列表翻页：`nextPageToken` 一直跟到没有为止 +#[tokio::test] +async fn a_listing_follows_page_tokens() -> anyhow::Result<()> { + let server = MockServer::start().await; + mount_folder(&server, "top", false).await; + let file = |id: &str| { + json!({"id": id, "name": format!("{id}.txt"), "mimeType": "text/plain", + "size": "3", "md5Checksum": id}) + }; + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param("pageToken", "p2")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"files": [file("b")]}))) + .with_priority(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files")) + .respond_with( + ResponseTemplate::new(200) + .set_body_json(json!({"files": [file("a")], "nextPageToken": "p2"})), + ) + .with_priority(2) + .mount(&server) + .await; + + let listing = drive(&server).walk(&["top".to_string()]).await?; + let ids: Vec<&str> = listing.files.iter().map(|f| f.id.as_str()).collect(); + assert_eq!(ids, ["a", "b"]); + Ok(()) +} + +/// Google 文档走 export 换成 DOCX，文件名补扩展名；别的原样下载，不猜 mime +#[tokio::test] +async fn google_formats_are_exported_and_files_are_downloaded() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/doc1/export")) + .and(query_param( + "mimeType", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + )) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"PK-docx".to_vec())) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files/pdf1")) + .and(query_param("alt", "media")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"%PDF-1.7".to_vec())) + .mount(&server) + .await; + + let d = drive(&server); + let doc = DriveFile { + id: "doc1".into(), + name: "Plan".into(), + mime: "application/vnd.google-apps.document".into(), + modified: None, + md5: None, + modified_raw: None, + }; + let c = d.content(&doc).await?; + assert_eq!(c.bytes, b"PK-docx"); + assert_eq!(c.filename, "Plan.docx"); + + let pdf = DriveFile { + id: "pdf1".into(), + name: "contract.pdf".into(), + mime: "application/pdf".into(), + modified: None, + md5: Some("x".into()), + modified_raw: None, + }; + let c = d.content(&pdf).await?; + assert_eq!(c.bytes, b"%PDF-1.7"); + assert_eq!(c.filename, "contract.pdf"); + assert_eq!(c.mime, "application/octet-stream"); + Ok(()) +} + +/// 导出超过 10 MB 时 Google 回 403 带 `exportSizeLimitExceeded`；错误里要留着这个原因， +/// 同步把它报给人看 +#[tokio::test] +async fn an_export_refusal_keeps_googles_reason() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/huge/export")) + .respond_with(ResponseTemplate::new(403).set_body_json(json!({ + "error": {"code": 403, "message": "This file is too large to be exported.", + "errors": [{"reason": "exportSizeLimitExceeded"}]} + }))) + .mount(&server) + .await; + let f = DriveFile { + id: "huge".into(), + name: "Budget".into(), + mime: "application/vnd.google-apps.spreadsheet".into(), + modified: None, + md5: None, + modified_raw: None, + }; + let Err(e) = drive(&server).content(&f).await else { + panic!("a 403 must not read as content"); + }; + let msg = format!("{e:#}"); + assert!(msg.contains("exportSizeLimitExceeded"), "{msg}"); + assert!(msg.contains("too large"), "{msg}"); +} + +/// 文件夹浏览器：「与我共享」用 sharedWithMe 查询，别的按父文件夹查，只要文件夹 +#[tokio::test] +async fn the_folder_browser_lists_only_folders() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param( + "q", + format!("sharedWithMe = true and mimeType = '{FOLDER_MIME}' and trashed = false"), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "files": [{"id": "s1", "name": "Client X"}] + }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files")) + .and(query_param( + "q", + format!("'root' in parents and mimeType = '{FOLDER_MIME}' and trashed = false"), + )) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "files": [{"id": "f1", "name": "Projects"}] + }))) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/drives")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "drives": [{"id": "0AAdrive", "name": "Team"}] + }))) + .mount(&server) + .await; + + let d = drive(&server); + assert_eq!( + d.child_folders(SHARED_WITH_ME).await?, + [FolderEntry { + id: "s1".into(), + name: "Client X".into() + }] + ); + assert_eq!( + d.child_folders("root").await?, + [FolderEntry { + id: "f1".into(), + name: "Projects".into() + }] + ); + let roots: Vec = d.roots().await?.into_iter().map(|r| r.id).collect(); + assert_eq!(roots, ["root", SHARED_WITH_ME, "0AAdrive"]); + // 拼进 q 的 id 不收引号 + assert!(d.child_folders("x' or '1'='1").await.is_err()); + Ok(()) +} + +/// 换令牌：要刷新令牌、要 Drive 那一项权限；邮箱从 ID token 里拿 +#[tokio::test] +async fn a_code_exchange_needs_a_refresh_token_and_the_drive_scope() -> anyhow::Result<()> { + let token_ok = json!({ + "access_token": "ya29.a", "refresh_token": "1//r", "expires_in": 3599, + "scope": format!("openid {DRIVE_SCOPE} https://www.googleapis.com/auth/userinfo.email"), + "id_token": id_token(json!({ + "iss": "https://accounts.google.com", "aud": cfg().client_id, + "sub": "1", "email": "bruno@example.com", "email_verified": true + })) + }); + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("code=good")) + .and(body_string_contains("code_verifier=v")) + .respond_with(ResponseTemplate::new(200).set_body_json(token_ok.clone())) + .mount(&server) + .await; + let mut no_drive = token_ok.clone(); + no_drive["scope"] = json!("openid email"); + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("code=nodrive")) + .respond_with(ResponseTemplate::new(200).set_body_json(no_drive)) + .mount(&server) + .await; + let mut no_refresh = token_ok.clone(); + no_refresh.as_object_mut().unwrap().remove("refresh_token"); + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("code=norefresh")) + .respond_with(ResponseTemplate::new(200).set_body_json(no_refresh)) + .mount(&server) + .await; + + let http = reqwest::Client::builder().no_proxy().build()?; + let url = format!("{}/token", server.uri()); + let grant = exchange_code_at(&http, &url, &cfg(), "good", "v").await?; + assert_eq!(grant.refresh_token, "1//r"); + assert_eq!(grant.account_email.as_deref(), Some("bruno@example.com")); + + let e = exchange_code_at(&http, &url, &cfg(), "nodrive", "v") + .await + .unwrap_err(); + assert!(e.to_string().contains("read access to Drive"), "{e}"); + let e = exchange_code_at(&http, &url, &cfg(), "norefresh", "v") + .await + .unwrap_err(); + assert!(e.to_string().contains("no refresh token"), "{e}"); + Ok(()) +} + +/// 撤了授权或刷新令牌过期：说「去重连」，而不是一句 400 +#[tokio::test] +async fn a_revoked_grant_says_reconnect() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/token")) + .respond_with(ResponseTemplate::new(400).set_body_json(json!({ + "error": "invalid_grant", "error_description": "Token has been expired or revoked." + }))) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let e = access_token_at(&http, &format!("{}/token", server.uri()), &cfg(), "1//old") + .await + .unwrap_err(); + assert!(e.to_string().contains("reconnect"), "{e}"); +} + +/// 一轮同步超过一小时，访问令牌就过期了：401 换一枚新的再试，不把文件报成「读不出来」 +#[tokio::test] +async fn an_expired_access_token_is_renewed_once_mid_sync() -> anyhow::Result<()> { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/files/f1")) + .and(header("authorization", "Bearer ya29.expired")) + .respond_with(ResponseTemplate::new(401)) + .mount(&server) + .await; + Mock::given(method("POST")) + .and(path("/token")) + .and(body_string_contains("refresh_token=1%2F%2Fgrant")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"access_token": "ya29.fresh"})), + ) + .expect(1) + .mount(&server) + .await; + Mock::given(method("GET")) + .and(path("/files/f1")) + .and(header("authorization", "Bearer ya29.fresh")) + .respond_with(ResponseTemplate::new(200).set_body_bytes(b"%PDF-1.4 body".to_vec())) + .mount(&server) + .await; + let drive = Drive { + http: reqwest::Client::builder().no_proxy().build()?, + token: std::sync::Mutex::new("ya29.expired".into()), + renewal: Some(Renewal { + cfg: cfg(), + refresh_token: "1//grant".into(), + token_url: format!("{}/token", server.uri()), + }), + api_root: server.uri(), + }; + let first = drive + .get("download", &format!("{}/files/f1", server.uri()), &[]) + .await?; + let second = drive + .get("download", &format!("{}/files/f1", server.uri()), &[]) + .await?; + assert_eq!( + (first.bytes().await?.as_ref(), second.status()), + (b"%PDF-1.4 body".as_ref(), StatusCode::OK) + ); + Ok(()) +} + +/// ID token：签给别的 client 的、别的 issuer 的不收；没验证过的邮箱不显示 +#[test] +fn an_id_token_must_be_googles_and_ours() { + let ours = cfg().client_id; + let ok = id_token(json!({"iss": "accounts.google.com", "aud": ours, + "email": "a@b.c", "email_verified": true})); + assert_eq!( + id_token_email(&ok, &ours).unwrap().as_deref(), + Some("a@b.c") + ); + + let unverified = id_token(json!({"iss": "https://accounts.google.com", "aud": ours, + "email": "a@b.c", "email_verified": false})); + assert_eq!(id_token_email(&unverified, &ours).unwrap(), None); + + let other_client = id_token( + json!({"iss": "https://accounts.google.com", "aud": "someone-else", + "email": "a@b.c", "email_verified": true}), + ); + assert!(id_token_email(&other_client, &ours).is_err()); + + let other_issuer = id_token(json!({"iss": "https://evil.example", "aud": ours, + "email": "a@b.c", "email_verified": true})); + assert!(id_token_email(&other_issuer, &ours).is_err()); + + let expired = id_token(json!({"iss": "accounts.google.com", "aud": ours, + "email": "a@b.c", "email_verified": true, + "exp": Utc::now().timestamp() - 60})); + assert!(id_token_email(&expired, &ours).is_err()); +} + +/// 配置里的文件夹：对象或字符串都认，id 不合字符表的丢掉 +#[test] +fn selected_folders_keep_only_valid_ids() { + let cfg = json!({"folders": [ + {"id": "1AbC-_x", "name": "Clients"}, "root", {"id": "bad'id"}, {"name": "no id"} + ]}); + assert_eq!(selected_folders(&cfg), ["1AbC-_x", "root"]); + assert!(selected_folders(&json!({})).is_empty()); +} + +/// 真连一个 Google Drive。给了令牌才跑，CI 上永远跳过。 +/// +/// ```text +/// # 在 Google Cloud 建一个 OAuth 客户端（Web 应用），在 Utopia 里连接一个 google_drive 来源， +/// # 然后用那个客户端和一枚刷新令牌： +/// UTOPIA_GOOGLE_DRIVE_CLIENT_ID=… UTOPIA_GOOGLE_DRIVE_CLIENT_SECRET=… \ +/// UTOPIA_GOOGLE_DRIVE_REDIRECT_URI=… UTOPIA_GOOGLE_DRIVE_TEST_REFRESH_TOKEN=1//… \ +/// UTOPIA_GOOGLE_DRIVE_TEST_FOLDER= cargo test -p utopia-server google_drive +/// ``` +#[tokio::test] +async fn it_reads_from_a_real_drive() -> anyhow::Result<()> { + let (Some(cfg), Ok(refresh), Ok(folder)) = ( + oauth_config(), + std::env::var("UTOPIA_GOOGLE_DRIVE_TEST_REFRESH_TOKEN"), + std::env::var("UTOPIA_GOOGLE_DRIVE_TEST_FOLDER"), + ) else { + eprintln!("跳过：未设 UTOPIA_GOOGLE_DRIVE_TEST_REFRESH_TOKEN / _FOLDER"); + return Ok(()); + }; + let d = Drive::connect(&cfg, &refresh).await?; + assert!(!d.roots().await?.is_empty()); + let listing = d.walk(&[folder]).await?; + assert!( + !listing.files.is_empty(), + "那个文件夹里一个能读的文件都没有" + ); + let first = &listing.files[0]; + assert!(first.modified.is_some(), "modifiedTime 是 doc_time 的来源"); + let c = d.content(first).await?; + assert!(!c.bytes.is_empty()); + Ok(()) +} diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs index 114c6d635..a45359802 100644 --- a/crates/utopia-server/src/ingest_sources.rs +++ b/crates/utopia-server/src/ingest_sources.rs @@ -7,8 +7,9 @@ use crate::state::AppState; use chrono::{DateTime, Utc}; use sha2::{Digest, Sha256}; -use utopia_core::models::Source; use utopia_core::models::SourceKind; +use utopia_core::models::{Role, Source}; +use utopia_core::AppError; use uuid::Uuid; #[cfg(test)] @@ -91,6 +92,7 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() } Some(SourceKind::Webdav) => sync_webdav(state, &source).await, Some(SourceKind::Notion) => sync_notion(state, &source).await, + Some(SourceKind::GoogleDrive) => sync_google_drive(state, &source).await, // 被动容器：folder / api / memory / upload 没有拉取语义 Some( SourceKind::Folder @@ -1523,6 +1525,172 @@ async fn sync_notion(state: &AppState, source: &Source) -> anyhow::Result anyhow::Result { + use crate::google_drive; + + let refresh = source.config["refresh_token"] + .as_str() + .map(str::trim) + .filter(|s| !s.is_empty()) + .ok_or_else(|| { + anyhow::anyhow!( + "google_drive source is not connected: connect a Google account in its settings" + ) + })?; + // 读的是连上的那个人的 Drive：他离开了这个库、被降成只读、或账号停用了， + // 这份授权就不再替库读东西，等库里的人重连 + let connector = match google_drive::connected_by(&source.config) { + Some(id) => utopia_store::accounts::find_user_by_id(&state.pool, id).await?, + None => None, + }; + let access = match connector { + Some(user) => { + utopia_store::access::require_kb(&state.pool, &user, source.kb_id, Role::Editor).await + } + None => Err(AppError::NotFound), + }; + let still_editor = match access { + Ok(_) => true, + Err(AppError::NotFound | AppError::Forbidden) => false, + Err(e) => return Err(e.into()), + }; + anyhow::ensure!( + still_editor, + "the person who connected this Google Drive is no longer an editor of this knowledge base: reconnect it in the source settings" + ); + let cfg = google_drive::oauth_config().ok_or_else(|| { + anyhow::anyhow!( + "Google Drive is not configured on this server (UTOPIA_GOOGLE_DRIVE_CLIENT_ID)" + ) + })?; + let folders = google_drive::selected_folders(&source.config); + if folders.is_empty() { + anyhow::bail!("google_drive source has no folders selected"); + } + + let drive = google_drive::Drive::connect(&cfg, refresh).await?; + let listing = drive.walk(&folders).await?; + + let known = google_drive::markers(&state.pool, source.id).await?; + let mut stats = SyncStats::default(); + let mut seen_keys = Vec::with_capacity(listing.files.len()); + let mut seen_ids = Vec::with_capacity(listing.files.len()); + let mut unread: Vec = Vec::new(); + let mut downloads = 0usize; + let mut deferred = 0usize; + for f in &listing.files { + let key = f.external_key(); + seen_keys.push(key.clone()); + seen_ids.push(f.id.clone()); + let marker = f.marker(); + if known.get(&f.id) == Some(&marker) { + let live = utopia_store::documents::find_by_external_key(&state.pool, source.id, &key) + .await? + .is_some_and(|d| d.deleted_at.is_none()); + if live { + continue; + } + } + if downloads == google_drive::MAX_DOWNLOADS_PER_SYNC { + deferred += 1; + continue; + } + downloads += 1; + // 读不出来的文件跳过，库里已有的原样留着，其余照常同步——同 Notion + let content = match drive.content(f).await { + Ok(c) => c, + Err(error) => { + tracing::warn!(id = %f.id, error = %format!("{error:#}"), "google drive file could not be read, leaving the stored document as it is"); + unread.push(format!("{}: {error:#}", f.name)); + continue; + } + }; + let action = ingest_item( + state, + source.kb_id, + source.id, + &key, + &content.filename, + content.mime, + &content.bytes, + f.modified, + ) + .await?; + google_drive::remember(&state.pool, source.id, &f.id, &marker).await?; + stats.absorb(action); + } + + if listing.truncated || !listing.unreachable.is_empty() { + utopia_store::documents::clear_missing_keys(&state.pool, source.id, &seen_keys).await?; + } else { + utopia_store::documents::reconcile_missing(&state.pool, source.id, &seen_keys).await?; + google_drive::forget_others(&state.pool, source.id, &seen_ids).await?; + } + + let mut gaps: Vec = Vec::new(); + if !listing.unreachable.is_empty() { + gaps.push(format!( + "{} selected folder(s) could not be opened (deleted, or no longer shared with the connected account): {}", + listing.unreachable.len(), + listing.unreachable.join(", ") + )); + } + if !unread.is_empty() { + gaps.push(format!( + "{} could not be read and keep their stored content: {}", + unread.len(), + unread + .iter() + .take(3) + .map(String::as_str) + .collect::>() + .join("; ") + )); + } + if !listing.broken_shortcuts.is_empty() { + gaps.push(format!( + "{} shortcut(s) point to files the connected account cannot open: {}", + listing.broken_shortcuts.len(), + listing + .broken_shortcuts + .iter() + .take(3) + .map(String::as_str) + .collect::>() + .join("; ") + )); + } + if listing.truncated { + gaps.push( + "the folders go deeper or hold more files than one walk lists, so the rest were not looked at" + .into(), + ); + } + if deferred > 0 { + gaps.push(format!( + "{deferred} more file(s) wait for the next sync (at most {} downloads per run)", + google_drive::MAX_DOWNLOADS_PER_SYNC + )); + } + anyhow::ensure!( + gaps.is_empty(), + "{} file(s) synced; {}", + stats.total(), + gaps.join("; ") + ); + Ok(stats) +} + /// 标题无扩展名时按 mime 补一个，接入解析矩阵的分派。 fn ensure_extension(title: &str, mime: &str) -> String { let has_ext = title diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs index 83d113057..df3141134 100644 --- a/crates/utopia-server/src/main.rs +++ b/crates/utopia-server/src/main.rs @@ -12,6 +12,7 @@ mod extraction_open; mod github_issues; mod github_repo; mod gitlab_repo; +mod google_drive; mod governance; mod http_fetch; mod implication; diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs index a669d0151..6eb3d01c0 100644 --- a/crates/utopia-store/src/sources.rs +++ b/crates/utopia-store/src/sources.rs @@ -310,6 +310,30 @@ pub async fn update( opened(source) } +/// 只动配置里的几个键：`set` 里的写进去（密钥照样封印），`remove` 里的删掉，别的键原样。 +/// 一条语句完成，不先读后写——两个请求同时改不同的键时，谁也不会把对方刚写的盖掉 +pub async fn patch_config( + pool: &PgPool, + id: Uuid, + set: &serde_json::Value, + remove: &[&str], +) -> AppResult { + let mut set = set.clone(); + secrets::seal_json_keys(&mut set, SOURCE_SECRET_KEYS); + let source = sqlx::query_as( + "UPDATE sources SET config = + (CASE WHEN jsonb_typeof(config) = 'object' THEN config ELSE '{}'::jsonb END - $3::text[]) || $2 + WHERE id = $1 RETURNING *", + ) + .bind(id) + .bind(set) + .bind(remove) + .fetch_optional(pool) + .await? + .ok_or(AppError::NotFound)?; + opened(source) +} + /// 删除来源；其文档保留（source_id 置 NULL，落回 Uploads 组）。 pub async fn delete(pool: &PgPool, id: Uuid) -> AppResult<()> { let res = sqlx::query("DELETE FROM sources WHERE id = $1") diff --git a/crates/utopia-store/tests/store/a_secret_is_sealed_at_rest.rs b/crates/utopia-store/tests/store/a_secret_is_sealed_at_rest.rs index 279e8437d..12ff065f5 100644 --- a/crates/utopia-store/tests/store/a_secret_is_sealed_at_rest.rs +++ b/crates/utopia-store/tests/store/a_secret_is_sealed_at_rest.rs @@ -173,6 +173,32 @@ async fn a_secret_is_sealed_at_rest() -> anyhow::Result<()> { sources::get(&pool, src.id).await?.config["auth_header"], "Bearer t0k" ); + // 只改几个键的那条路（Google Drive 的连接）照样封印，别的键原样留着 + let patched = sources::patch_config( + &pool, + src.id, + &serde_json::json!({ "refresh_token": "1//grant" }), + &[], + ) + .await?; + let stored = raw( + &pool, + "SELECT config->>'refresh_token' FROM sources WHERE id = $1", + src.id, + ) + .await? + .unwrap(); + assert!(secrets::is_sealed(&stored)); + assert_eq!( + ( + &patched.config["refresh_token"], + &patched.config["auth_header"] + ), + ( + &serde_json::json!("1//grant"), + &serde_json::json!("Bearer t0k") + ) + ); sources::set_ingest_token(&pool, src.id, "utp_push_1").await?; let stored = raw( &pool, diff --git a/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md b/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md new file mode 100644 index 000000000..0ae245121 --- /dev/null +++ b/docs/decisions/0074-a-google-drive-source-is-connected-by-a-person.md @@ -0,0 +1,48 @@ +# 0074 · A Google Drive source is connected by a person + +- **Status**: Implemented · 2026-10-23 · migration 0120 · tried by hand against a real Drive; the test fixtures are written from the documentation, not recorded (0013 decision 4) · open: revision history, change-feed sync +- **Written**: 2026-10-14, revised 2026-10-23 after two reviews: the connecting person owns the selection and the connection, the cap is on downloads, a revoke spares a grant still in use (conventions in the [README](README.md)) +- **Related**: [0013](0013-a-source-should-hand-over-its-history.md) (what a source must pass, fixtures from real responses); [0014](0014-identity-from-the-person-scope-from-the-token.md) (identity comes from the person); migration `0056` (`oidc_flows`, the shape the connection flow copies); [#246](https://github.com/deeplethe/utopia/issues/246) (a credential never reaches a viewer) + +## Problem + +Most teams keep their documents in Google Drive. Today the way in is to export them by hand, or to copy them into a bucket and sync that, which loses each file's identity and its modification time. Drive passes the tests of 0013: `modifiedTime` is the content's own clock, documents are rewritten in place, a file id survives renames and moves, and the knowledge actually lives there. + +Every existing connector is configured with a key an administrator pastes in. Drive has no such key for a person's files. A service account sees only what is shared with it, so every folder would first have to be shared with a robot address, and domain-wide delegation lets one key read everyone's Drive, far more than anyone chose to include. The natural grant is the one a person gives: "read these folders of mine, as me". + +## Decisions + +1. **A person connects the source with OAuth, and it reads Drive as them.** An editor of the knowledge base opens the source's settings and clicks Connect. Google's consent screen asks for `drive.readonly` plus `openid email`, and the refresh token is stored in the source's config under `refresh_token`, which is in `SOURCE_SECRET_KEYS`: sealed at rest, never in a listing, never in a response (#246). The browser never holds a Google token. Even the folder browser asks the server, which reads Drive with the stored grant. `drive.file` was not enough: it covers only files the app created or the person picked one by one, and picking a folder does not include what is inside it. + +2. **The OAuth client belongs to the deployment.** `UTOPIA_GOOGLE_DRIVE_CLIENT_ID`, `_CLIENT_SECRET` and `_REDIRECT_URI` name a Google "web application" client the operator registers. When any of the three is missing, `GET /google-drive/status` says so, creating the kind is refused (`google_drive_not_configured`), and a sync fails with that sentence rather than a 401. This client is separate from the SSO client of 0056. The two ask for different scopes, and one deployment may sign people in elsewhere and still read Drive. + +3. **The flow is bound to the person who started it.** `POST …/google-drive/connect` records a single-use `state`, a PKCE verifier, the user and the source in `google_drive_flows` (ten minutes, at most 5 pending per person and 1000 in all, both checked in the insert itself), and returns Google's URL. The callback deletes the row before using it, requires the browser's session to be that same user, and checks again that the user is still an editor. Without the session check, someone could start a flow for their own source and get another person to click through consent, and the other person's Drive would be connected to their source. `prompt=consent` and `access_type=offline` are always sent, so a reconnect after revocation gets a new refresh token. The access token a sync works with lasts an hour and a run can last longer, so a 401 mid-run gets a new access token from the refresh token and the request is tried once more. + +4. **The ID token names the account, and that is all it does.** `openid email` exists so the settings can say "connected as bruno@…". The ID token comes straight from Google's token endpoint over TLS, so issuer, audience and expiry are checked and the signature is not (OIDC Core 3.1.3.7). An unverified email is not shown. The email never identifies a Utopia account and never links one; that stays as 0056 decided. `connected_at` in the config is what the interface reads as "connected", because the email may legitimately be absent. + +5. **The person who connected chooses the folders, and subfolders come with them.** The callback records `connected_by`. Only that person may browse (`GET …/google-drive/folders`) and save a selection (`PUT …/google-drive/folders`); another editor gets 403. Everyone in the base can read what is synced, so a colleague who could browse would be choosing what leaves someone else's Drive. For the same reason a Drive source's config is written only by its connect, select and disconnect routes: the general source create and update leave it alone. Only the person who connected may reconnect: anyone else gets 403 from `connect`, and a callback that finds someone else connected in the meantime is refused. Otherwise one click by a colleague would replace that person's Drive with their own. To change who reads, an editor disconnects first, which removes the selection with the grant (decision 11), and then connects. When the person who connected reconnects with a different Google account, the selection made in the old account is dropped and the old grant is revoked as in decision 11. A reconnect with the same account never revokes, because revoking ends the whole grant and would take the new token with it. Accounts compare by email ignoring case; when either email is unknown the account counts as changed, so the folders are dropped, but nothing is revoked. `config.folders` is `[{id, name}]`. The name is only for the interface. The browser lists My Drive, Shared with me and every shared drive. Saving the selection queues a sync at once. Ids are checked against `[A-Za-z0-9_-]` before they go into a Drive query or URL, which is simpler to trust than escaping. That holds for the ids Drive itself returns too: a child folder, file or shortcut target outside that alphabet is skipped. Each of these routes changes only its own keys in one statement (`patch_config`), so a selection saved during a callback cannot drop the token the callback just wrote. + +6. **The grant reads only while its person is an editor.** A sync first checks that `connected_by` is still an active editor of the base. When they have left, been made a viewer or been deactivated, the run fails with "reconnect it in the source settings" and reads nothing. Their Google grant would otherwise keep pulling their files into a base they no longer belong to. + +7. **The cap is on downloads, not on the listing.** A run downloads or exports at most 2000 files, the same order as WebDAV and object storage. The listing is metadata only and unchanged files cost nothing (decision 9), so it is bounded only by a guard of 50,000 files, 16 levels and 32 MB per file. A run that hits the download cap fails with how many files wait, and the next run starts where it stopped, because the files it already took now match their markers. A listing cut at the same place every time would never reach the tail (the gap in #1212). Every limit that is hit is reported, and none of them is reported as success. + +8. **Google formats are exported to Office formats.** Docs become DOCX, Sheets XLSX and Slides PPTX, and the existing parsers read them with headings, tables and every sheet. Plain text or CSV would have lost the headings that 0039's chunks and 0064's section dates depend on, and CSV keeps only the first sheet. Forms, drawings and sites have no readable body and are skipped. A shortcut is followed to what it points at: a folder is walked, and a file is taken with its own metadata, marker and id, so a file reached directly and through a shortcut is one document. This matters for meetings someone else organised, whose notes and transcripts live in the organiser's Drive and reach the person only as shortcuts. A shortcut to a file the connected account cannot open is named in the run's result, which is reported as failed, not as an empty success. Everything else is downloaded as bytes with `application/octet-stream`, because the parser trusts magic bytes over names, as object storage already does. + +9. **Change is detected from Drive's marker, not from the bytes.** An export of an unchanged Doc is a new zip with new timestamps, so a content hash changes on every run, and every run would re-extract every Doc. `google_drive_files` keeps one marker per file: `md5Checksum` for binary files, and `modifiedTime` for Google formats, which have no checksum. When the marker is unchanged and the document is live, the file is not downloaded at all. + +10. **Retraction follows a complete walk only.** Files not seen are marked "not in source" (`reconcile_missing`) only when every selected folder opened and the listing was complete. The download cap does not stop it, because the listing still saw every file. A folder that returns 404, because it was deleted or is no longer shared with the connected person, fails the run by name and marks nothing. Not being able to look is not the same as finding it empty. A file that cannot be read (Google refuses exports over 10 MB) keeps its stored document, the rest of the run continues, and the run is reported as failed with the names, as Notion does. + +11. **Disconnecting forgets the grant, and revokes it at Google when nothing else uses it.** `DELETE …/google-drive/connection` removes the token and the selection from the config first. Revoking a refresh token at Google ends the person's whole grant to our OAuth client, every token issued under it, so it would also stop their other Drive sources. The revoke is therefore sent only when no other Google Drive source holds a token for the same account (`account_email`). When the account is unknown because the email was unverified, nothing is revoked: the token is gone from Utopia, and the person can still remove the grant in their Google account. A failed revoke, by network or by a non-2xx status, is logged and does not undo the disconnect. Any editor may disconnect, since stopping a read never exposes anything. Deleting a source does not revoke: its token goes with the row, and the grant stays at Google until the person removes it there or disconnects another source of the same account. Connecting and disconnecting are audited with the account email and never the token. + +## Dead ends + +- **A service account with folders shared to it.** It is robust, and nobody's departure breaks it. But selecting a folder becomes "share it with a robot address and paste an id", and a shared drive must add the robot as a member. It may still be worth having for organisations that will not tie a sync to one person; it would be a second way to fill the same `refresh_token`-shaped slot, not a different source. +- **The Google Picker.** It needs a browser API key and an app id beside the OAuth client, puts a Google token in the browser, and with `drive.file` would still not grant a folder's contents. A server-side folder list costs one endpoint and keeps decision 1 intact. +- **Comparing content hashes**, as every other connector does. That costs a re-extraction of every Google Doc on every run (decision 9). +- **The changes feed** (`changes.list` with a saved page token). It is cheaper on large drives, but it reports changes across the whole Drive and not under a folder, so each change needs an ancestry walk, and moves out of a selected folder are the hard case. The full walk is simple and bounded. Revisit when a deployment hits the 50,000-file listing guard. + +## Open questions + +- **Revision history.** Drive keeps revisions, and 0013 prefers a source that hands its history over. Exporting old revisions of Google formats is possible but expensive; which revisions to sample is its own decision, as it is for Feishu and Confluence. +- **When the person leaves.** The grant is theirs, and a sync stops once they are no longer an editor (decision 6). The failed run raises the usual sync alert, but a dedicated "this source needs a new reader" alert would say more plainly what to do. +- **Verification by Google.** `drive.readonly` is a restricted scope. An "internal" OAuth client in a Workspace organisation needs no verification; an "external" one used beyond a hundred test users needs Google's security assessment. The operator decides that, but the setup guide should say so. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 8fc45e2d8..64b16381b 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -116,6 +116,7 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | 0054 | [A source may push statements in the open contract](0054-a-source-may-push-statements-in-the-open-contract.md) | Implemented | | | 0065 | [The OCR reader chooses its provider](0065-readers-choose-their-provider.md) | Implemented | | | 0072 | [A source checkpoint ends at a timestamp boundary](0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md) | Proposed | | +| 0074 | [A Google Drive source is connected by a person](0074-a-google-drive-source-is-connected-by-a-person.md) | Implemented | | ### [lakehouse-and-actions](../design/lakehouse-and-actions.md) diff --git a/docs/design/sources.md b/docs/design/sources.md index 0e6869d2f..8af6d16c9 100644 --- a/docs/design/sources.md +++ b/docs/design/sources.md @@ -1,6 +1,6 @@ # A source hands over its history -Records: [0013] (connectors), [0023] and [0033] (RSS), [0039] (chunks), [0040] (origin and media), +Records: [0013] (connectors), [0074] (Google Drive), [0023] and [0033] (RSS), [0039] (chunks), [0040] (origin and media), [0035] (vector index), [0036] d7 (schema documents), [0005] (sync alerts), [0022] and [0045] d3 (document dates), [0016] E. @@ -14,7 +14,7 @@ source, upload_time, none) is the document's own date; upload time is recorded t document date, so an upload-dated document counts as undated for attestation and ordering [0022, 0045, #714]. -**Connectors.** GitHub issues, Jira, Notion, RSS, WebDAV, S3 / Azure / GCS, and custom push with an +**Connectors.** GitHub issues, Jira, Notion, Google Drive, RSS, WebDAV, S3 / Azure / GCS, and custom push with an `ingest_token`; the kinds come from one `SourceKind` enum that the allowlist, the dispatcher and the web list derive from; credentials never leave the server (`SOURCE_SECRET_KEYS`) and a blank field keeps the stored value; a truncated round says so [0013]. A ticket is one document whose body @@ -26,6 +26,8 @@ changed is not fetched again, and identical files at several paths go in once [# runs past the page cap is a truncated round, and its entries carry no size, so the size cap is read from a `HEAD` on the file before anything is downloaded [#1327]. +**Google Drive.** A person connects it with OAuth and it reads Drive as them; the refresh token is a sealed source secret and the browser never holds a Google token; the flow is bound to the session that started it; only the person who connected picks folders, subfolders come along, and a sync stops once that person is no longer an editor; a run downloads at most 2000 files and says so when it stops there; Docs, Sheets and Slides are exported to DOCX, XLSX and PPTX; change is Drive's marker (`md5Checksum`, or `modifiedTime` for Google formats) because an export's bytes differ every time; "not in source" only after a complete walk [0074]. + **Statements.** A `statements` source takes the open extraction contract itself (`e`, `s`, `n`) on `POST /sources/{id}/statements`, with the `api` push's identity, versions, tombstones and run history; the payload is the document and its single chunk, and extraction parses the chunk instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement with the item as its evidence (null offsets) and reaches the typed graph only through alignment. The contract has no slot for a property or a class and unknown keys are refused; a second push under the same identity marks the earlier statements stale rather than closing them; tables belong on a mount, not here [0054]. **RSS.** Observations are not documents: one table with baseline, candidate and no_source rows; diff --git a/migrations/0120_a_google_drive_source_is_connected_by_a_person.sql b/migrations/0120_a_google_drive_source_is_connected_by_a_person.sql new file mode 100644 index 000000000..986a01318 --- /dev/null +++ b/migrations/0120_a_google_drive_source_is_connected_by_a_person.sql @@ -0,0 +1,25 @@ +-- Google Drive 来源（0074）：由一个人授权连上，按他选的文件夹同步。 +-- +-- 一次连接尝试的临时状态：授权码换令牌之前，state 与 PKCE verifier 得先落着， +-- 还要记下是谁、替哪个来源发起的——回调时这个浏览器里的会话必须还是他。 +-- 短命、一次性：回调一到就删，半途弃单的过期扫走。与 oidc_flows 同一个形状。 +CREATE TABLE IF NOT EXISTS google_drive_flows ( + state TEXT PRIMARY KEY, + verifier TEXT NOT NULL, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS google_drive_flows_expiry_idx ON google_drive_flows (expires_at); + +-- 每个文件上一次摄入时 Drive 给的变更标记：二进制文件是 md5Checksum，Google 文档 +-- 没有 md5，用 modifiedTime。**不能靠内容哈希判「没变」**：Docs/Sheets/Slides 每次 +-- 导出成 Office 格式，压缩包里的时间戳都不同，哈希每轮都变，每轮都会重新抽取。 +-- 标记没变、文档还在，就连下载都省了。 +CREATE TABLE IF NOT EXISTS google_drive_files ( + source_id UUID NOT NULL REFERENCES sources(id) ON DELETE CASCADE, + file_id TEXT NOT NULL, + marker TEXT NOT NULL, + PRIMARY KEY (source_id, file_id) +); diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index 709965c24..ff17a18dc 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -651,6 +651,7 @@ export const en = { gcs: "Google Cloud Storage", webdav: "WebDAV", notion: "Notion", + google_drive: "Google Drive", }, sourceKindHints: { folder: "Holds the files you upload: add, drag in and remove them here.", @@ -689,6 +690,10 @@ export const en = { "Syncs the pages a Notion integration can see — share a page with the integration " + "and it appears here. Dated by when the page was last edited, which is the page's " + "own clock rather than ours.", + google_drive: + "Syncs the folders you choose from your Google Drive, shared drives included. " + + "**You connect with your own Google account**, read-only; Docs, Sheets and Slides " + + "arrive as Word, Excel and PowerPoint files. Dated by each file's last modification.", api: "External systems push JSON documents here, authenticated with this source's own token.", statements: "Your system pushes statements already in the extraction contract; no model reads them. Send events, not the state of a table.", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 6f6d0a3bb..98a486d27 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -590,6 +590,7 @@ export const zh: Strings = { gcs: "Google Cloud Storage", webdav: "WebDAV", notion: "Notion", + google_drive: "Google Drive", }, sourceKindHints: { folder: "可在文件夹中管理上传的文件：上传、拖入、删除都在这里。", @@ -624,6 +625,10 @@ export const zh: Strings = { notion: "同步一个 Notion 集成能看见的页面——把页面分享给集成，它就会出现在这里。" + "日期取页面最后一次编辑的时刻，那是页面自己的时钟，不是我们抓它的时刻。", + google_drive: + "同步你在 Google Drive 里选的文件夹，共享云端硬盘也算。" + + "**用你自己的 Google 账号连接**，只读；Docs、Sheets、Slides 以 Word、Excel、PowerPoint " + + "文件进来。日期取每个文件最后一次修改的时刻。", api: "外部系统把 JSON 文档推送到这里，用这个来源自己的令牌认证。", statements: "外部系统把已经是抽取契约形状的陈述推送到这里，不经模型；发事件，别发整张表的状态。", diff --git a/web/src/pages/SourcesRail.tsx b/web/src/pages/SourcesRail.tsx index 5e9e5fe98..1be67f7d8 100644 --- a/web/src/pages/SourcesRail.tsx +++ b/web/src/pages/SourcesRail.tsx @@ -80,6 +80,7 @@ export const KIND_ICON: Record = { gcs: Cloud, webdav: FolderOpen, notion: Notebook, + google_drive: HardDrive, memory: Brain, upload: Upload, }; @@ -98,6 +99,7 @@ export const SYNCING_KINDS = new Set([ "gcs", "webdav", "notion", + "google_drive", ]); export const SYNC_DOT: Record = { diff --git a/web/src/sourceKinds.ts b/web/src/sourceKinds.ts index 3eb713d67..3174e0792 100644 --- a/web/src/sourceKinds.ts +++ b/web/src/sourceKinds.ts @@ -21,6 +21,7 @@ export const CREATABLE_SOURCE_KINDS = [ "gcs", "webdav", "notion", + "google_drive", "api", "statements", "custom", From a35edc032e471f2c9992c9542962ea071bcffaed Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Sun, 25 Oct 2026 08:25:42 +0100 Subject: [PATCH 2/7] The library can connect a Google Drive source, and only the person who connected it sees the folder browser (#1388) * A person can connect Google Drive, and only they choose which of their folders a base reads Co-authored-by: muemich Signed-off-by: Bogdan Szabo * The library can connect a Google Drive source, and only the person who connected it sees the folder browser Co-authored-by: muemich Signed-off-by: Bogdan Szabo --------- Signed-off-by: Bogdan Szabo Co-authored-by: muemich Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- web/src/api.ts | 35 ++++ web/src/i18n/en.ts | 45 +++++ web/src/i18n/zh.ts | 42 +++++ web/src/pages/GoogleDrivePanel.tsx | 278 +++++++++++++++++++++++++++++ web/src/pages/Library.tsx | 33 +++- 5 files changed, 429 insertions(+), 4 deletions(-) create mode 100644 web/src/pages/GoogleDrivePanel.tsx diff --git a/web/src/api.ts b/web/src/api.ts index 661b585c1..cd21e34c7 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -246,6 +246,15 @@ export interface SourceView { base_url?: string; /** jira_issues：项目 key，如 KAFKA；gitlab_repo：项目路径，如 group/subgroup/name */ project?: string; + /** google_drive：选中的文件夹（名字只给界面看，同步只认 id） */ + folders?: { id: string; name: string }[]; + /** google_drive：连上的 Google 账号；`connected_at` 有值即已连接（令牌从不下发） */ + account_email?: string | null; + connected_at?: string; + /** google_drive：连上的那个 Utopia 用户；浏览与选文件夹只给他 */ + connected_by?: string; + /** google_drive：连上的那个人连接时的名字，断开前的确认里用 */ + connected_by_name?: string; /** false = 这个来源下的文档只检索、不抽取（schema 文档，0035 决定 7）；缺省抽取 */ extract?: boolean; } | null; @@ -2507,6 +2516,32 @@ export const api = { method: "POST", }, ), + googleDriveStatus: () => + request<{ enabled: boolean }>(`/api/v1/google-drive/status`), + googleDriveConnect: (kbId: string, sourceId: string) => + request<{ url: string }>( + `/api/v1/kbs/${kbId}/sources/${sourceId}/google-drive/connect`, + { method: "POST" }, + ), + googleDriveDisconnect: (kbId: string, sourceId: string) => + request<{ ok: boolean }>( + `/api/v1/kbs/${kbId}/sources/${sourceId}/google-drive/connection`, + { method: "DELETE" }, + ), + googleDriveFolders: (kbId: string, sourceId: string, parent?: string) => + request<{ folders: { id: string; name: string }[] }>( + `/api/v1/kbs/${kbId}/sources/${sourceId}/google-drive/folders` + + (parent ? `?parent=${encodeURIComponent(parent)}` : ""), + ), + googleDriveSelect: ( + kbId: string, + sourceId: string, + folders: { id: string; name: string }[], + ) => + request<{ source: SourceView }>( + `/api/v1/kbs/${kbId}/sources/${sourceId}/google-drive/folders`, + { method: "PUT", body: JSON.stringify({ folders }) }, + ), syncSource: (kbId: string, sourceId: string) => request<{ queued: boolean }>( `/api/v1/kbs/${kbId}/sources/${sourceId}/sync`, diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index ff17a18dc..c728becb6 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -127,6 +127,17 @@ export const en = { "This data source is not granted to this workspace. Ask a deployment admin to grant it in Administration → Data sources.", memory_source_permanent: "The Memory source is part of the knowledge base and stays.", + google_drive_not_configured: + "Google Drive is not set up on this server. An administrator sets UTOPIA_GOOGLE_DRIVE_CLIENT_ID, _CLIENT_SECRET and _REDIRECT_URI.", + google_drive_connected_by_someone_else: + "This source reads someone else's Google Drive. Disconnect it first, then connect your own.", + google_drive_not_connector: + "Only the person who connected this Google Drive can choose its folders.", + google_drive_not_connected: "Connect a Google account first.", + google_drive_unavailable: "Google Drive could not be read.", + google_drive_busy: "Too many Google Drive connections are in progress. Try again in a few minutes.", + google_drive_too_many_folders: "Choose at most 50 folders. Choose a common parent instead.", + google_drive_invalid_folder: "That is not a Google Drive folder id.", source_name_required: "Give this source a name.", bad_cron: "That cron expression could not be parsed.", bad_cron_fields: @@ -772,6 +783,40 @@ export const en = { davPassField: "Password", notionTokenField: "Internal integration token", notionQueryField: "Search term (optional — empty takes every shared page)", + googleDrive: { + createHint: "After creating the source, connect a Google account and choose the folders.", + notConfigured: + "Google Drive is not set up on this server. An administrator sets UTOPIA_GOOGLE_DRIVE_CLIENT_ID, _CLIENT_SECRET and _REDIRECT_URI.", + account: "Google account", + connect: "Connect Google Drive", + reconnect: "Reconnect", + disconnect: "Disconnect", + waiting: "Finish in the Google window…", + connectedAs: (email: string) => `Connected as ${email}`, + connectedNoEmail: "Connected", + notConnected: "Not connected. Syncing reads Drive with the permissions of whoever connects it.", + folders: "Folders to sync", + onlyConnector: + "Only the person who connected this Google Drive can choose its folders, because they are folders from their own Drive.", + foldersHint: "Subfolders are included. Files outside these folders are never read.", + myDrive: "My Drive", + sharedWithMe: "Shared with me", + top: "Google Drive", + empty: "No folders here.", + open: "Open", + noneSelected: "No folders chosen yet.", + remove: "Remove", + saveFolders: "Save folders and sync", + saved: "Folders saved. A sync has started.", + disconnectConfirm: (account: string, mine: boolean, connector: string) => + mine + ? `Stop reading your Google Drive (${account})? Its access and the chosen folders are removed here, and connecting again starts from scratch.` + : `Stop reading the Google Drive ${connector} connected (${account})? Only ${connector} chose its folders. Its access and those folders are removed here.`, + unnamedAccount: "the connected account", + someoneElse: "someone else", + cancel: "Cancel", + disconnected: "Google Drive disconnected.", + }, tokenField: "GitHub token (optional, never shown again)", gitlabTokenField: "GitLab token with read_api (optional, never shown again)", includePullRequests: "Treat pull requests as tickets too", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index 98a486d27..f0d98729c 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -116,6 +116,15 @@ export const zh: Strings = { source_not_granted: "这个数据源没有授权给本工作区。请部署管理员在「管理 → 数据源」里授权。", memory_source_permanent: "「记忆」是知识库自带的来源，会一直在。", + google_drive_not_configured: + "这台服务器没有配置 Google Drive。需要管理员设置 UTOPIA_GOOGLE_DRIVE_CLIENT_ID、_CLIENT_SECRET 与 _REDIRECT_URI。", + google_drive_connected_by_someone_else: "这个来源读的是别人的 Google Drive。先断开，再连你自己的。", + google_drive_not_connector: "只有连上这个 Google Drive 的人能选它的文件夹。", + google_drive_not_connected: "先连接一个 Google 账号。", + google_drive_unavailable: "读不了 Google Drive。", + google_drive_busy: "同时进行的 Google Drive 连接太多，过几分钟再试。", + google_drive_too_many_folders: "最多选 50 个文件夹，可以改选它们共同的上级。", + google_drive_invalid_folder: "这不是一个 Google Drive 文件夹 id。", source_name_required: "给这个来源起个名字。", bad_cron: "这个 cron 表达式解析不了。", bad_cron_fields: "cron 表达式有五段：分 时 日 月 周。", @@ -704,6 +713,39 @@ export const zh: Strings = { davPassField: "密码", notionTokenField: "内部集成令牌", notionQueryField: "搜索词（可选——留空则取全部已分享页面）", + googleDrive: { + createHint: "建好来源之后，连接一个 Google 账号并选择文件夹。", + notConfigured: + "这台服务器没有配置 Google Drive。需要管理员设置 UTOPIA_GOOGLE_DRIVE_CLIENT_ID、_CLIENT_SECRET 与 _REDIRECT_URI。", + account: "Google 账号", + connect: "连接 Google Drive", + reconnect: "重新连接", + disconnect: "断开", + waiting: "请在 Google 的窗口里完成……", + connectedAs: (email: string) => `已连接：${email}`, + connectedNoEmail: "已连接", + notConnected: "尚未连接。同步以连接者的权限读取 Drive。", + folders: "要同步的文件夹", + onlyConnector: "只有连上这个 Google Drive 的人能选文件夹：这些是他自己 Drive 里的文件夹。", + foldersHint: "包含子文件夹。这些文件夹之外的文件一概不读。", + myDrive: "我的云端硬盘", + sharedWithMe: "与我共享", + top: "Google Drive", + empty: "这里没有文件夹。", + open: "打开", + noneSelected: "还没有选文件夹。", + remove: "移除", + saveFolders: "保存并同步", + saved: "文件夹已保存，同步已开始。", + disconnectConfirm: (account: string, mine: boolean, connector: string) => + mine + ? `停止读取你的 Google Drive（${account}）？这里会删掉它的授权和选好的文件夹，再连接要从头选。` + : `停止读取 ${connector} 连上的 Google Drive（${account}）？文件夹只有 ${connector} 选过。这里会删掉它的授权和那些文件夹。`, + unnamedAccount: "已连接的账号", + someoneElse: "别人", + cancel: "取消", + disconnected: "Google Drive 已断开。", + }, tokenField: "GitHub 令牌（可选，不再显示）", gitlabTokenField: "带 read_api 的 GitLab 令牌（可选，不再显示）", includePullRequests: "把 PR 也当工单收进来", diff --git a/web/src/pages/GoogleDrivePanel.tsx b/web/src/pages/GoogleDrivePanel.tsx new file mode 100644 index 000000000..4c79c34c6 --- /dev/null +++ b/web/src/pages/GoogleDrivePanel.tsx @@ -0,0 +1,278 @@ +/* Google Drive 来源的连接与选文件夹（0074），住在来源设置弹窗里。 + * + * 连接走弹窗而不是整页跳转：设置弹窗里的东西（改了一半的名字、日程）不会丢。 + * 回调页不带脚本，不 postMessage 回来——这边在弹窗开着的时候轮询来源列表， + * 看见 `connected_at` 变了就关掉它。人自己关了弹窗，轮询也就停了。 + * + * 浏览器从头到尾拿不到 Google 的令牌：文件夹列表是服务器以连接者的身份去取的。 + * 读的是连接者自己的 Drive，所以浏览与选文件夹只给他本人；库里别的编辑者只看得见选了什么。 */ +import { useEffect, useRef, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { ChevronRight, Folder, X } from "lucide-react"; +import { api, type SourceView } from "../api"; +import { fetchMe } from "../authSession"; +import { S } from "../i18n"; +import { toast } from "../toast"; +import { Button, Checkbox, Chip, DangerConfirm, IconButton, LinkButton } from "../ui"; + +type Entry = { id: string; name: string }; + +/** 服务器给的两个固定入口按 id 换成界面语言；其余是 Drive 里的真名 */ +function label(e: Entry): string { + if (e.id === "root") return S.library.googleDrive.myDrive; + if (e.id === "shared:with-me") return S.library.googleDrive.sharedWithMe; + return e.name; +} + +export function GoogleDrivePanel({ kbId, source }: { kbId: string; source: SourceView }) { + const G = S.library.googleDrive; + const queryClient = useQueryClient(); + const cfg = source.config ?? {}; + const connectedAt = cfg.connected_at ?? null; + const me = useQuery({ queryKey: ["me"], queryFn: fetchMe }); + const mine = Boolean(connectedAt) && cfg.connected_by === me.data?.id; + const [selected, setSelected] = useState(cfg.folders ?? []); + // 浏览路径：空 = 第一层（我的云端硬盘、与我共享、共享云端硬盘） + const [path, setPath] = useState([]); + const [waiting, setWaiting] = useState(false); + // 断开会丢掉连接者的令牌与文件夹：先问一句，说清断开的是谁的 Drive + const [confirmingDisconnect, setConfirmingDisconnect] = useState(false); + const popup = useRef(null); + const startedFrom = useRef(null); + + const status = useQuery({ + queryKey: ["google-drive-status"], + queryFn: api.googleDriveStatus, + }); + const parent = path.at(-1)?.id; + const folders = useQuery({ + queryKey: ["google-drive-folders", source.id, connectedAt, parent ?? ""], + queryFn: () => api.googleDriveFolders(kbId, source.id, parent), + enabled: mine, + retry: false, + }); + + // 弹窗开着时轮询：连上了（connected_at 变了）就关弹窗；人关了弹窗就停 + useEffect(() => { + if (!waiting) return; + const timer = window.setInterval(() => { + if (popup.current?.closed) { + setWaiting(false); + return; + } + queryClient.invalidateQueries({ queryKey: ["sources", kbId] }); + }, 1500); + return () => window.clearInterval(timer); + }, [waiting, kbId, queryClient]); + useEffect(() => { + if (waiting && connectedAt && connectedAt !== startedFrom.current) { + popup.current?.close(); + setWaiting(false); + setPath([]); + } + }, [waiting, connectedAt]); + + const connect = useMutation({ + mutationFn: async () => { + // 先同步地开一个空弹窗，再填地址：等请求回来再 window.open 会被拦 + const w = window.open("", "utopia-google-drive", "width=520,height=680"); + try { + const { url } = await api.googleDriveConnect(kbId, source.id); + if (w) w.location.href = url; + else window.location.href = url; + popup.current = w; + startedFrom.current = connectedAt; + setWaiting(true); + } catch (e) { + w?.close(); + throw e; + } + }, + onError: (e) => toast.error((e as Error).message), + }); + + const disconnect = useMutation({ + mutationFn: () => api.googleDriveDisconnect(kbId, source.id), + onSuccess: () => { + setConfirmingDisconnect(false); + toast.success(G.disconnected); + queryClient.invalidateQueries({ queryKey: ["sources", kbId] }); + }, + onError: (e) => toast.error((e as Error).message), + }); + + const save = useMutation({ + mutationFn: () => api.googleDriveSelect(kbId, source.id, selected), + onSuccess: () => { + toast.success(G.saved); + queryClient.invalidateQueries({ queryKey: ["sources", kbId] }); + }, + onError: (e) => toast.error((e as Error).message), + }); + + const isSelected = (id: string) => selected.some((f) => f.id === id); + const toggle = (e: Entry) => + setSelected((cur) => + cur.some((f) => f.id === e.id) + ? cur.filter((f) => f.id !== e.id) + : [...cur, { id: e.id, name: label(e) }], + ); + const unchanged = + JSON.stringify(selected.map((f) => f.id)) === + JSON.stringify((cfg.folders ?? []).map((f) => f.id)); + + if (status.data && !status.data.enabled) { + return

{G.notConfigured}

; + } + + return ( +
+
{G.account}
+
+ + {waiting + ? G.waiting + : connectedAt + ? cfg.account_email + ? G.connectedAs(cfg.account_email) + : G.connectedNoEmail + : G.notConnected} + +
+ {connectedAt && ( + + )} + {/* 别人连着的只能先断开再连：重连会换掉他的 Drive，服务器也会拒绝 */} + {(!connectedAt || mine) && ( + + )} +
+
+ + {confirmingDisconnect && ( + disconnect.mutate()} + onCancel={() => setConfirmingDisconnect(false)} + /> + )} + + {connectedAt && !mine && ( + <> +
{G.folders}
+

{G.onlyConnector}

+ {(cfg.folders ?? []).length === 0 ? ( +

{G.noneSelected}

+ ) : ( +
+ {(cfg.folders ?? []).map((f) => ( + {f.name} + ))} +
+ )} + + )} + + {mine && ( + <> +
{G.folders}
+

{G.foldersHint}

+ + {/* 面包屑：点哪一级回哪一级 */} +
+ setPath([])}>{G.top} + {path.map((p, i) => ( + + + setPath(path.slice(0, i + 1))}> + {label(p)} + + + ))} +
+
+ {folders.isPending ? ( +
…
+ ) : folders.isError ? ( +
{(folders.error as Error).message}
+ ) : folders.data.folders.length === 0 ? ( +
{G.empty}
+ ) : ( + folders.data.folders.map((f) => ( +
+ {/* 「与我共享」只是一个列表，不是一个能同步的文件夹 */} + {f.id === "shared:with-me" ? ( + + + {label(f)} + + ) : ( + toggle(f)} + label={label(f)} + /> + )} + setPath([...path, f])} + > + + +
+ )) + )} +
+ + {selected.length === 0 ? ( +

{G.noneSelected}

+ ) : ( +
+ {/* 点一下就从选中里拿掉 */} + {selected.map((f) => ( + toggle(f)}> + {f.name} + + + ))} +
+ )} + + + )} +
+ ); +} diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx index 22badcaa5..c6a820549 100644 --- a/web/src/pages/Library.tsx +++ b/web/src/pages/Library.tsx @@ -51,6 +51,7 @@ import { SYNCING_KINDS, type LibrarySelection, } from "./SourcesRail"; +import { GoogleDrivePanel } from "./GoogleDrivePanel"; const PAGE_SIZE = 15; @@ -801,8 +802,10 @@ export function Library() { onDone={(id, pushKind) => { setAdding(false); if (id) setSelection(id); + // Google Drive 建好直接打开设置：连接账号、选文件夹都在那里 + if (id && pushKind === "google_drive") setEditing(true); // 推送来源建好直接打开密钥弹窗（onboarding：端点 + 密钥一步拿全） - if (id && pushKind) setTokenReveal({ sourceId: id, kind: pushKind }); + else if (id && pushKind) setTokenReveal({ sourceId: id, kind: pushKind }); invalidate(); }} /> @@ -1330,7 +1333,8 @@ function SourceModal({ }: { kbId: string; /** isApi=true 时父级紧接着打开密钥弹窗 */ - /** 第二个参数：建的是推送来源时它的 kind（`api` / `statements`），别的不给 */ + /** 第二个参数：建的是推送来源时它的 kind（`api` / `statements`）、或 `google_drive` + * （建好还得连账号、选文件夹），别的不给 */ onDone: (id?: string, pushKind?: string) => void; }) { const [kind, setKind] = useState("folder"); @@ -1381,7 +1385,14 @@ function SourceModal({ kind === "azure_blob" || kind === "gcs" || kind === "webdav" || - kind === "notion"; + kind === "notion" || + kind === "google_drive"; + // 部署没配 OAuth 客户端时建出来也连不上，不让建 + const driveStatus = useQuery({ + queryKey: ["google-drive-status"], + queryFn: api.googleDriveStatus, + enabled: kind === "google_drive", + }); const create = useMutation({ mutationFn: () => { @@ -1469,7 +1480,10 @@ function SourceModal({ }); }, onSuccess: (data) => - onDone(data.source.id, kind === "api" || kind === "statements" ? kind : undefined), + onDone( + data.source.id, + kind === "api" || kind === "statements" || kind === "google_drive" ? kind : undefined, + ), }); const valid = @@ -1482,6 +1496,8 @@ function SourceModal({ ? endpoint.trim() : kind === "notion" ? notionToken.trim() + : kind === "google_drive" + ? driveStatus.data?.enabled : kind === "webdav" ? davUrl.trim() : kind === "s3" || kind === "azure_blob" || kind === "gcs" @@ -1794,6 +1810,13 @@ function SourceModal({ )} )} + {kind === "google_drive" && ( +

+ {driveStatus.data && !driveStatus.data.enabled + ? S.library.googleDrive.notConfigured + : S.library.googleDrive.createHint} +

+ )} {kind === "notion" && ( <> {field( @@ -2141,6 +2164,8 @@ function SourceEditModal({ )} + {kind === "google_drive" && } + {ingestChanged && (

{S.library.editKeepNote} From f0b31b120ccdbaa0a4b28ff98e81c6341cec5b73 Mon Sep 17 00:00:00 2001 From: Szabo Bogdan Date: Sun, 25 Oct 2026 08:30:12 +0100 Subject: [PATCH 3/7] Search answers a service with each hit's score, ranks and source, and pages (#1273) Signed-off-by: Bogdan Szabo Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-core/src/models.rs | 11 ++ crates/utopia-search/src/lib.rs | 68 +++++++- crates/utopia-server/src/api/search_routes.rs | 33 +++- .../utopia-server/src/api/token_read_tests.rs | 81 +++++++--- crates/utopia-server/src/auth.rs | 1 + crates/utopia-server/src/retrieval.rs | 149 ++++++++++++++++-- crates/utopia-server/src/retrieval_tests.rs | 130 +++++++++++++++ crates/utopia-store/src/documents.rs | 17 +- .../a_search_reads_the_base_as_it_was.rs | 4 +- ...ty-from-the-person-scope-from-the-token.md | 8 +- docs/design/access-and-audit.md | 6 +- docs/design/chat-and-mcp.md | 9 +- web/src/docs/ingest.md | 31 ++++ 13 files changed, 488 insertions(+), 60 deletions(-) diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index 890493356..178a9fffc 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -479,6 +479,17 @@ pub struct ChunkView { pub filename: String, } +/// 分块带上它来自哪个来源。`external_id` 只有推送来源（api / statements）才有： +/// 推送方给的那个 id，去掉库里加的 `{kind}:` 前缀，好让它拿回去对自己的记录（#1271） +#[derive(Debug, Clone, Serialize, sqlx::FromRow)] +pub struct SourcedChunk { + #[serde(flatten)] + #[sqlx(flatten)] + pub chunk: ChunkView, + pub source_id: Option, + pub external_id: Option, +} + #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct LlmSettings { pub workspace_id: Uuid, diff --git a/crates/utopia-search/src/lib.rs b/crates/utopia-search/src/lib.rs index 4fe45b517..f518825bd 100644 --- a/crates/utopia-search/src/lib.rs +++ b/crates/utopia-search/src/lib.rs @@ -180,28 +180,53 @@ impl SearchIndex { } } +/// 融合后的一条：RRF 总分，和它在每一路里的名次（0 起；没进那一路是 None）。 +/// 分数只在同一次查询里可比，换一句话就是另一把尺子 +#[derive(Debug, Clone, PartialEq)] +pub struct Fused { + pub id: String, + pub score: f64, + pub ranks: Vec>, +} + /// Reciprocal Rank Fusion：融合多路召回的排名（k=60 为经验常数）。 pub fn rrf_fuse(lists: &[Vec], limit: usize) -> Vec { + rrf_fuse_scored(lists, limit) + .into_iter() + .map(|fused| fused.id) + .collect() +} + +/// 同 [`rrf_fuse`]，但把分数和各路名次一并交回来 +pub fn rrf_fuse_scored(lists: &[Vec], limit: usize) -> Vec { const K: f64 = 60.0; - let mut scores: HashMap = HashMap::new(); - for list in lists { + let mut by_id: HashMap = HashMap::new(); + for (channel, list) in lists.iter().enumerate() { for (rank, id) in list.iter().enumerate() { - *scores.entry(id.clone()).or_default() += 1.0 / (K + rank as f64 + 1.0); + let fused = by_id.entry(id.clone()).or_insert_with(|| Fused { + id: id.clone(), + score: 0.0, + ranks: vec![None; lists.len()], + }); + fused.score += 1.0 / (K + rank as f64 + 1.0); + fused.ranks[channel] = Some(rank); } } - let mut ranked: Vec<(String, f64)> = scores.into_iter().collect(); + let mut ranked: Vec = by_id.into_values().collect(); // 同分按 id 定序，否则 HashMap 的随机顺序会改变上限内保留的证据和口径。 ranked.sort_by(|a, b| { - b.1.partial_cmp(&a.1) + b.score + .partial_cmp(&a.score) .unwrap_or(std::cmp::Ordering::Equal) - .then_with(|| a.0.cmp(&b.0)) + .then_with(|| a.id.cmp(&b.id)) }); - ranked.into_iter().take(limit).map(|(id, _)| id).collect() + ranked.truncate(limit); + ranked } #[cfg(test)] mod rrf_tests { - use super::rrf_fuse; + use super::{rrf_fuse, rrf_fuse_scored, Fused}; fn ids(list: &[&str]) -> Vec { list.iter().map(|s| s.to_string()).collect() @@ -222,6 +247,33 @@ mod rrf_tests { assert_eq!(rrf_fuse(&lists, 10), ids(&["a", "c", "b"])); } + /// 分数就是各路名次倒数之和，名次记在各自那一路的位置上（#1271） + #[test] + fn scored_fusion_reports_the_sum_and_each_channels_rank() { + let lists = vec![ids(&["a", "b", "c"]), ids(&["c", "a"])]; + let fused = rrf_fuse_scored(&lists, 10); + assert_eq!( + fused, + vec![ + Fused { + id: "a".into(), + score: 1.0 / 61.0 + 1.0 / 62.0, + ranks: vec![Some(0), Some(1)], + }, + Fused { + id: "c".into(), + score: 1.0 / 63.0 + 1.0 / 61.0, + ranks: vec![Some(2), Some(0)], + }, + Fused { + id: "b".into(), + score: 1.0 / 62.0, + ranks: vec![Some(1), None], + }, + ] + ); + } + /// 空的那一路不是失败的那一路：它什么分都不加，也不影响别人 #[test] fn an_empty_list_contributes_nothing() { diff --git a/crates/utopia-server/src/api/search_routes.rs b/crates/utopia-server/src/api/search_routes.rs index 0b776059e..10ae4066d 100644 --- a/crates/utopia-server/src/api/search_routes.rs +++ b/crates/utopia-server/src/api/search_routes.rs @@ -2,20 +2,23 @@ use axum::extract::{Path, State}; use axum::Json; use serde::Deserialize; use serde_json::json; -use utopia_core::models::Role; use utopia_core::AppError; use uuid::Uuid; -use crate::auth::AuthUser; +use crate::auth::SessionOrToken; use crate::error::ApiResult; use crate::retrieval; use crate::state::AppState; +/// 不认识的字段答 422：过滤条件还没做，悄悄忽略一个 `filters` 会让调用方以为它生效了 #[derive(Deserialize)] +#[serde(deny_unknown_fields)] pub struct SearchReq { pub q: String, #[serde(default)] pub top_k: Option, + #[serde(default)] + pub offset: usize, /// 记录轴（0019）：按**那一刻库里有的东西**检索。YYYY-MM-DD 或 RFC3339。 /// 命中的是当时活着的块、当时还没被删的文档 #[serde(default)] @@ -24,7 +27,7 @@ pub struct SearchReq { pub async fn search( State(state): State, - AuthUser(user): AuthUser, + reader: SessionOrToken, Path(kb_id): Path, Json(req): Json, ) -> ApiResult> { @@ -32,10 +35,28 @@ pub async fn search( if q.is_empty() { return Err(AppError::invalid("empty_query", "Search query cannot be empty").into()); } - let kb = utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Viewer).await?; + let kb = super::documents_routes::require_reader_kb(&state, &reader, kb_id).await?; let top_k = req.top_k.unwrap_or(10).min(50); + if req.offset.saturating_add(top_k) > retrieval::MAX_DEPTH { + return Err(AppError::invalid( + "page_too_deep", + format!("offset + top_k may not exceed {}", retrieval::MAX_DEPTH), + ) + .into()); + } let as_of = super::graph_routes::parse_instant("as_of", req.as_of.as_deref())?; - let chunks = retrieval::hybrid(&state, kb_id, kb.workspace_id, q, top_k, as_of, None).await?; - Ok(Json(json!({ "results": chunks }))) + let opts = retrieval::SearchOpts { + top_k, + offset: req.offset, + as_of, + }; + let page = retrieval::hybrid_scored(&state, kb_id, kb.workspace_id, q, opts, None).await?; + Ok(Json(json!({ + "results": page.results, + "offset": req.offset, + "top_k": top_k, + "has_more": page.has_more, + "vector_used": page.vector_used, + }))) } diff --git a/crates/utopia-server/src/api/token_read_tests.rs b/crates/utopia-server/src/api/token_read_tests.rs index 22887e4fe..339e7654f 100644 --- a/crates/utopia-server/src/api/token_read_tests.rs +++ b/crates/utopia-server/src/api/token_read_tests.rs @@ -1,8 +1,8 @@ //! 个人令牌读图（0014 决定 7）：搜实体、实体详情、认知历史、事实证据、图总览、邻域、本体、 -//! 某类的实例、文档的抽取结果和整库导出这些读路由收任何一枚活着的令牌。角色照旧是上限，`kb_ids` 排除了的库答 404；改实体仍只认会话。 +//! 某类的实例、文档的抽取结果、整库导出和分块检索这些读路由收任何一枚活着的令牌。角色照旧是上限，`kb_ids` 排除了的库答 404；改实体仍只认会话。 use crate::state::AppState; use axum::body::Body; -use axum::http::{Request, StatusCode}; +use axum::http::{Method, Request, StatusCode}; use serde_json::json; use std::sync::Arc; use tower::ServiceExt; @@ -87,28 +87,49 @@ impl Fixture { .1 } - fn read_paths(&self) -> [String; 10] { + fn read_paths(&self) -> [(Method, String); 11] { let (kb, person, entity, fact, doc) = (self.kb, self.person, self.entity, self.fact, self.doc); [ - format!("/kbs/{kb}/entities?q=Zhang"), - format!("/kbs/{kb}/entities/{entity}"), - format!("/kbs/{kb}/entities/{entity}/history"), - format!("/kbs/{kb}/facts/{fact}/evidence"), - format!("/kbs/{kb}/graph/overview"), - format!("/kbs/{kb}/graph/neighborhood?entity={entity}"), - format!("/kbs/{kb}/ontology"), - format!("/kbs/{kb}/ontology/entity-types/{person}/entities"), - format!("/documents/{doc}/extractions"), - format!("/kbs/{kb}/export?format=turtle"), + (Method::GET, format!("/kbs/{kb}/entities?q=Zhang")), + (Method::GET, format!("/kbs/{kb}/entities/{entity}")), + (Method::GET, format!("/kbs/{kb}/entities/{entity}/history")), + (Method::GET, format!("/kbs/{kb}/facts/{fact}/evidence")), + (Method::GET, format!("/kbs/{kb}/graph/overview")), + ( + Method::GET, + format!("/kbs/{kb}/graph/neighborhood?entity={entity}"), + ), + (Method::GET, format!("/kbs/{kb}/ontology")), + ( + Method::GET, + format!("/kbs/{kb}/ontology/entity-types/{person}/entities"), + ), + (Method::GET, format!("/documents/{doc}/extractions")), + (Method::GET, format!("/kbs/{kb}/export?format=turtle")), + (Method::POST, format!("/kbs/{kb}/search")), ] } - async fn get(&self, path: &str, token: &str) -> anyhow::Result { + /// 每个请求都带同一个检索体；GET 路由不读它 + async fn send(&self, method: Method, path: &str, token: &str) -> anyhow::Result { + self.send_json(method, path, token, r#"{"q":"Zhang"}"#) + .await + } + + async fn send_json( + &self, + method: Method, + path: &str, + token: &str, + body: &str, + ) -> anyhow::Result { let request = Request::builder() + .method(method) .uri(format!("/api/v1{path}")) .header("authorization", format!("Bearer {token}")) - .body(Body::empty())?; + .header("content-type", "application/json") + .body(Body::from(body.to_string()))?; Ok(self.app.clone().oneshot(request).await?.status()) } @@ -128,8 +149,8 @@ async fn a_read_token_of_a_base_viewer_gets_200_on_every_graph_read() -> anyhow: }; let result = async { let token = f.read_token(f.kb).await; - for path in f.read_paths() { - let status = f.get(&path, &token).await?; + for (method, path) in f.read_paths() { + let status = f.send(method, &path, &token).await?; anyhow::ensure!(status == StatusCode::OK, "{path}: {status}"); } Ok(()) @@ -146,8 +167,8 @@ async fn a_token_limited_to_another_base_gets_404_on_every_graph_read() -> anyho }; let result = async { let token = f.read_token(f.other_kb).await; - for path in f.read_paths() { - let status = f.get(&path, &token).await?; + for (method, path) in f.read_paths() { + let status = f.send(method, &path, &token).await?; anyhow::ensure!(status == StatusCode::NOT_FOUND, "{path}: {status}"); } Ok(()) @@ -168,7 +189,7 @@ async fn an_export_by_token_is_audited_under_its_owner_with_the_token_id() -> an .await? .token_id; let path = format!("/kbs/{}/export?format=turtle", f.kb); - let status = f.get(&path, &token).await?; + let status = f.send(Method::GET, &path, &token).await?; anyhow::ensure!(status == StatusCode::OK, "{path}: {status}"); let audited: (Option, serde_json::Value) = sqlx::query_as( @@ -188,3 +209,23 @@ async fn an_export_by_token_is_audited_under_its_owner_with_the_token_id() -> an f.cleanup().await?; result } + +#[tokio::test] +async fn a_search_past_depth_100_or_with_an_unknown_filters_field_gets_422() -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let token = f.read_token(f.kb).await; + let path = format!("/kbs/{}/search", f.kb); + let mut statuses = Vec::new(); + for body in [ + r#"{"q":"Zhang","top_k":50,"offset":51}"#, + r#"{"q":"Zhang","filters":{"source_id":"x"}}"#, + ] { + statuses.push(f.send_json(Method::POST, &path, &token, body).await); + } + f.cleanup().await?; + let statuses = statuses.into_iter().collect::>>()?; + assert_eq!(statuses, [StatusCode::UNPROCESSABLE_ENTITY; 2]); + Ok(()) +} diff --git a/crates/utopia-server/src/auth.rs b/crates/utopia-server/src/auth.rs index a40be2926..1dee4d37b 100644 --- a/crates/utopia-server/src/auth.rs +++ b/crates/utopia-server/src/auth.rs @@ -181,6 +181,7 @@ impl FromRequestParts for AuthUser { /// 整库导出（决定 7）。 /// /// 按推送时的 `external_id` 读回一篇文档（#1276）也是文档原件，开给令牌的理由写在决定 7。 +/// 分块检索（#1271）同样按决定 7 开给令牌。 /// /// 令牌只收窄：`pat` 在的时候，路由先问 [`Self::require_write`] 或 /// [`Self::reaches`]，角色照旧由 `require_kb` 与工作区角色判。 diff --git a/crates/utopia-server/src/retrieval.rs b/crates/utopia-server/src/retrieval.rs index 4f2cbccca..4a2938107 100644 --- a/crates/utopia-server/src/retrieval.rs +++ b/crates/utopia-server/src/retrieval.rs @@ -16,12 +16,42 @@ use crate::llm_util; use crate::state::AppState; -use utopia_core::models::ChunkView; +use serde::Serialize; +use std::collections::HashMap; +use utopia_core::models::{ChunkView, SourcedChunk}; use utopia_core::AppResult; use uuid::Uuid; const RECALL_PER_CHANNEL: usize = 24; +/// 翻页最深到第几条（#1271）：再深就是在拿检索当导出用 +pub const MAX_DEPTH: usize = 100; + +pub struct SearchOpts { + pub top_k: usize, + pub offset: usize, + pub as_of: Option>, +} + +/// 一条命中连同它的分数（#1271）。`score` 是 RRF 总分，只在同一次查询里能排先后， +/// 跨查询不可比；两个名次从 1 数，没进那一路是 None +#[derive(Debug, Serialize)] +pub struct ScoredChunk { + #[serde(flatten)] + pub chunk: SourcedChunk, + pub score: f64, + pub bm25_rank: Option, + pub vector_rank: Option, +} + +#[derive(Debug)] +pub struct SearchPage { + pub results: Vec, + pub has_more: bool, + /// 向量那一路跑了没有；false 就是模块头说的静默降级 + pub vector_used: bool, +} + pub async fn hybrid( state: &AppState, kb_id: Uuid, @@ -31,32 +61,128 @@ pub async fn hybrid( as_of: Option>, embeddings: Option<&mut llm_util::EmbedCache>, ) -> AppResult> { + let (hits, _) = fused_hits( + state, + kb_id, + workspace_id, + query, + RECALL_PER_CHANNEL, + as_of, + embeddings, + ) + .await?; + Ok(hits + .into_iter() + .take(top_k) + .map(|hit| hit.chunk.chunk) + .collect()) +} + +/// 一页命中（#1271）。每一页都向每路要同样的 `MAX_DEPTH` 条，所以各页切的是 +/// 同一张融合表：页与页不重不漏。到了 `MAX_DEPTH` 的那一页没有下一页（再翻就是 422）， +/// `has_more` 是 false +pub async fn hybrid_scored( + state: &AppState, + kb_id: Uuid, + workspace_id: Uuid, + query: &str, + opts: SearchOpts, + embeddings: Option<&mut llm_util::EmbedCache>, +) -> AppResult { + let (hits, vector_used) = fused_hits( + state, + kb_id, + workspace_id, + query, + MAX_DEPTH, + opts.as_of, + embeddings, + ) + .await?; + let end = opts.offset + opts.top_k; + let has_more = end < MAX_DEPTH && hits.len() > end; + let results = hits + .into_iter() + .skip(opts.offset) + .take(opts.top_k) + .collect(); + Ok(SearchPage { + results, + has_more, + vector_used, + }) +} + +/// 两路各取 `per_channel` 条、RRF 融合、按记录轴过滤后的整张表，连同向量那一路跑了没有 +async fn fused_hits( + state: &AppState, + kb_id: Uuid, + workspace_id: Uuid, + query: &str, + per_channel: usize, + as_of: Option>, + embeddings: Option<&mut llm_util::EmbedCache>, +) -> AppResult<(Vec, bool)> { let bm25 = { let search = state.search.clone(); let kb = kb_id.to_string(); let query = query.to_string(); - tokio::task::spawn_blocking(move || search.search(&kb, &query, RECALL_PER_CHANNEL)) + tokio::task::spawn_blocking(move || search.search(&kb, &query, per_channel)) }; // `join!` 不是 `try_join!`：向量那一路失败是降级不是错误（见 vector_channel）， // `try_join!` 会把模型故障变成检索报错 let (bm25, vector) = tokio::join!( bm25, - vector_channel(state, kb_id, workspace_id, query, as_of, embeddings) + vector_channel( + state, + kb_id, + workspace_id, + query, + per_channel, + as_of, + embeddings + ) ); let bm25 = bm25 .map_err(|e| utopia_core::AppError::Other(anyhow::anyhow!("BM25 检索线程退出：{e}")))? .map_err(utopia_core::AppError::Other)?; - let lists = channel_lists(bm25.into_iter().map(|h| h.chunk_id).collect(), vector?); + let vector = vector?; + let vector_used = vector.is_some(); + let lists = channel_lists(bm25.into_iter().map(|h| h.chunk_id).collect(), vector); // Keep the bounded channel candidates until their record-time and document // filters have run: newer hits must not consume a historical query's limit. let candidate_limit = lists.iter().map(Vec::len).sum(); - let fused = utopia_search::rrf_fuse(&lists, candidate_limit); - let ids: Vec = fused.iter().filter_map(|s| s.parse().ok()).collect(); - let mut chunks = - utopia_store::documents::chunks_by_ids(&state.pool, kb_id, &ids, as_of).await?; - chunks.truncate(top_k); - Ok(chunks) + let fused: Vec<(Uuid, utopia_search::Fused)> = + utopia_search::rrf_fuse_scored(&lists, candidate_limit) + .into_iter() + .filter_map(|f| Some((f.id.parse().ok()?, f))) + .collect(); + let ids: Vec = fused.iter().map(|(id, _)| *id).collect(); + let live = utopia_store::documents::chunks_by_ids(&state.pool, kb_id, &ids, as_of).await?; + let mut fused: HashMap = fused.into_iter().collect(); + let hits = live + .into_iter() + .filter_map(|chunk| { + let f = fused.remove(&chunk.chunk.id)?; + Some(ScoredChunk { + chunk, + score: f.score, + bm25_rank: position(&f.ranks, 0), + vector_rank: position(&f.ranks, 1), + }) + }) + .collect(); + Ok((hits, vector_used)) +} + +/// 第 `channel` 路里的名次，从 1 数 +fn position(ranks: &[Option], channel: usize) -> Option { + ranks + .get(channel) + .copied() + .flatten() + .map(|rank| rank as u32 + 1) } /// 向量那一路。没配嵌入模型、嵌入请求失败、回了空，都是 `Ok(None)`：检索照常， @@ -70,6 +196,7 @@ async fn vector_channel( kb_id: Uuid, workspace_id: Uuid, query: &str, + per_channel: usize, as_of: Option>, cache: Option<&mut llm_util::EmbedCache>, ) -> AppResult>> { @@ -101,7 +228,7 @@ async fn vector_channel( &state.pool, kb_id, &vector, - RECALL_PER_CHANNEL as i64, + per_channel as i64, as_of, ) .await?; diff --git a/crates/utopia-server/src/retrieval_tests.rs b/crates/utopia-server/src/retrieval_tests.rs index aa85174d7..5f7d9da64 100644 --- a/crates/utopia-server/src/retrieval_tests.rs +++ b/crates/utopia-server/src/retrieval_tests.rs @@ -182,6 +182,41 @@ async fn both_channels_fuse_with_bm25_first() -> anyhow::Result<()> { .mount(&server) .await; f.point_embeddings_at(&server).await?; + let source = Uuid::now_v7(); + sqlx::raw_sql(&format!( + "INSERT INTO sources(id,kb_id,kind,name) VALUES ('{source}','{}','api','ci'); + UPDATE documents SET source_id='{source}', external_key='api:obs-7' WHERE id='{}'", + f.kb, f.doc + )) + .execute(&f.pool) + .await?; + let opts = super::SearchOpts { + top_k: 8, + offset: 0, + as_of: None, + }; + let page = super::hybrid_scored(&f.state, f.kb, f.ws, "apple harvest", opts, None).await?; + let hits: Vec<_> = page + .results + .iter() + .map(|h| { + ( + h.score, + h.bm25_rank, + h.vector_rank, + h.chunk.source_id, + h.chunk.external_id.as_deref(), + ) + }) + .collect(); + assert!(page.vector_used); + assert_eq!( + hits, + vec![ + (2.0 / 61.0, Some(1), Some(1), Some(source), Some("obs-7")), + (1.0 / 62.0, None, Some(2), Some(source), Some("obs-7")), + ] + ); let ids = f.search("apple harvest").await?; assert_eq!( ids, @@ -264,3 +299,98 @@ async fn historical_search_keeps_live_candidates_before_limiting_results() -> an ); Ok(()) } + +/// `count` chunks replace the fixture's two (#1271): BM25 ranks them c0 first (more +/// "apple" in the same length), the vector channel ranks them the other way round. +async fn seed_ranked_chunks(f: &Fixture, count: usize) -> anyhow::Result { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({ + "data": [{ "embedding": [1.0, 0.0, 0.0, 0.0] }] + }))) + .mount(&server) + .await; + f.point_embeddings_at(&server).await?; + sqlx::query("DELETE FROM chunks WHERE document_id=$1") + .bind(f.doc) + .execute(&f.pool) + .await?; + let mut indexed = Vec::new(); + for seq in 0..count { + let id = Uuid::now_v7(); + let text = std::iter::repeat_n("apple", count - seq) + .chain(std::iter::repeat_n("pear", seq)) + .collect::>() + .join(" "); + sqlx::query( + "INSERT INTO chunks(id,kb_id,document_id,seq,text,embedding) + VALUES($1,$2,$3,$4,$5,$6::text::vector)", + ) + .bind(id) + .bind(f.kb) + .bind(f.doc) + .bind(seq as i32) + .bind(&text) + .bind(format!("[{},1,0,0]", seq + 1)) + .execute(&f.pool) + .await?; + indexed.push((id.to_string(), text)); + } + f.state + .search + .reindex_document(&f.kb.to_string(), &f.doc.to_string(), &indexed)?; + Ok(server) +} + +fn page(offset: usize, top_k: usize) -> super::SearchOpts { + super::SearchOpts { + top_k, + offset, + as_of: None, + } +} + +/// Pages slice one fused list (#1271). A page asking each channel for fewer +/// candidates than a deeper page fuses a different list, so two pages of fifteen +/// would not join into one of thirty. +#[tokio::test] +async fn two_pages_of_fifteen_join_into_the_first_thirty_hits() -> anyhow::Result<()> { + let Some(f) = fixture().await? else { + return Ok(()); + }; + let _server = seed_ranked_chunks(&f, 30).await?; + let walk = |opts| super::hybrid_scored(&f.state, f.kb, f.ws, "apple", opts, None); + let first = walk(page(0, 15)).await?; + let second = walk(page(15, 15)).await?; + let whole = walk(page(0, 30)).await?; + let ids = + |hits: &[super::ScoredChunk]| hits.iter().map(|h| h.chunk.chunk.id).collect::>(); + let joined = [ids(&first.results), ids(&second.results)].concat(); + f.cleanup().await?; + assert_eq!(whole.results.len(), 30); + assert_eq!( + joined, + ids(&whole.results), + "no hit repeats and none is skipped" + ); + assert_eq!( + (first.has_more, second.has_more, whole.has_more), + (true, false, false) + ); + Ok(()) +} + +/// The next page past depth 100 is a 422, so the page ending there has no more +#[tokio::test] +async fn a_page_ending_at_depth_100_of_101_hits_has_no_more() -> anyhow::Result<()> { + let Some(f) = fixture().await? else { + return Ok(()); + }; + let _server = seed_ranked_chunks(&f, 101).await?; + let last = super::hybrid_scored(&f.state, f.kb, f.ws, "apple", page(50, 50), None).await; + f.cleanup().await?; + let last = last?; + assert_eq!((last.results.len(), last.has_more), (50, false)); + Ok(()) +} diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs index 45c12acf1..6f1e30fe4 100644 --- a/crates/utopia-store/src/documents.rs +++ b/crates/utopia-store/src/documents.rs @@ -2,7 +2,7 @@ use chrono::{DateTime, Utc}; use pgvector::Vector; use serde::Serialize; use sqlx::{PgPool, Postgres, Transaction}; -use utopia_core::models::{ChunkView, Document, DocumentPage}; +use utopia_core::models::{ChunkView, Document, DocumentPage, SourcedChunk}; use utopia_core::{AppError, AppResult}; use utopia_ingest::ChunkPiece; use uuid::Uuid; @@ -1928,7 +1928,7 @@ pub async fn vector_search( Ok(rows.into_iter().map(|(id,)| id).collect()) } -/// 按 id 集合取分块（带文档名），保持传入顺序。 +/// 按 id 集合取分块（带文档名与来源），保持传入顺序。 /// /// `as_of`：记录轴（0019）——那一刻还活着的块、还没被删的文档。文档的删除 /// 同样是记录轴上的事件（#268 留了墓碑），所以两个条件一起倒。 @@ -1937,10 +1937,13 @@ pub async fn chunks_by_ids( kb_id: Uuid, ids: &[Uuid], as_of: Option>, -) -> AppResult> { - let rows: Vec = sqlx::query_as(&format!( - "SELECT c.id, c.document_id, c.seq, c.text, d.filename +) -> AppResult> { + let rows: Vec = sqlx::query_as(&format!( + "SELECT c.id, c.document_id, c.seq, c.text, d.filename, d.source_id, + CASE WHEN s.kind IN ('api', 'statements') + THEN substr(d.external_key, length(s.kind) + 2) END AS external_id FROM chunks c JOIN documents d ON d.id = c.document_id + LEFT JOIN sources s ON s.id = d.source_id WHERE c.kb_id = $1 AND c.id = ANY($2) AND {live} AND {doc_live}", live = crate::record_axis::chunk_live_at("c", as_of.map(|_| 3)), @@ -1952,8 +1955,8 @@ pub async fn chunks_by_ids( .fetch_all(pool) .await?; // 恢复 RRF 排名顺序 - let mut by_id: std::collections::HashMap = - rows.into_iter().map(|c| (c.id, c)).collect(); + let mut by_id: std::collections::HashMap = + rows.into_iter().map(|c| (c.chunk.id, c)).collect(); Ok(ids.iter().filter_map(|id| by_id.remove(id)).collect()) } diff --git a/crates/utopia-store/tests/store/a_search_reads_the_base_as_it_was.rs b/crates/utopia-store/tests/store/a_search_reads_the_base_as_it_was.rs index 7e0826033..6baccb786 100644 --- a/crates/utopia-store/tests/store/a_search_reads_the_base_as_it_was.rs +++ b/crates/utopia-store/tests/store/a_search_reads_the_base_as_it_was.rs @@ -182,7 +182,7 @@ async fn a_superseded_chunk_keeps_its_vector_and_answers_as_of_then() -> anyhow: None, ) .await?; - let ids: Vec = live_now.iter().map(|c| c.id).collect(); + let ids: Vec = live_now.iter().map(|c| c.chunk.id).collect(); assert!( !ids.contains(&f.deleted_doc_chunk), "删掉的文档不出现在现在" @@ -196,7 +196,7 @@ async fn a_superseded_chunk_keeps_its_vector_and_answers_as_of_then() -> anyhow: Some(t("2026-05-15T00:00:00Z")), ) .await?; - let ids: Vec = live_then.iter().map(|c| c.id).collect(); + let ids: Vec = live_then.iter().map(|c| c.chunk.id).collect(); assert!( ids.contains(&f.deleted_doc_chunk), "五月中那篇文档还没被删，它的块该拿得回来" diff --git a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md index 8c10d626a..9825b075f 100644 --- a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md +++ b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md @@ -1,6 +1,6 @@ # 0014 · Identity from the person, scope from the token -- **Status**: Implemented · 2026-10-16 · #180, #318, #1243, #1262 · open: tokens across workspaces +- **Status**: Implemented · 2026-10-16 · #180, #318, #1243, #1262, #1271 · open: tokens across workspaces - **Written**: 2026-09-01 · condensed into English 2026-09-03 - **Related**: migration `0014_data_source_grants` gave data sources a grant layer; this is the same question where a machine knocks. [0004](0004-language-and-localization.md) has the server speak English only, MCP error codes included. [0015](0015-recording-a-sentence-is-not-asserting-a-fact.md) removes the main objection to opening `remember` over MCP. @@ -120,6 +120,10 @@ push documents in. mirror to hold a write token for a read-only job. The control is the `kb.exported` audit row, which names the owner and, through `via_token`, the token. + Chunk search (`POST /kbs/{id}/search`, added 2026-10-24, #1271) takes a token on the same + terms, so the service can ask the base about what it mirrored. Each hit carries its score, its + BM25 and vector ranks and its source; `offset` pages through one fused list, at most 100 deep. + ## Dead ends - **Base-level machine tokens**, one per base and unrelated to a person. It would be a third @@ -148,7 +152,7 @@ push documents in. and document content" now also counts them. 2026-10-19: the graph overview and neighbourhood join them, and so does reading a pushed document back by its `external_id` (#1276). The ontology and a class's instances follow (#1265), then a document's extractions and the - base's export (#1266). + base's export (#1266). 2026-10-24: chunk search joins them (#1271). ## Open questions diff --git a/docs/design/access-and-audit.md b/docs/design/access-and-audit.md index 2f533da5a..6353d9a34 100644 --- a/docs/design/access-and-audit.md +++ b/docs/design/access-and-audit.md @@ -19,10 +19,10 @@ writes an audit row `mcp.tool_called`; the tokens page shows the plaintext once snippet [0014]. A `write` token also sets up a base as its owner: creating a base and a `statements` or `api` source, reading the source's push key, importing an ontology and deciding a kind word, with the owner's role unchanged as the ceiling; any token lists a workspace's bases; a token limited by `kb_ids` -creates no base. These routes, document content and eight graph reads (entity search, detail, +creates no base. These routes, document content, eight graph reads (entity search, detail, history, fact evidence, overview, neighbourhood, ontology, a class's instances; Viewer suffices, -`kb_ids` answers 404), a document's extractions and the base's export (its `kb.exported` row -names the token) take `SessionOrToken`; apart from MCP, every other route is session-only [0014 d6, d7]. `ingest_token` on a source is a different kind: plaintext, push-only [0014]. +`kb_ids` answers 404), a document's extractions, the base's export (its `kb.exported` row +names the token) and chunk search take `SessionOrToken`; apart from MCP, every other route is session-only [0014 d6, d7]. `ingest_token` on a source is a different kind: plaintext, push-only [0014]. **A client reads a pushed document back by its `external_id`.** `GET /kbs/{id}/sources/{sid}/documents/by-external-id?external_id=…` finds an `api` or `statements` diff --git a/docs/design/chat-and-mcp.md b/docs/design/chat-and-mcp.md index d29b880cf..eaa3fbe57 100644 --- a/docs/design/chat-and-mcp.md +++ b/docs/design/chat-and-mcp.md @@ -72,7 +72,14 @@ sources for the citation numbers it repeats (#943). "now" only; neither takes `at` [0035, 0019, 0022]. Confirmed mappings and a schema document reach the prompt through retrieval; a conventions document works today where a rule would be exact [0011, 0036]. A turn embeds its question once: choosing the mappings, `search_chunks` and the -graph lookups share the turn's embedding cache (#971). +graph lookups share the turn's embedding cache (#971). The search route answers with each hit's RRF score, its +1-based rank in each channel, `vector_used` (false when the vector channel was skipped) and +`offset`/`has_more` paging to a depth of 100; a pushed source's hit carries its `external_id`. An +RRF score ranks hits within one query only; it is not comparable across queries (#1271). Every +page fuses 100 candidates per channel where chat fuses 24, so the search page can rank the same +query differently from chat and from before #1271. Pages join up only while the base is unchanged +between requests, and each channel counts its ranks before the final `as_of` filter, so a +historical page can skip rank numbers. **`remember`** writes a memory document at once; its statements go through open extraction and wait in `pending_facts`; the assistant says the sentence is recorded and its statements will be shown diff --git a/web/src/docs/ingest.md b/web/src/docs/ingest.md index 35bc59479..b577f0251 100644 --- a/web/src/docs/ingest.md +++ b/web/src/docs/ingest.md @@ -197,6 +197,37 @@ The credential is a personal access token (or a signed-in session) of someone wh `content` is the current version's text; older versions are at `GET /api/v1/documents/{id}/content?version=N`. An identity marked "Not in source", a document in the trash or purged, and an identity this source never pushed are all `404`. +## Searching what you pushed + +A service can search the base's chunks with the same personal access token. The push token is not accepted here either. + +``` +POST {your-utopia-base}/api/v1/kbs/{kb_id}/search +Authorization: Bearer utp_pat_… +Content-Type: application/json + +{ "q": "harvest report", "top_k": 20, "offset": 0 } +``` + +`q` is required. `top_k` defaults to 10 and is capped at 50; `offset` defaults to 0. `as_of` (an RFC 3339 instant) reads the base as it was then; ranks are counted before `as_of` filters the hits. Any other field is a `422`. + +```json +{ + "results": [ + { + "id": "0199…", "document_id": "0199…", "seq": 3, "text": "…", "filename": "obs-000412.json", + "source_id": "0199…", "external_id": "obs-000412", + "score": 0.0328, "bm25_rank": 1, "vector_rank": 2 + } + ], + "offset": 0, "top_k": 20, "has_more": true, "vector_used": true +} +``` + +`external_id` is the identity you pushed the document under, so you can match a hit to your own record; it is `null` for documents that did not come from an API or Statements push. `score` orders hits within this one query and means nothing across queries. `bm25_rank` and `vector_rank` count from 1 and are `null` when the hit was not in that channel. `vector_used` is `false` when no embedding model answered and the hits are keyword-only. + +Page by raising `offset` by `top_k` while `has_more` is `true`. Pages stop at a depth of 100: a request with `offset + top_k` above 100 is a `422` (`page_too_deep`), and the page that reaches 100 has `has_more: false`. Pages join without repeats only while the base does not change between requests. + --- ## Shared semantics From a321b3f136d8bb542501caf0271f19e14a40661c Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:36:18 -0400 Subject: [PATCH 4/7] A MySQL YEAR or BIT column reads as its number, so a fiscal year of 2024 is not null (#1410) Signed-off-by: Rudy Celekli Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- .../utopia-server/src/query_engine/mysql.rs | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/utopia-server/src/query_engine/mysql.rs b/crates/utopia-server/src/query_engine/mysql.rs index cf2df7086..df25f7d42 100644 --- a/crates/utopia-server/src/query_engine/mysql.rs +++ b/crates/utopia-server/src/query_engine/mysql.rs @@ -88,6 +88,8 @@ fn cell_kind(type_name: &str) -> Cell { match ty.as_str() { "BOOLEAN" | "BOOL" => Cell::Bool, "TINYINT" | "SMALLINT" | "MEDIUMINT" | "INT" | "BIGINT" => Cell::Int, + // 名字里没有 UNSIGNED，驱动却只认 u64：i64 与 String 都读不出，整列变 null + "YEAR" | "BIT" => Cell::UnsignedInt, "FLOAT" | "DOUBLE" => Cell::Float, "DECIMAL" => Cell::Decimal, "JSON" => Cell::Json, @@ -469,6 +471,24 @@ mod tests { assert!(super::super::guard_sql_for("mysql", "DELETE FROM sales.orders").is_err()); } + /// YEAR 与 BIT 只有无符号整数读得出：字符串、字节都不收它们，落进兜底档就整列 null + #[tokio::test] + async fn a_year_or_bit_column_reads_as_its_number_mysql() { + let Some(fx) = Fx::new(1).await else { return }; + let a = fx.schemas[0].clone(); + fx.exec(&format!( + "CREATE TABLE {a}.budget (fiscal YEAR, active BIT(1), mask BIT(8)); + INSERT INTO {a}.budget VALUES (2024, b'1', b'00000101');" + )) + .await; + let r = MysqlEngine::new(&fx.url) + .execute(&format!("SELECT fiscal, active, mask FROM {a}.budget")) + .await + .expect("execute"); + assert_eq!(r.rows, vec![r#"{"fiscal":2024,"active":1,"mask":5}"#]); + fx.cleanup().await; + } + /// 每个测试自己的库，名字带随机后缀：并行跑不撞，也不碰服务器上已有的库。 /// 测完逐个 `DROP DATABASE`。**这是 MySQL 的「schema」，跟 PG 的命名空间 /// 不同——MySQL 的 schema 就是 database**（见 fetch_schema 顶上的注释） @@ -730,6 +750,10 @@ mod tests { ] { assert_eq!(cell_kind(t), Cell::UnsignedInt, "{t}"); } + // YEAR 与 BIT 的类型名不带 UNSIGNED，驱动却只肯用无符号整数读它们 + for t in ["YEAR", "BIT"] { + assert_eq!(cell_kind(t), Cell::UnsignedInt, "{t}"); + } // DECIMAL 被驱动同时挡在 f64 与 String 之外，只有 BigDecimal 读得出。 // 金额列几乎都是这个类型，掉档的代价最大 assert_eq!(cell_kind("DECIMAL"), Cell::Decimal); From 22d4ae92f3c5cd3a55ba5c0d4a2392795f325876 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:41:17 -0400 Subject: [PATCH 5/7] =?UTF-8?q?A=20typeset=20minus=20sign=20reads=20as=20n?= =?UTF-8?q?egative,=20so=20"=E2=88=923.2%"=20becomes=20-3.2=20instead=20of?= =?UTF-8?q?=20no=20value=20(#1416)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rudy Celekli Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-extract/src/lib.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs index b1c1f63e9..a87277bf9 100644 --- a/crates/utopia-extract/src/lib.rs +++ b/crates/utopia-extract/src/lib.rs @@ -492,14 +492,18 @@ fn scan_quantity(s: &str, strict: bool) -> Option { } } } - // 2. 数字：前导的 [-+0-9.,_] + // 2. 数字：前导的 [-+0-9.,_]，负号也可以是排版用的 U+2212 let num_end = rest .char_indices() - .find(|(_, c)| !matches!(c, '0'..='9' | '.' | ',' | '_' | '-' | '+')) + .find(|(_, c)| !matches!(c, '0'..='9' | '.' | ',' | '_' | '-' | '+' | '−')) .map(|(i, _)| i) .unwrap_or(rest.len()); let (num, after) = rest.split_at(num_end); - let cleaned: String = num.chars().filter(|c| !matches!(c, ',' | '_')).collect(); + let cleaned: String = num + .chars() + .filter(|c| !matches!(c, ',' | '_')) + .map(|c| if c == '−' { '-' } else { c }) + .collect(); let mut n: f64 = cleaned.parse().ok()?; // 3. 数字后面：紧贴或空格隔开的量级词、货币词，逐个吃；吃不动的就是「其余」 let mut tail = after.trim_start(); @@ -1216,6 +1220,19 @@ mod tests { normalize_attr_value("number", &json!("52%")), Some(json!(52.0)) ); + // 排版过的负号是 U+2212（维基百科、排版出来的 PDF 都这么写），读法同 `-` + assert_eq!( + normalize_attr_value("number", &json!("−3.2%")), + Some(json!(-3.2)) + ); + assert_eq!( + normalize_attr_value("number", &json!("−1,250")), + Some(json!(-1250.0)) + ); + assert_eq!( + attribute_value("number", Some("万元"), &json!("−2.3亿元")), + Some(json!(-23000.0)) + ); // 原来就认的两种写法不受影响 assert_eq!( normalize_attr_value("number", &json!("35,000")), From ae2449bf9ccb62dcc7278d301f29dda31aea8746 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:45:43 -0400 Subject: [PATCH 6/7] A declared unit with its currency first is read in its magnitude, so "$5 billion" under "USD million" is 5000 (#1417) Signed-off-by: Rudy Celekli Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-extract/src/lib.rs | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs index a87277bf9..4653f8301 100644 --- a/crates/utopia-extract/src/lib.rs +++ b/crates/utopia-extract/src/lib.rs @@ -762,10 +762,14 @@ pub fn attribute_value( return normalize_attr_value(datatype, raw); }; // 声明的单位拆成量级和其余：「万元」→ (4, 元)，「million dollars」→ (6, dollars) + // 币种写在前面的「USD million」「€ million」，量级在末尾 let (tok, rest) = next_token(declared); + let trailing = declared + .rsplit_once(char::is_whitespace) + .and_then(|(head, last)| Some((magnitude(&last.to_ascii_lowercase())?, head.trim()))); let (exponent, base) = match magnitude(&tok.to_ascii_lowercase()) { Some(m) => (m, rest.trim()), - None => (0, declared), + None => trailing.unwrap_or((0, declared)), }; let written = q.unit.as_deref().and_then(currency_unit); let wanted = currency_unit(base); @@ -1531,6 +1535,24 @@ mod an_attribute_value_is_read_in_its_declared_unit { assert_eq!(value("date", None, "1982年"), Some(json!("1982"))); } + /// 财报的单位常把币种写在量级前面：「USD million」「€ million」，同「million dollars」 + #[test] + fn a_unit_with_its_currency_first_is_still_read_in_millions() { + assert_eq!( + value("number", Some("USD million"), "$5 billion"), + Some(json!(5000.0)) + ); + assert_eq!( + value("number", Some("€ million"), "€30 million"), + Some(json!(30.0)) + ); + assert_eq!(value("number", Some("USD million"), "€30 million"), None); + assert_eq!( + value("number", Some("million dollars"), "$5 billion"), + Some(json!(5000.0)) + ); + } + /// 换不了的不换：货币对不上、读不出数的，宁缺勿脏 #[test] fn what_cannot_be_read_is_not_written() { From 2eb32b6afd30feef5e6ef2f0055f7f535e0e6546 Mon Sep 17 00:00:00 2001 From: Rudy Mizrahi Celekli <47457359+rudycelekli@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:51:15 -0400 Subject: [PATCH 7/7] =?UTF-8?q?An=20ontology=20term=20named=20in=20Chinese?= =?UTF-8?q?=20takes=20its=20key=20from=20its=20English=20label,=20so=20ex:?= =?UTF-8?q?=E5=85=AC=E5=8F=B8=20labelled=20"Company"=20imports=20as=20comp?= =?UTF-8?q?any=20(#1419)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rudy Celekli Co-authored-by: WaylandYang <145302500+WaylandYang@users.noreply.github.com> --- crates/utopia-ingest/src/ontology_rdf.rs | 34 ++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/crates/utopia-ingest/src/ontology_rdf.rs b/crates/utopia-ingest/src/ontology_rdf.rs index 824e5a0d0..cb16586b4 100644 --- a/crates/utopia-ingest/src/ontology_rdf.rs +++ b/crates/utopia-ingest/src/ontology_rdf.rs @@ -840,13 +840,20 @@ fn is_opaque_local_name(local: &str) -> bool { } /// 派生 key：局部名是纯编号且词汇表给了 label 时用 label，否则用局部名。 +/// 局部名派生不出 key 的（`ex:公司` 一个 ASCII 字母都没有）也改用 label。 /// /// **IRI 始终是身份**，这里只换那个给模型读的标签，所以重导入、对齐、 /// `e_by_iri` 的认领都不受影响。 pub fn key_from_iri_or_label(iri: &str, label: Option<&str>) -> String { + let from_iri = key_from_iri(iri); match label { - Some(l) if is_opaque_local_name(local_name(iri)) && !l.trim().is_empty() => to_key(l), - _ => key_from_iri(iri), + Some(l) + if (from_iri.is_empty() || is_opaque_local_name(local_name(iri))) + && !l.trim().is_empty() => + { + to_key(l) + } + _ => from_iri, } } @@ -1292,6 +1299,29 @@ mod tests { assert_eq!(k("http://x#part_12", "part twelve"), "part_12"); } + /// 中文写的 IRI 派生不出 key：有英文 label 就用它，不然每个类都是空 key，导入整个被拒 + #[test] + fn a_chinese_iri_takes_its_key_from_the_label() { + let ttl = r#" +@prefix rdfs: . +@prefix owl: . +@prefix xsd: . +@prefix ex: . + +ex:公司 a owl:Class ; rdfs:label "Company"@en, "公司"@zh . +ex:人员 a owl:Class ; rdfs:label "Person"@en, "人员"@zh . +ex:注册资本 a owl:DatatypeProperty ; rdfs:label "registered capital"@en ; + rdfs:domain ex:公司 ; rdfs:range xsd:decimal . +"#; + let p = project(ttl.as_bytes(), RdfFormat::Turtle).unwrap(); + let mut keys: Vec<&str> = p.classes.iter().map(|c| c.key.as_str()).collect(); + keys.sort(); + assert_eq!(keys, ["company", "person"]); + assert_eq!(p.properties[0].key, "registered_capital"); + // 没有 label 的仍退回局部名 + assert_eq!(key_from_iri_or_label("http://x#Company", None), "company"); + } + /// schema.org 那一套的最小复刻：数据类型自报家门，属性用 /// domainIncludes / rangeIncludes，全部只声明成 rdf:Property。 const SCHEMA_ISH: &str = r#"