diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index bcef7e2dd..9f0cacd62 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -47,7 +47,9 @@ jobs: # migrations 再编一遍才真的跑。给这个 job 也起一个 Postgres，一次编译全部跑完， # migrations job 只留迁移本身的检查。连库测试由此进了 backend 汇总，才真正成为合并门槛 test: - runs-on: ubuntu-latest + # 钉在 24.04：ubuntu-latest 换到 26.04 后 apt 里没有 postgresql-client-16， + # 装测试工具这一步直接失败，dev 与所有 PR 一起红（2026-10-29） + runs-on: ubuntu-24.04 services: postgres: image: pgvector/pgvector:pg16 @@ -66,8 +68,10 @@ jobs: - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.event_name == 'push' }} - - name: Install PDF text extraction tools - run: sudo apt-get update && sudo apt-get install -y poppler-utils poppler-data + - name: Install PDF and PostgreSQL test tools + run: | + sudo apt-get update && sudo apt-get install -y poppler-utils poppler-data postgresql-client-16 + echo '/usr/lib/postgresql/16/bin' >> "$GITHUB_PATH" # 库要先迁移好：store 的一百多个集成测试和 server 的多数 fixture 不自己跑迁移， # 对着空库并发起几百个测试会成片撞 relation "organizations" does not exist（#869）。 # 不用 sqlx-cli——装它要一分多钟，而这个 job 在关键路径上；这个 example 复用 diff --git a/.roadmap-proposals/backup-restore.md b/.roadmap-proposals/backup-restore.md index 3cec0735f..2f8572813 100644 --- a/.roadmap-proposals/backup-restore.md +++ b/.roadmap-proposals/backup-restore.md @@ -81,11 +81,40 @@ $ utopia restore --from [flags] --from # archive to restore from (required) --target-data-dir # override the data_dir path --pg-restore # path to pg_restore binary - --dry-run # print the plan, don't write anything - --force # allow restore into a non-empty database + --dry-run # validate and print the plan; don't change targets + --force # allow overwriting a non-empty database or data dir --yes # skip the "are you sure?" prompt ``` +Before changing either target, restore checks required archive components, +checksums, version, target path types and overwrite permission. Declared data +must exist in the archive. `--force` is required for a database containing user +objects (even empty tables, schemas, functions or types), and for a non-empty +data directory when the archive includes data. `--yes` only skips confirmation. +`--dry-run` uses the same preflight, including a read-only database check; it +extracts temporary staging files in the current working directory but never +restores the database or writes the target data directory. After confirmation +and data preparation, restore repeats component, target-path and overwrite +checks immediately before running `pg_restore`. + +Database-only archives leave the data directory untouched. Otherwise, data is +prepared separately and replaces the old directory after `pg_restore` succeeds; +files and keys are never merged with the old directory. A database restore +failure preserves the old files. With `--force`, `pg_restore --clean --if-exists` +replaces objects in the archive without removing unrelated database objects. +Every restore uses `--single-transaction --exit-on-error`: a SQL execution error +rolls back the database restore instead of leaving partial SQL changes. + +Stop all writers and retain a backup of both targets before restoring. Preflight +does not prevent concurrent changes or later permission and disk failures. +Database and filesystem restoration are **not atomic**: the database may be +committed before a later directory installation fails. Failed installation +attempts to put the old directory back; a failed rollback reports its retained +location. After an interruption, also check the target's parent for +`.utopia-restore-data-*/previous`. Keep the service stopped, resolve the error, +and retry the archive with `--force`, or restore both targets from the previous +backup. Moving old files back alone does not undo database changes. + Both halves are implemented. `backup` landed first, and `restore` followed in the same crate, for two reasons worth keeping on the record: @@ -150,15 +179,18 @@ binary first, the image second. - No docker image change (see Open question #3). - No `UTOPIA_BACKUP_DIR` config (see Open question #5). -- No automatic migration of older manifests — `restore` will only - read `schema_version == current`. +- No automatic migration of older manifests — older versions are accepted + with a warning; newer versions are refused. - **The sealing key is left out of the archive by default.** The dump carries credentials sealed with `data/secret.key`, and an archive is the artifact that gets copied between hosts and handed to whoever runs the restore, so shipping both halves in one file is not an unencrypted archive — it is no sealing at - all. `--include-secret-key` puts it in on purpose, the manifest records - whether it is there, and a restore without it says so rather than letting the - server fail to open its own credentials later. + all. `--include-secret-key` opts in to copying an existing key file. The + manifest records that flag even when an environment-key deployment has no + key file; restore remains compatible with those archives. When replacing + data, an absent key is never taken from the old directory: restore warns to + set `UTOPIA_SECRET_KEY` to the original key, or re-enter sealed credentials + after starting the server. - No encryption-at-rest. Out of scope; the operator's filesystem encryption (LUKS, EBS encryption, etc.) is the right layer. - No streaming upload to S3. Out of scope; the operator can pipe @@ -166,34 +198,26 @@ binary first, the image second. ## Test plan -8 unit tests, all pure (no live DB, no Docker): - -1. `parses_backup_minimal` — `utopia backup --dry-run` produces a - `BackupArgs` with all defaults. -2. `parses_backup_full` — every flag set, every field populated. -3. `parses_restore_requires_from` — `--from` is mandatory. -4. `parses_restore_full` — every flag set, every field populated. -5. `rejects_unknown_subcommand` — `utopia frobnicate` fails clearly. -6. `redact_url_host_keeps_userinfo_at_host` — `postgres://u:***@h/db` - stays redacted on round-trip. -7. `redact_url_host_handles_no_at` — `postgres://h/db` redacts - nothing (no userinfo). -8. `hex_encode_known_value` — `hex_encode(&[0xde, 0xad]) == "dead"`. - -Live integration (smoke test, not part of CI): - -``` -$ createdb utopia_test_restore -$ UTOPIA_DATABASE_URL=postgres://utopia:utopia@localhost:1543/utopia_test \ - utopia backup --output /tmp/utopia.tar.gz --include-data-dir -$ dropdb utopia_test_restore && createdb utopia_test_restore -$ UTOPIA_DATABASE_URL=postgres://utopia:utopia@localhost:1543/utopia_test \ - utopia restore --from /tmp/utopia.tar.gz --yes +Unit tests cover parsing, manifests, versions, checksums and key-copy policy. +The CLI integration suite runs in `cargo test --workspace`: a recording +`pg_restore` double checks preflight ordering, post-confirmation validation, +overwrite guards, replacement, rollback, dry runs and database-only archives. +Real `pg_dump`/`pg_restore` tests exercise database restoration and SQL rollback +using disposable test databases. +CI provides PostgreSQL 16 clients; the double also needs `rustc`, and +archive operations need `tar`. + +Tests use `utopia_store::test_db::url()` and create uniquely named databases on +that test cluster; they never modify the guard database's contents. Use only a +dedicated test cluster with `CREATEDB` and catalog-fixture privileges, never a +business database. Without a URL, tests skip unless `UTOPIA_TEST_REQUIRE_DB=1` +requires them to run, as in CI. + +```bash +UTOPIA_DATABASE_URL=postgres://test_role:test_password@localhost:1543/test_admin \ +UTOPIA_TEST_REQUIRE_DB=1 cargo test -p utopia-cli ``` -Smoke-tested manually on the maintainer's local docker-compose stack -once the restore side lands. - ## Migration concern: existing data is already there Backups are forward-looking. Anything written before this lands has diff --git a/Cargo.lock b/Cargo.lock index 069c37471..1d1f08d4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5398,9 +5398,13 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", + "sqlx", + "tempfile", + "tokio", "tracing", "tracing-subscriber", "utopia-core", + "utopia-store", ] [[package]] diff --git a/crates/utopia-cli/Cargo.toml b/crates/utopia-cli/Cargo.toml index 5554c5dde..e695be816 100644 --- a/crates/utopia-cli/Cargo.toml +++ b/crates/utopia-cli/Cargo.toml @@ -18,3 +18,9 @@ tracing.workspace = true tracing-subscriber.workspace = true sha2.workspace = true dotenvy.workspace = true +sqlx.workspace = true +tokio.workspace = true +tempfile.workspace = true + +[dev-dependencies] +utopia-store.workspace = true diff --git a/crates/utopia-cli/src/main.rs b/crates/utopia-cli/src/main.rs index 855c9f9f2..42562537e 100644 --- a/crates/utopia-cli/src/main.rs +++ b/crates/utopia-cli/src/main.rs @@ -1,15 +1,15 @@ //! `utopia` — operator-facing command-line entry point. //! //! See `.roadmap-proposals/backup-restore.md` for the design rationale. -//! This is the draft PR (branch `feat/backup-restore`): `backup` is -//! implemented, `restore` is stubbed. use std::collections::HashMap; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; +use anyhow::Context; use chrono::Utc; use serde::{Deserialize, Serialize}; +use sqlx::{Connection, PgConnection}; use tracing_subscriber::EnvFilter; use utopia_core::config::AppConfig; @@ -82,9 +82,10 @@ struct ManifestDataDir { /// not a side effect of a code change. // 是迁移文件的**个数**，不是最大的编号（守卫 `schema_version_policy_compares_against_current` // 按个数比）：编号有空缺时两者不同——0071 由一个开放 PR 占着，0072 先落，个数是 71 -const CURRENT_SCHEMA_VERSION: u32 = 114; +const CURRENT_SCHEMA_VERSION: u32 = 120; -fn main() -> anyhow::Result<()> { +#[tokio::main] +async fn main() -> anyhow::Result<()> { dotenvy::dotenv().ok(); tracing_subscriber::fmt() .with_env_filter( @@ -96,7 +97,7 @@ fn main() -> anyhow::Result<()> { let cmd = parse(&args)?; match cmd { Command2::Backup(a) => run_backup(a), - Command2::Restore(a) => run_restore(a), + Command2::Restore(a) => run_restore(a).await, } } @@ -127,7 +128,7 @@ fn print_help() { \n\ SUBCOMMANDS:\n \ backup Snapshot the Postgres database (and optionally the data dir) into a tarball.\n \ - restore Restore from a tarball produced by `utopia backup`. (TODO: stubbed in this PR.)\n" + restore Restore from a tarball produced by `utopia backup`.\n" ); } @@ -192,9 +193,9 @@ fn parse_restore<'a, I: Iterator>(iter: &mut I) -> anyhow::Re --from Path to a tarball produced by `utopia backup`.\n \ --target-data-dir Where to place the restored data dir.\n \ --pg-restore Override the pg_restore binary.\n \ - --dry-run Plan only.\n \ - --force Required if target database is non-empty.\n \ - --yes Skip the confirmation prompt.\n" + --dry-run Validate and print the plan; do not change targets.\n \ + --force Required for a non-empty target database or restored data dir.\n \ + --yes Skip confirmation only; does not imply --force.\n" ); std::process::exit(0); } @@ -494,10 +495,10 @@ fn hex_encode(bytes: &[u8]) -> String { } // --------------------------------------------------------------------------- -// restore (stubbed) +// 恢复 // --------------------------------------------------------------------------- -fn run_restore(args: RestoreArgs) -> anyhow::Result<()> { +async fn run_restore(args: RestoreArgs) -> anyhow::Result<()> { let cfg = AppConfig::load()?; let from = args .from @@ -518,19 +519,11 @@ fn run_restore(args: RestoreArgs) -> anyhow::Result<()> { .clone() .unwrap_or_else(|| PathBuf::from(&cfg.data_dir)); - // Stage: extract the tarball into a tempdir we own. The cleanup at the - // end of every path uses `let _ = remove_dir_all(&stage)` so a panic or - // early return doesn't leave a half-written data dir behind. - let stage = tempdir_in(std::env::current_dir()?.as_path())?; - let manifest_path = stage.join("manifest.json"); - - let manifest = match extract_and_verify(&tar_bin, from, &stage, &manifest_path) { - Ok(m) => m, - Err(e) => { - let _ = std::fs::remove_dir_all(&stage); - return Err(e); - } - }; + let stage = tempfile::Builder::new() + .prefix(".utopia-restore-") + .tempdir_in(std::env::current_dir()?)?; + let manifest_path = stage.path().join("manifest.json"); + let manifest = extract_and_verify(&tar_bin, from, stage.path(), &manifest_path)?; // Manifest version policy: refuse forward, warn older. Forward refusal // is fatal because restoring into an older server with a newer schema @@ -539,7 +532,6 @@ fn run_restore(args: RestoreArgs) -> anyhow::Result<()> { // a newer server can usually still load them (the schema migrations are // forward-only; older files just exercise older code paths). if manifest.schema_version > CURRENT_SCHEMA_VERSION { - let _ = std::fs::remove_dir_all(&stage); anyhow::bail!( "manifest schema_version {} is newer than this binary's {}; refusing to restore an incompatible backup", manifest.schema_version, @@ -574,41 +566,43 @@ fn run_restore(args: RestoreArgs) -> anyhow::Result<()> { args.dry_run, ); + let target_data_dir = preflight_restore( + &pg_restore_bin, + &target_db, + &target_data_dir, + stage.path(), + &manifest, + args.force, + ) + .await?; + if args.dry_run { print!("{plan}"); - let _ = std::fs::remove_dir_all(&stage); - // Verify pre-flight so a bad plan still exits non-zero. - preflight_restore(&pg_restore_bin, &target_data_dir, &manifest)?; eprintln!("[dry-run] plan resolved cleanly; no changes made."); return Ok(()); } - preflight_restore(&pg_restore_bin, &target_data_dir, &manifest)?; - if !args.yes { eprint!( - "{plan}\nAbout to drop and recreate the target database, then restore from the archive. Continue? [y/N] " + "{plan}\nAbout to restore database objects and replace data if included in the archive. Stop all writers first. Continue? [y/N] " ); let mut buf = String::new(); std::io::stdin().read_line(&mut buf)?; if !matches!(buf.trim().to_ascii_lowercase().as_str(), "y" | "yes") { - let _ = std::fs::remove_dir_all(&stage); anyhow::bail!("aborted by user"); } } - if let Err(e) = apply_restore( + apply_restore( &pg_restore_bin, &target_db, &target_data_dir, - &stage, + stage.path(), &manifest, - ) { - let _ = std::fs::remove_dir_all(&stage); - return Err(e); - } + args.force, + ) + .await?; - let _ = std::fs::remove_dir_all(&stage); tracing::info!(path = %from.display(), "restore complete"); Ok(()) } @@ -655,44 +649,204 @@ fn extract_and_verify( Ok(manifest) } -fn preflight_restore( +fn check_archive_component(stage: &Path, name: &str, directory: bool) -> anyhow::Result<()> { + let relative = Path::new(name); + if name.is_empty() + || relative + .components() + .any(|part| !matches!(part, std::path::Component::Normal(_))) + { + anyhow::bail!("invalid archive component path: `{name}`"); + } + let relative: PathBuf = relative.components().collect(); + let path = stage.join(relative); + let metadata = std::fs::symlink_metadata(&path) + .with_context(|| format!("archive did not contain required component `{name}`"))?; + if metadata.file_type().is_symlink() + || (directory && !metadata.is_dir()) + || (!directory && !metadata.is_file()) + || !path.canonicalize()?.starts_with(stage.canonicalize()?) + { + anyhow::bail!("archive component `{name}` has an invalid type or location"); + } + Ok(()) +} + +fn check_restore_data_target(target: &Path, force: bool) -> anyhow::Result { + if target.file_name().is_none() { + anyhow::bail!( + "target data dir must name a replaceable directory: {}", + target.display() + ); + } + // 先移除末尾分隔符和 `/.`，否则 lstat 也会跟随符号链接。 + let normalized: PathBuf = target.components().collect(); + let absolute = std::path::absolute(normalized)?; + // 允许父目录尚未创建，但文件或悬空链接不能充当父目录。 + let mut resolved = None; + for parent in absolute.ancestors().skip(1) { + match std::fs::metadata(parent) { + Ok(metadata) if metadata.is_dir() => { + let suffix = absolute.strip_prefix(parent)?; + if suffix + .components() + .any(|part| !matches!(part, std::path::Component::Normal(_))) + { + anyhow::bail!( + "target data dir traverses a missing parent: {}", + target.display() + ); + } + // 固定实际父路径，避免移走旧目录后无法访问暂存文件和回滚路径。 + resolved = Some(parent.canonicalize()?.join(suffix)); + break; + } + Ok(_) => anyhow::bail!( + "target data dir parent is not a directory: {}", + parent.display() + ), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + if std::fs::symlink_metadata(parent).is_ok() { + anyhow::bail!( + "target data dir parent is a dangling link: {}", + parent.display() + ); + } + } + Err(error) => return Err(error).context("cannot inspect target data dir parent"), + } + } + let target = resolved.context("target data dir has no accessible directory parent")?; + match std::fs::symlink_metadata(&target) { + Ok(metadata) => { + if !metadata.is_dir() || metadata.file_type().is_symlink() { + anyhow::bail!( + "target data dir is not a regular directory: {}", + target.display() + ); + } + if std::fs::read_dir(&target)?.next().transpose()?.is_some() && !force { + anyhow::bail!( + "target data dir {} is non-empty; pass --force to overwrite", + target.display() + ); + } + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => return Err(error).context("cannot inspect target data dir"), + } + Ok(target) +} + +async fn preflight_restore( pg_restore_bin: &Path, + target_db: &str, target_data_dir: &Path, - _manifest: &Manifest, -) -> anyhow::Result<()> { + stage: &Path, + manifest: &Manifest, + force: bool, +) -> anyhow::Result { + check_archive_component(stage, &manifest.components.pg_dump.path, false)?; + let target_data_dir = if manifest.components.data_dir.present { + check_archive_component(stage, &manifest.components.data_dir.path, true)?; + check_restore_data_target(target_data_dir, force)? + } else { + target_data_dir.to_path_buf() + }; if !binary_works(pg_restore_bin) { anyhow::bail!( "pg_restore not found or not executable: {}. Install postgresql-client or pass --pg-restore .", pg_restore_bin.display() ); } - // Data dir may not exist yet — restore creates it. Don't preflight its - // existence; that's not a precondition failure. - let _ = target_data_dir; - Ok(()) + let mut connection = PgConnection::connect(target_db) + .await + .context("cannot connect to target database for restore preflight")?; + // 空表、schema 等用户对象也算非空；命名空间依赖还能覆盖函数、类型等对象。 + let nonempty: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM pg_catalog.pg_namespace n + WHERE n.nspname !~ '^pg_' AND n.nspname <> 'information_schema' + AND (n.nspname <> 'public' OR EXISTS ( + SELECT 1 FROM pg_catalog.pg_depend d + WHERE d.refclassid = 'pg_catalog.pg_namespace'::regclass + AND d.refobjid = n.oid))) + OR EXISTS (SELECT 1 FROM pg_catalog.pg_extension WHERE extname <> 'plpgsql') + OR EXISTS (SELECT 1 FROM pg_catalog.pg_largeobject_metadata) + OR EXISTS (SELECT 1 FROM pg_catalog.pg_foreign_data_wrapper) + OR EXISTS (SELECT 1 FROM pg_catalog.pg_foreign_server) + OR EXISTS (SELECT 1 FROM pg_catalog.pg_publication) + OR EXISTS (SELECT 1 FROM pg_catalog.pg_event_trigger) + OR EXISTS (SELECT 1 FROM pg_catalog.pg_subscription + WHERE subdbid = (SELECT oid FROM pg_catalog.pg_database + WHERE datname = pg_catalog.current_database()))", + ) + .fetch_one(&mut connection) + .await + .context("cannot inspect target database for restore preflight")?; + if nonempty && !force { + anyhow::bail!("target database is non-empty; pass --force to overwrite (--yes only skips confirmation)"); + } + Ok(target_data_dir) } -fn apply_restore( +async fn apply_restore( pg_restore_bin: &Path, target_db: &str, target_data_dir: &Path, stage: &Path, manifest: &Manifest, + force: bool, ) -> anyhow::Result<()> { - let pg_dump_path = stage.join(&manifest.components.pg_dump.path); - if !pg_dump_path.exists() { + let dump_relative: PathBuf = Path::new(&manifest.components.pg_dump.path) + .components() + .collect(); + let pg_dump_path = stage.join(dump_relative); + // 先在目标文件系统准备新目录；复制或数据库恢复失败时，旧目录仍保留原位。 + let prepared = if manifest.components.data_dir.present { + let parent = target_data_dir + .parent() + .context("target data dir has no parent")?; + std::fs::create_dir_all(parent)?; + let prepared = tempfile::Builder::new() + .prefix(".utopia-restore-data-") + .tempdir_in(parent)?; + copy_dir_recursive( + &stage.join(&manifest.components.data_dir.path), + &prepared.path().join("new"), + true, + )?; + Some(prepared) + } else { + None + }; + // 等待确认和准备文件期间目标可能变化，启动 pg_restore 前必须重新检查。 + let checked_target = preflight_restore( + pg_restore_bin, + target_db, + target_data_dir, + stage, + manifest, + force, + ) + .await?; + // 暂存文件已绑定首次解析的物理路径，复查不能悄悄切换到另一个目标。 + if checked_target.as_path() != target_data_dir { anyhow::bail!( - "manifest references pg_dump at `{}` but the archive did not contain it", - manifest.components.pg_dump.path + "target data dir resolved to a different location after confirmation; retry with writers stopped" ); } tracing::info!( archive_pg_dump = %pg_dump_path.display(), - "running pg_restore --clean --if-exists" + force, + "running pg_restore" ); - let status = Command::new(pg_restore_bin) - .arg("--clean") - .arg("--if-exists") + let mut command = Command::new(pg_restore_bin); + if force { + command.args(["--clean", "--if-exists"]); + } + let status = command + .args(["--exit-on-error", "--single-transaction"]) .arg("--dbname") .arg(target_db) .arg(&pg_dump_path) @@ -700,42 +854,41 @@ fn apply_restore( .status()?; if !status.success() { anyhow::bail!( - "pg_restore exited with status {}; database may be in a partial state — verify with a follow-up pg_dump", + "pg_restore exited with status {}; SQL restore errors are rolled back; verify the database before retrying", status ); } - if manifest.components.data_dir.present { - let archive_data_dir = stage.join(&manifest.components.data_dir.path); - if archive_data_dir.exists() { - tracing::info!( - target = %target_data_dir.display(), - "restoring data dir" - ); - // Drop the existing target dir if it's non-empty so the copy - // below doesn't merge two worlds. This is what `--force` - // guards upstream — if `--force` is missing and the target - // already has files, fail loud. - if target_data_dir.exists() && std::fs::read_dir(target_data_dir)?.next().is_some() { - anyhow::bail!( - "target data dir {} is non-empty; pass --force to overwrite", - target_data_dir.display() - ); + if let Some(prepared) = prepared { + check_restore_data_target(target_data_dir, force) + .context("database restored, but target data dir changed during restore; retry with writers stopped")?; + let previous = prepared.path().join("previous"); + let had_target = target_data_dir.exists(); + if had_target { + std::fs::rename(target_data_dir, &previous) + .context("database restored, but could not move old data dir; original files remain in place")?; + } + if let Err(error) = std::fs::rename(prepared.path().join("new"), target_data_dir) { + if had_target { + if let Err(rollback) = std::fs::rename(&previous, target_data_dir) { + let retained = prepared.keep(); + anyhow::bail!("database restored, but data dir installation failed: {error}; rollback failed: {rollback}; original data retained at {} — stop writers and recover manually", retained.join("previous").display()); + } } - // 恢复时照搬压缩包里有的：包里没有密钥，是打包那一步的决定，不是这里的 - copy_dir_recursive(&archive_data_dir, target_data_dir, true)?; - if !manifest.components.data_dir.secret_key { - // 没有钥匙，库里那些封存的凭据就打不开。与其让人在服务起来之后 - // 看见一串解不开的错误，不如现在说清楚该做什么 - tracing::warn!( + return Err(error).context("database restored, but data dir installation failed; original data dir preserved — resolve the error and retry restore"); + } + prepared.close().with_context(|| { + format!( + "restore completed, but could not remove previous data at {}", + previous.display() + ) + })?; + if !target_data_dir.join(SECRET_KEY_FILE).is_file() { + // 没有钥匙，库里那些封存的凭据就打不开。与其让人在服务起来之后 + // 看见一串解不开的错误，不如现在说清楚该做什么 + tracing::warn!( "this archive carries no {SECRET_KEY_FILE}: sealed credentials (model keys, source credentials) will not open. Set UTOPIA_SECRET_KEY on this host to the key the backup was taken with, or re-enter them after starting the server" ); - } - } else { - tracing::warn!( - path = %archive_data_dir.display(), - "manifest says data_dir is present but the archive did not contain it; skipping" - ); } } diff --git a/crates/utopia-cli/tests/restore.rs b/crates/utopia-cli/tests/restore.rs new file mode 100644 index 000000000..d211178ce --- /dev/null +++ b/crates/utopia-cli/tests/restore.rs @@ -0,0 +1,872 @@ +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Output, Stdio}; +use std::str::FromStr; +use std::sync::{mpsc, OnceLock}; +use std::time::Duration; + +use sha2::{Digest, Sha256}; +use sqlx::{postgres::PgConnectOptions, ConnectOptions, Connection, Executor, PgConnection}; + +fn pg_restore_stub() -> PathBuf { + static BIN: OnceLock = OnceLock::new(); + BIN.get_or_init(|| { + // 共享替身放在构建目录，由 cargo clean 统一清理。 + let binary = std::env::current_exe().unwrap().with_file_name(format!( + "utopia-pg-restore-test-{}{}", + std::process::id(), + std::env::consts::EXE_SUFFIX + )); + let status = Command::new("rustc") + .arg(Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/support/pg_restore.rs")) + .arg("-o") + .arg(&binary) + .status() + .unwrap(); + assert!(status.success(), "compile the recording pg_restore double"); + binary + }) + .clone() +} + +struct RestoreTest { + root: tempfile::TempDir, + admin_url: String, + database: String, + url: String, +} + +impl RestoreTest { + async fn new(with_data: bool) -> Option { + let admin_url = utopia_store::test_db::url()?; + let root = tempfile::tempdir().unwrap(); + let database = format!( + "utopia_restore_{}_{}", + std::process::id(), + root.path() + .file_name() + .unwrap() + .to_string_lossy() + .replace('.', "") + ) + .to_lowercase(); + let options = PgConnectOptions::from_str(&admin_url).unwrap(); + let mut admin = PgConnection::connect_with(&options).await.unwrap(); + // 只修改和删除本例新建的数据库，不触碰 guard 提供的数据库。 + admin + .execute(format!("CREATE DATABASE {database}").as_str()) + .await + .unwrap(); + let mut target_url = options.database(&database).to_url_lossy(); + // SQLx 自动追加的客户端参数不被 libpq 的 pg_dump/pg_restore 接受。 + let parameters: Vec<_> = target_url + .query_pairs() + .filter(|(name, _)| name != "statement-cache-capacity") + .map(|(name, value)| (name.into_owned(), value.into_owned())) + .collect(); + target_url + .query_pairs_mut() + .clear() + .extend_pairs(parameters); + let fixture = Self { + root, + admin_url, + url: target_url.to_string(), + database, + }; + std::fs::create_dir(fixture.source()).unwrap(); + std::fs::write(fixture.source().join("pg_dump.custom"), b"test dump").unwrap(); + if with_data { + std::fs::create_dir_all(fixture.source().join("data/files")).unwrap(); + std::fs::write(fixture.source().join("data/files/new.bin"), b"new data").unwrap(); + std::fs::write(fixture.source().join("data/secret.key"), b"archive key").unwrap(); + } + let version = + std::fs::read_dir(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../migrations")) + .unwrap() + .map(|entry| entry.expect("read migration directory entry")) + .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "sql")) + .count(); + let digest: String = Sha256::digest(b"test dump") + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + let checksum = format!("sha256:{digest}"); + let manifest = serde_json::json!({ + "schema_version": version, "utopia_version": "0.1.0", "created_at": "2026-10-25T00:00:00Z", + "components": { + "pg_dump": {"path": "pg_dump.custom", "format": "pg_dump -Fc", "bytes": 9}, + "data_dir": {"path": "data", "present": with_data, "secret_key": with_data} + }, + "checksums": {"pg_dump.custom": checksum} + }); + std::fs::write( + fixture.source().join("manifest.json"), + serde_json::to_vec(&manifest).unwrap(), + ) + .unwrap(); + fixture.pack(); + Some(fixture) + } + + fn source(&self) -> PathBuf { + self.root.path().join("source") + } + fn target(&self) -> PathBuf { + self.root.path().join("target") + } + fn log(&self) -> PathBuf { + self.root.path().join("restore.log") + } + fn pack(&self) { + let status = Command::new("tar") + .arg("-czf") + .arg(self.root.path().join("backup.tar.gz")) + .arg("-C") + .arg(self.source()) + .arg(".") + .status() + .unwrap(); + assert!(status.success()); + } + + fn edit_manifest(&self, edit: impl FnOnce(&mut serde_json::Value)) { + let path = self.source().join("manifest.json"); + let mut manifest = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + edit(&mut manifest); + std::fs::write(path, serde_json::to_vec(&manifest).unwrap()).unwrap(); + } + + fn cli(&self) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_utopia")); + command + .current_dir(self.root.path()) + .env("UTOPIA_DATABASE_URL", &self.url) + .env("UTOPIA_MIGRATION_URL", &self.url) + .env("UTOPIA_DATA_DIR", self.source().join("data")) + .stdin(Stdio::null()); + command + } + + fn command(&self, flags: &[&str]) -> Command { + let mut command = self.cli(); + command + .args(["restore", "--from"]) + .arg(self.root.path().join("backup.tar.gz")) + .arg("--target-data-dir") + .arg(self.target()) + .arg("--pg-restore") + .arg(pg_restore_stub()) + .args(flags) + .env("UTOPIA_RESTORE_TEST_LOG", self.log()); + command + } + + fn restore(&self, flags: &[&str]) -> Output { + self.command(flags).output().unwrap() + } + fn awaiting_confirmation(&self) -> Child { + let mut child = self + .command(&[]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut stderr = child.stderr.take().unwrap(); + let (sender, receiver) = mpsc::channel(); + let reader = std::thread::spawn(move || { + let mut prompt = Vec::new(); + let result = loop { + let mut byte = [0]; + if let Err(error) = stderr.read_exact(&mut byte) { + break Err(error); + } + prompt.push(byte[0]); + if prompt.ends_with(b"Continue? [y/N] ") { + break Ok(()); + } + }; + // 超时后接收端会关闭，此时发送失败属于正常清理。 + let _ = sender.send((result, stderr, prompt)); + }); + let error = match receiver.recv_timeout(Duration::from_secs(30)) { + Ok((Ok(()), stderr, _)) => { + reader.join().unwrap(); + child.stderr = Some(stderr); + return child; + } + Ok((Err(error), _, prompt)) => { + format!("{error}; stderr: {}", String::from_utf8_lossy(&prompt)) + } + Err(error) => format!("confirmation wait failed: {error}"), + }; + drop(receiver); + let _ = child.kill(); + let status = child + .wait() + .expect("reap restore after confirmation failure"); + reader.join().unwrap(); + panic!("restore did not reach confirmation: {error}; child status: {status}"); + } + fn calls(&self) -> String { + let log = self.log(); + match std::fs::read_to_string(&log) { + Ok(calls) => calls, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(error) => panic!("cannot read pg_restore call log {}: {error}", log.display()), + } + } + fn old_data(&self) { + std::fs::create_dir_all(self.target().join("files")).unwrap(); + std::fs::write(self.target().join("files/old.bin"), b"old data").unwrap(); + std::fs::write(self.target().join("secret.key"), b"old key").unwrap(); + } + fn assert_old_data(&self) { + assert_eq!( + std::fs::read(self.target().join("files/old.bin")).unwrap(), + b"old data" + ); + assert_eq!( + std::fs::read(self.target().join("secret.key")).unwrap(), + b"old key" + ); + assert!(!self.target().join("files/new.bin").exists()); + } + async fn connection(&self) -> PgConnection { + PgConnection::connect(&self.url).await.unwrap() + } +} + +impl Drop for RestoreTest { + fn drop(&mut self) { + let admin_url = self.admin_url.clone(); + let database = self.database.clone(); + let cleanup = std::thread::spawn(move || { + tokio::runtime::Runtime::new().unwrap().block_on(async { + let mut admin = PgConnection::connect(&admin_url).await.unwrap(); + admin + .execute(format!("DROP DATABASE {database} WITH (FORCE)").as_str()) + .await + .unwrap(); + }); + }) + .join(); + if let Err(panic_payload) = cleanup { + eprintln!("failed to clean up test database {}", self.database); + // 清理失败时保留最初的测试错误，避免 Drop 中二次 panic 中止整个进程。 + if !std::thread::panicking() { + std::panic::resume_unwind(panic_payload); + } + } + } +} + +fn assert_success(output: &Output) { + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn assert_failure(output: &Output, message: &str) { + assert!( + !output.status.success(), + "unexpected success: {}", + String::from_utf8_lossy(&output.stdout) + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains(message), + "stderr: {}", + String::from_utf8_lossy(&output.stderr) + ); +} + +#[tokio::test] +async fn nonempty_directory_requires_force_before_database_restore() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + fixture.old_data(); + for flags in [vec!["--yes"], vec!["--yes", "--dry-run"]] { + assert_failure(&fixture.restore(&flags), "--force"); + assert_eq!(fixture.calls(), "", "preflight must not call pg_restore"); + fixture.assert_old_data(); + } +} + +#[tokio::test] +async fn database_changed_during_confirmation_is_rejected_before_restore() { + let Some(fixture) = RestoreTest::new(false).await else { + return; + }; + let mut child = fixture.awaiting_confirmation(); + let mut connection = fixture.connection().await; + connection + .execute( + "CREATE TABLE confirmation_sentinel (value integer); + INSERT INTO confirmation_sentinel VALUES (99)", + ) + .await + .unwrap(); + child.stdin.take().unwrap().write_all(b"y\n").unwrap(); + assert_failure( + &child.wait_with_output().unwrap(), + "target database is non-empty", + ); + assert_eq!( + fixture.calls(), + "", + "confirmation must not bypass the database overwrite guard" + ); + let unchanged: i32 = sqlx::query_scalar("SELECT value FROM confirmation_sentinel") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(unchanged, 99); + assert!(!fixture.target().exists()); +} + +#[tokio::test] +async fn data_directory_changed_during_confirmation_is_rejected_before_restore() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + let mut child = fixture.awaiting_confirmation(); + fixture.old_data(); + child.stdin.take().unwrap().write_all(b"y\n").unwrap(); + assert_failure(&child.wait_with_output().unwrap(), "--force"); + assert_eq!( + fixture.calls(), + "", + "confirmation must not bypass the directory overwrite guard" + ); + fixture.assert_old_data(); +} + +#[tokio::test] +async fn force_replaces_old_files_and_key_instead_of_merging() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + fixture.old_data(); + assert_success(&fixture.restore(&["--yes", "--force", "--dry-run"])); + fixture.assert_old_data(); + assert_eq!(fixture.calls(), ""); + let output = fixture + .command(&["--yes", "--force"]) + .env( + "UTOPIA_RESTORE_TEST_OLD_FILE", + fixture.target().join("files/old.bin"), + ) + .output() + .unwrap(); + assert_success(&output); + let calls = fixture.calls(); + assert_eq!(calls.lines().count(), 1); + for flag in [ + "--clean", + "--if-exists", + "--exit-on-error", + "--single-transaction", + ] { + assert!(calls.contains(flag), "{calls}"); + } + assert!(!fixture.target().join("files/old.bin").exists()); + assert_eq!( + std::fs::read(fixture.target().join("files/new.bin")).unwrap(), + b"new data" + ); + assert_eq!( + std::fs::read(fixture.target().join("secret.key")).unwrap(), + b"archive key" + ); +} + +#[tokio::test] +async fn empty_tables_and_other_user_objects_also_require_force() { + for statement in [ + "CREATE TABLE restore_empty (value integer)", + "CREATE SCHEMA restore_schema", + "CREATE FUNCTION restore_function() RETURNS integer LANGUAGE sql AS 'SELECT 1'", + "CREATE TYPE restore_type AS ENUM ('value')", + "CREATE VIEW restore_view AS SELECT 1 AS value", + "CREATE PUBLICATION restore_publication", + "CREATE FOREIGN DATA WRAPPER restore_fdw", + "SELECT pg_catalog.lo_create(0)", + ] { + let Some(fixture) = RestoreTest::new(false).await else { + return; + }; + fixture.connection().await.execute(statement).await.unwrap(); + for flags in [vec!["--yes"], vec!["--yes", "--dry-run"]] { + assert_failure(&fixture.restore(&flags), "--force"); + assert_eq!( + fixture.calls(), + "", + "{statement} bypassed overwrite protection" + ); + } + } +} + +#[tokio::test] +async fn missing_components_and_invalid_targets_fail_before_restore() { + for case in [ + "missing-data", + "missing-dump", + "dump-dir", + "data-file", + "target-file", + "parent-file", + ] { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + match case { + "missing-data" => std::fs::remove_dir_all(fixture.source().join("data")).unwrap(), + "missing-dump" | "dump-dir" => { + std::fs::remove_file(fixture.source().join("pg_dump.custom")).unwrap(); + if case == "dump-dir" { + std::fs::create_dir(fixture.source().join("pg_dump.custom")).unwrap(); + } + fixture.edit_manifest(|manifest| manifest["checksums"] = serde_json::json!({})); + } + "data-file" => { + std::fs::remove_dir_all(fixture.source().join("data")).unwrap(); + std::fs::write(fixture.source().join("data"), b"not a directory").unwrap(); + } + "target-file" => std::fs::write(fixture.target(), b"keep this file").unwrap(), + "parent-file" => { + std::fs::write(fixture.target(), b"keep this parent").unwrap(); + } + _ => unreachable!(), + } + fixture.pack(); + for dry_run in [false, true] { + let mut command = fixture.command(&["--yes", "--force"]); + if case == "parent-file" { + command + .arg("--target-data-dir") + .arg(fixture.target().join("nested")); + } + if dry_run { + command.arg("--dry-run"); + } + let output = command.output().unwrap(); + assert!( + !output.status.success(), + "{case} accepted (dry_run={dry_run})" + ); + assert_eq!( + fixture.calls(), + "", + "{case} ran pg_restore (dry_run={dry_run})" + ); + if fixture.target().is_file() { + assert!(std::fs::read(fixture.target()) + .unwrap() + .starts_with(b"keep this")); + } else { + assert!(!fixture.target().exists()); + } + } + } +} + +#[tokio::test] +async fn unreplaceable_targets_fail_before_restore() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + let filesystem_root = fixture.root.path().ancestors().last().unwrap(); + for target in [Path::new("."), Path::new(".."), filesystem_root] { + for dry_run in [false, true] { + let mut command = fixture.command(&["--yes", "--force"]); + command + .arg("--target-data-dir") + .arg(target) + .env("UTOPIA_RESTORE_TEST_FAIL", "1"); + if dry_run { + command.arg("--dry-run"); + } + assert_failure(&command.output().unwrap(), "replaceable directory"); + assert_eq!( + fixture.calls(), + "", + "invalid target reached pg_restore: {target:?}" + ); + assert!(!fixture.target().exists()); + } + } +} + +#[cfg(unix)] +#[tokio::test] +async fn dump_component_with_trailing_dot_or_slash_uses_its_normalized_file_path() { + for dump in ["pg_dump.custom/", "pg_dump.custom/."] { + let Some(fixture) = RestoreTest::new(false).await else { + return; + }; + fixture.edit_manifest(|manifest| manifest["components"]["pg_dump"]["path"] = dump.into()); + fixture.pack(); + assert_success(&fixture.restore(&["--yes", "--dry-run"])); + assert_eq!(fixture.calls(), ""); + assert_success(&fixture.restore(&["--yes"])); + assert_eq!(fixture.calls().lines().count(), 1); + assert!(!fixture.target().exists()); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn missing_parent_followed_by_traversal_is_rejected_before_restore() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + fixture.old_data(); + let missing = fixture.root.path().join("missing"); + for flags in [vec!["--yes"], vec!["--yes", "--dry-run"]] { + let output = fixture + .command(&flags) + .arg("--target-data-dir") + .arg(missing.join("../target")) + .output() + .unwrap(); + assert_failure(&output, "target data dir"); + assert_eq!( + fixture.calls(), + "", + "unresolved traversal reached pg_restore" + ); + assert!( + !missing.exists(), + "preflight must not create missing parents" + ); + fixture.assert_old_data(); + } +} + +#[cfg(unix)] +#[tokio::test] +async fn force_restores_through_parent_traversal_in_an_existing_target() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + fixture.old_data(); + let output = fixture + .command(&["--yes", "--force"]) + .arg("--target-data-dir") + .arg(fixture.target().join("../target")) + .env( + "UTOPIA_RESTORE_TEST_OLD_FILE", + fixture.target().join("files/old.bin"), + ) + .output() + .unwrap(); + assert_success(&output); + assert_eq!(fixture.calls().lines().count(), 1); + assert!(!fixture.target().join("files/old.bin").exists()); + assert_eq!( + std::fs::read(fixture.target().join("files/new.bin")).unwrap(), + b"new data" + ); +} + +#[cfg(unix)] +#[tokio::test] +async fn symlink_targets_with_trailing_dot_or_slash_fail_before_restore() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + fixture.old_data(); + std::os::unix::fs::symlink(fixture.target(), fixture.root.path().join("link")).unwrap(); + for suffix in ["link", "link/.", "link/"] { + for dry_run in [false, true] { + let mut command = fixture.command(&["--yes", "--force"]); + command + .arg("--target-data-dir") + .arg(fixture.root.path().join(suffix)) + .env("UTOPIA_RESTORE_TEST_FAIL", "1"); + if dry_run { + command.arg("--dry-run"); + } + assert_failure(&command.output().unwrap(), "target data dir"); + assert_eq!( + fixture.calls(), + "", + "symlink target reached pg_restore: {suffix}" + ); + fixture.assert_old_data(); + } + } +} + +#[tokio::test] +async fn archive_component_paths_must_stay_inside_the_archive() { + for component in ["pg_dump", "data_dir"] { + for absolute in [false, true] { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + let outside = if absolute { + fixture.root.path().join("outside") + } else { + PathBuf::from("../outside") + }; + fixture.edit_manifest(|manifest| { + manifest["components"][component]["path"] = + outside.to_string_lossy().as_ref().into(); + manifest["checksums"] = serde_json::json!({}); + }); + fixture.pack(); + for flags in [ + vec!["--yes", "--force"], + vec!["--yes", "--force", "--dry-run"], + ] { + assert_failure(&fixture.restore(&flags), "invalid archive component path"); + assert_eq!(fixture.calls(), ""); + assert!(!fixture.target().exists()); + } + } + } +} + +#[tokio::test] +async fn pg_restore_failure_preserves_the_entire_old_directory() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + fixture.old_data(); + let output = fixture + .command(&["--yes", "--force"]) + .env("UTOPIA_RESTORE_TEST_FAIL", "1") + .env( + "UTOPIA_RESTORE_TEST_OLD_FILE", + fixture.target().join("files/old.bin"), + ) + .output() + .unwrap(); + assert_failure(&output, "pg_restore exited"); + assert!(String::from_utf8_lossy(&output.stderr).contains("rolled back")); + assert_eq!(fixture.calls().lines().count(), 1); + fixture.assert_old_data(); +} + +#[tokio::test] +async fn failed_data_installation_rolls_back_the_old_directory() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + fixture.old_data(); + let output = fixture + .command(&["--yes", "--force"]) + .env("UTOPIA_RESTORE_TEST_REMOVE_PREPARED", fixture.root.path()) + .env( + "UTOPIA_RESTORE_TEST_OLD_FILE", + fixture.target().join("files/old.bin"), + ) + .output() + .unwrap(); + assert_failure(&output, "database restored"); + assert!(String::from_utf8_lossy(&output.stderr).contains("installation failed")); + assert_eq!(fixture.calls().lines().count(), 1); + fixture.assert_old_data(); + for entry in std::fs::read_dir(fixture.root.path()).unwrap() { + assert!( + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".utopia-"), + "staging directory was not cleaned up" + ); + } +} + +#[tokio::test] +async fn empty_target_restores_normally() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + std::fs::create_dir(fixture.target()).unwrap(); + assert_success(&fixture.restore(&["--yes"])); + let calls = fixture.calls(); + assert_eq!(calls.lines().count(), 1); + assert!(!calls.contains("--clean"), "--yes must not grant overwrite"); + assert!(!calls.contains("--if-exists")); + assert!(calls.contains("--single-transaction")); + assert_eq!( + std::fs::read(fixture.target().join("files/new.bin")).unwrap(), + b"new data" + ); +} + +#[tokio::test] +async fn database_only_backup_leaves_existing_data_untouched() { + let Some(fixture) = RestoreTest::new(false).await else { + return; + }; + fixture.old_data(); + assert_success(&fixture.restore(&["--yes", "--dry-run"])); + assert_eq!(fixture.calls(), ""); + fixture.assert_old_data(); + assert_success(&fixture.restore(&["--yes"])); + assert_eq!(fixture.calls().lines().count(), 1); + fixture.assert_old_data(); + + let Some(fixture) = RestoreTest::new(false).await else { + return; + }; + std::fs::write( + fixture.target(), + b"a database-only restore ignores this file", + ) + .unwrap(); + for flags in [vec!["--yes", "--dry-run"], vec!["--yes"]] { + assert_success(&fixture.restore(&flags)); + assert_eq!( + std::fs::read(fixture.target()).unwrap(), + b"a database-only restore ignores this file" + ); + } +} + +#[tokio::test] +async fn real_postgres_backup_and_restore_require_force_and_preserve_files_on_sql_failure() { + let Some(fixture) = RestoreTest::new(true).await else { + return; + }; + for binary in ["pg_dump", "pg_restore"] { + assert!( + Command::new(binary) + .arg("--version") + .output() + .is_ok_and(|output| output.status.success()), + "{binary} is required for database-backed CLI tests; install PostgreSQL client tools" + ); + } + let mut connection = fixture.connection().await; + connection + .execute("CREATE TABLE restored (value integer); INSERT INTO restored VALUES (7)") + .await + .unwrap(); + // 使用环境密钥的部署即使要求备份密钥，也可能没有磁盘 secret.key。 + std::fs::remove_file(fixture.source().join("data/secret.key")).unwrap(); + let backup = |archive: &Path| { + fixture + .cli() + .args([ + "backup", + "--include-data-dir", + "--include-secret-key", + "--output", + ]) + .arg(archive) + .env("UTOPIA_SECRET_KEY", "11".repeat(32)) + .output() + .unwrap() + }; + let restore = |archive: &Path, target: &Path, flags: &[&str]| { + fixture + .cli() + .args(["restore", "--from"]) + .arg(archive) + .arg("--target-data-dir") + .arg(target) + .args(flags) + .output() + .unwrap() + }; + let archive = fixture.root.path().join("real-backup.tar.gz"); + assert_success(&backup(&archive)); + connection.execute("DROP TABLE restored").await.unwrap(); + assert_success(&restore( + &archive, + &fixture.target(), + &["--yes", "--dry-run"], + )); + let tables: i64 = + sqlx::query_scalar("SELECT count(*) FROM pg_catalog.pg_tables WHERE schemaname = 'public'") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(tables, 0); + assert!(!fixture.target().exists()); + assert_success(&restore(&archive, &fixture.target(), &["--yes"])); + let restored: i32 = sqlx::query_scalar("SELECT value FROM restored") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(restored, 7); + assert_eq!( + std::fs::read(fixture.target().join("files/new.bin")).unwrap(), + b"new data" + ); + assert!(!fixture.target().join("secret.key").exists()); + + connection + .execute("UPDATE restored SET value = 99") + .await + .unwrap(); + let unused_target = fixture.root.path().join("database-guard-target"); + for flags in [vec![], vec!["--yes"], vec!["--yes", "--dry-run"]] { + assert_failure( + &restore(&archive, &unused_target, &flags), + "target database is non-empty", + ); + let unchanged: i32 = sqlx::query_scalar("SELECT value FROM restored") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(unchanged, 99); + assert!(!unused_target.exists()); + } + fixture.old_data(); + assert_success(&restore(&archive, &fixture.target(), &["--yes", "--force"])); + let restored: i32 = sqlx::query_scalar("SELECT value FROM restored") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(restored, 7); + assert!(!fixture.target().join("files/old.bin").exists()); + assert!(!fixture.target().join("secret.key").exists()); + + // 准备连接的会话设置允许建索引；pg_restore 的新连接没有该设置，索引重建会失败。 + connection + .execute( + "CREATE FUNCTION fail_restore(integer) RETURNS integer + LANGUAGE plpgsql IMMUTABLE AS $$ + BEGIN + IF current_setting('restore.test_source', true) IS DISTINCT FROM 'yes' THEN + RAISE EXCEPTION 'simulated index restore failure'; + END IF; + RETURN $1; + END $$; + SET restore.test_source='yes'; + CREATE INDEX restore_failure ON restored (fail_restore(value))", + ) + .await + .unwrap(); + let failing_archive = fixture.root.path().join("failing-backup.tar.gz"); + assert_success(&backup(&failing_archive)); + connection + .execute("UPDATE restored SET value = 99") + .await + .unwrap(); + std::fs::remove_file(fixture.target().join("files/new.bin")).unwrap(); + fixture.old_data(); + let failed = restore(&failing_archive, &fixture.target(), &["--yes", "--force"]); + assert_failure(&failed, "pg_restore exited"); + assert!(String::from_utf8_lossy(&failed.stderr).contains("simulated index restore failure")); + fixture.assert_old_data(); + // 索引重建错误应回滚整次数据库恢复，恢复前的值和原数据目录都必须保留。 + let unchanged: i32 = sqlx::query_scalar("SELECT value FROM restored") + .fetch_one(&mut connection) + .await + .unwrap(); + assert_eq!(unchanged, 99); + assert!(String::from_utf8_lossy(&failed.stderr).contains("rolled back")); +} diff --git a/crates/utopia-cli/tests/support/pg_restore.rs b/crates/utopia-cli/tests/support/pg_restore.rs new file mode 100644 index 000000000..e3d7c9118 --- /dev/null +++ b/crates/utopia-cli/tests/support/pg_restore.rs @@ -0,0 +1,54 @@ +use std::io::Write; + +fn main() { + let args: Vec<_> = std::env::args().skip(1).collect(); + if args == ["--version"] { + println!("pg_restore test double"); + return; + } + let mut log = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(std::env::var_os("UTOPIA_RESTORE_TEST_LOG").unwrap()) + .unwrap(); + writeln!(log, "{}", args.join(" ")).unwrap(); + let dump = std::path::Path::new(args.last().expect("pg_restore dump argument")); + assert!( + dump.is_file(), + "pg_restore dump argument must name a file: {}", + dump.display() + ); + assert!( + dump.starts_with(std::env::current_dir().unwrap()), + "restore staging must stay in current working directory: {}", + dump.display() + ); + if let Some(old_file) = std::env::var_os("UTOPIA_RESTORE_TEST_OLD_FILE") { + assert!( + std::path::Path::new(&old_file).is_file(), + "old data was removed before pg_restore" + ); + } + if std::env::var_os("UTOPIA_RESTORE_TEST_FAIL").is_some() { + eprintln!("simulated pg_restore failure"); + std::process::exit(23); + } + if let Some(root) = std::env::var_os("UTOPIA_RESTORE_TEST_REMOVE_PREPARED") { + let root = std::path::Path::new(&root).canonicalize().unwrap(); + for entry in std::fs::read_dir(&root).unwrap() { + let entry = entry.unwrap(); + if entry + .file_name() + .to_string_lossy() + .starts_with(".utopia-restore-data-") + { + let prepared = entry.path().join("new").canonicalize().unwrap(); + assert!( + prepared.starts_with(&root), + "only remove this fixture's staged files" + ); + std::fs::remove_dir_all(prepared).unwrap(); + } + } + } +} diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs index e2ebc1840..8bac97522 100644 --- a/crates/utopia-core/src/models.rs +++ b/crates/utopia-core/src/models.rs @@ -160,6 +160,14 @@ pub struct Source { pub last_sync_status: String, pub last_sync_error: Option, pub last_sync_added: i32, + /// 工单来源读到了哪里（ADR 0072）。NULL = 没有游标：首轮同步，或者这一类 + /// 来源不用游标。 + /// + /// **说的是「读到哪」，不是「这一轮什么时候跑」**——后者是 `last_sync_at`。 + /// 一轮读不完是常事（同一个更新时刻上可能挂着一千条工单），游标说的是 + /// 「消费到了哪个更新时刻，且那一刻的整组都读完了」；下一轮从那儿接着读， + /// 而不是把这一轮没读完的跳过去（#1212） + pub sync_checkpoint: Option>, /// api 来源的推送密钥（明文；查看走 Editor 权限的专用端点，列表响应不带） #[serde(skip_serializing)] pub ingest_token: Option, @@ -211,6 +219,7 @@ mod source_extracts_tests { last_sync_status: "never".into(), last_sync_error: None, last_sync_added: 0, + sync_checkpoint: None, ingest_token: None, created_at: chrono::Utc::now(), } @@ -393,6 +402,8 @@ pub struct SourceView { pub last_sync_status: String, pub last_sync_error: Option, pub last_sync_added: i32, + /// 工单来源读到了哪里（ADR 0072）。见 [`Source::sync_checkpoint`] + pub sync_checkpoint: Option>, pub doc_count: i64, /// 已标记"不在来源中"的文档数（url 全集对账 / custom 墓碑产生） pub missing_count: i64, @@ -483,15 +494,18 @@ pub struct ChunkView { pub filename: String, } -/// 分块带上它来自哪个来源。`external_id` 只有推送来源（api / statements）才有： -/// 推送方给的那个 id，去掉库里加的 `{kind}:` 前缀，好让它拿回去对自己的记录（#1271） +/// 分块带上它来自哪个来源，连同来源的 kind 和文档的 `external_key`：推送来源的 +/// `external_id` 由这两样反推（`sources_routes::pushed_identity`，#1271） #[derive(Debug, Clone, Serialize, sqlx::FromRow)] pub struct SourcedChunk { #[serde(flatten)] #[sqlx(flatten)] pub chunk: ChunkView, pub source_id: Option, - pub external_id: Option, + #[serde(skip)] + pub source_kind: Option, + #[serde(skip)] + pub external_key: Option, } #[derive(Debug, Clone, Serialize, sqlx::FromRow)] @@ -1136,10 +1150,12 @@ pub struct ConflictView { pub old_subject: String, pub old_object: Option, pub old_valid_from: Option>, + pub old_valid_from_precision: Option, pub new_fact_id: Uuid, pub new_subject: String, pub new_object: Option, pub new_valid_from: Option>, + pub new_valid_from_precision: Option, pub new_confidence: f32, } diff --git a/crates/utopia-extract/src/ontology_agent.rs b/crates/utopia-extract/src/ontology_agent.rs index 510749465..944841f7c 100644 --- a/crates/utopia-extract/src/ontology_agent.rs +++ b/crates/utopia-extract/src/ontology_agent.rs @@ -128,8 +128,10 @@ AND serve the questions. Rules:\n\ Do not propose an element that binds nothing.\n\ - Prefer serving a question. An element that serves no question must bind at least two statements' \ worth of shapes, or be a class that several kind words name.\n\ -- Write the definition for the aligner, in the language of the ontology: one or two sentences saying \ -what the property asserts, what its subject and object are, and what it does NOT cover.\n\ +- Write every label and definition in {LANGUAGE}, even where the shapes, the ontology or these \ +instructions use another; keys stay snake_case ASCII. The definition is for the aligner: one or two \ +sentences saying what the property asserts, what its subject and object are, and what it does NOT \ +cover.\n\ - A property whose object is a value (a number, a date, text) is an attribute: value=true with a \ datatype (number | date | text | bool) and no ranges. Otherwise value=false with domains and ranges \ as class keys, existing or proposed in this same answer.\n\ @@ -152,7 +154,15 @@ pub fn build_proposal_messages( kind_words: &[OpenKindWord<'_>], glossary: &Glossary<'_>, questions: &[QuestionItem<'_>], + ontology_lang: &str, ) -> Vec { + // 标签与定义按库的 ontology_lang 写（0004 决定 3、5），不跟文档、本体或这段英文指令： + // zh 的库读英文文档、装英文包，照样写中文。写给模型看的是 "Chinese"，它认得懂，未必认得懂 "zh" + let language = if ontology_lang == "zh" { + "Chinese" + } else { + "English" + }; let mut user = String::new(); user.push_str("Ontology classes:\n"); if glossary.classes.is_empty() { @@ -240,7 +250,7 @@ pub fn build_proposal_messages( vec![ ChatMessage { role: "system".into(), - content: SYSTEM.into(), + content: SYSTEM.replace("{LANGUAGE}", language), }, ChatMessage { role: "user".into(), @@ -522,8 +532,9 @@ mod tests { id: 1, text: "Who supplies whom?", }]; - let m = build_proposal_messages(&sigs, &kws, &glossary, &qs); + let m = build_proposal_messages(&sigs, &kws, &glossary, &qs, "zh"); assert_eq!(m.len(), 2); + assert!(m[0].content.contains("definition in Chinese")); let user = &m[1].content; assert!(user.contains("Other existing properties"), "{user}"); assert!(user.contains("- p999 · somethingElse"), "{user}"); diff --git a/crates/utopia-llm/src/lib.rs b/crates/utopia-llm/src/lib.rs index bd95945d0..b4df40d5e 100644 --- a/crates/utopia-llm/src/lib.rs +++ b/crates/utopia-llm/src/lib.rs @@ -953,6 +953,7 @@ impl LlmClient { stream: bool, ) -> anyhow::Result { use std::sync::atomic::Ordering; + let mut retried = false; loop { let learned = self.tools_without_effort.load(Ordering::Relaxed); let resp = self @@ -975,8 +976,19 @@ impl LlmClient { { tracing::info!(model = %self.model, "端点不收带工具的推理强度，带工具的请求改发 none 并记住"); self.tools_without_effort.store(true, Ordering::Relaxed); + retried = true; continue; } + // 改发 none 也被拒，且错误还是在说推理强度：这个端点连 none 都不收，记住它只会让以后每一问都带着一个错的值。 + // 别的 400（上下文超长、工具定义不对）跟 none 无关，记住的值留着 + if retried + && status == reqwest::StatusCode::BAD_REQUEST + && err_detail(&parsed, &raw) + .to_lowercase() + .contains("reasoning_effort") + { + self.tools_without_effort.store(false, Ordering::Relaxed); + } return Err(failure("LLM", status, retry_after, &parsed, &raw)); } } @@ -1764,6 +1776,70 @@ mod tests { ); } + #[tokio::test] + async fn a_deployment_that_refuses_none_too_is_not_sent_none_again() { + use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400).set_body_json(serde_json::json!({ "error": { + "message": "Function tools with reasoning_effort are not supported for gpt-x." } }))) + .mount(&server) + .await; + let client = + LlmClient::new(&server.uri(), None, "m").with_reasoning_effort(Some("low".into())); + let tool = serde_json::json!([{ "type": "function", "function": { "name": "ping" } }]); + let ask = [serde_json::json!({ "role": "user", "content": "x" })]; + assert!(client.chat_tools(&ask, &tool).await.is_err()); + let sent: Vec = server + .received_requests() + .await + .unwrap() + .iter() + .map(|r| serde_json::from_slice(&r.body).unwrap()) + .collect(); + assert_eq!( + sent.iter() + .map(|b| b["reasoning_effort"].clone()) + .collect::>(), + ["low", "none"] + ); + assert_eq!( + client.tools_body(&ask, Some(&tool), None, false)["reasoning_effort"], + "low", + "the flag is cleared" + ); + } + + #[tokio::test] + async fn an_unrelated_400_on_the_retry_keeps_none() { + use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; + let server = MockServer::start().await; + Mock::given(method("POST")) + .respond_with(ResponseTemplate::new(400).set_body_json(serde_json::json!({ "error": { + "message": "Function tools with reasoning_effort are not supported for gpt-x." } }))) + .up_to_n_times(1) + .mount(&server) + .await; + Mock::given(method("POST")) + .respond_with( + ResponseTemplate::new(400).set_body_json(serde_json::json!({ "error": { + "message": "This model's maximum context length is 8192 tokens." } })), + ) + .mount(&server) + .await; + let client = + LlmClient::new(&server.uri(), None, "m").with_reasoning_effort(Some("low".into())); + let tool = serde_json::json!([{ "type": "function", "function": { "name": "ping" } }]); + let ask = [serde_json::json!({ "role": "user", "content": "x" })]; + assert!(client.chat_tools(&ask, &tool).await.is_err()); + assert_eq!(server.received_requests().await.unwrap().len(), 2); + assert_eq!( + client.tools_body(&ask, Some(&tool), None, false)["reasoning_effort"], + "none", + "the flag is kept" + ); + } + #[tokio::test] async fn another_400_on_a_tool_request_is_not_retried() { use wiremock::{matchers::method, Mock, MockServer, ResponseTemplate}; diff --git a/crates/utopia-search/src/lib.rs b/crates/utopia-search/src/lib.rs index cd0920a84..f62fcaf62 100644 --- a/crates/utopia-search/src/lib.rs +++ b/crates/utopia-search/src/lib.rs @@ -62,7 +62,13 @@ impl SearchIndex { .register(JIEBA, tantivy_jieba::JiebaTokenizer::new()); let writer = index.writer(WRITER_HEAP)?; - let reader = index.reader()?; + // 每次写都在 writer 锁里自己 reload；再留着自动重载，那条异步重载可能晚到、 + // 带着更旧的提交把刚写进去的文档又遮掉。这个进程握着唯一的 writer， + // 没有别人会提交，手动重载不会漏掉什么 + let reader = index + .reader_builder() + .reload_policy(tantivy::ReloadPolicy::Manual) + .try_into()?; let analyzer = index .tokenizers() .get(JIEBA) @@ -135,8 +141,7 @@ impl SearchIndex { /// 查询用与索引完全一致的 jieba analyzer 切词后按 OR 组合—— /// 不能走 QueryParser：CJK 整句会被当成短语查询（要求词连续出现），召回归零。 /// - /// `documents`：只在这些文档里找。它和 kb 一样是 `Must`，`limit` 数的就只是 - /// 允许的块，而不是先取全库的前几名再筛 + /// `documents`：只在这些文档里找。它和 kb 一样是 `Must`，`limit` 数的就只是允许的块 pub fn search( &self, kb_id: &str, diff --git a/crates/utopia-server/src/adjudication.rs b/crates/utopia-server/src/adjudication.rs index 9f67f2bde..015ff1d65 100644 --- a/crates/utopia-server/src/adjudication.rs +++ b/crates/utopia-server/src/adjudication.rs @@ -37,6 +37,11 @@ fn pair_key(item: &ReviewItem, precedents: &[String]) -> String { digest.iter().map(|b| format!("{b:02x}")).collect() } +#[cfg(test)] +pub(crate) fn seed_measurement_cache_key(item: &ReviewItem, precedents: &[String]) -> String { + pair_key(item, precedents) +} + /// 攒批没定的对要不要带工具再看一遍（0028）：没判决或把握不到线；硬规则拦得住的不看 /// （大类不同、版本尾巴、含名字的一句话——再看也改不了规则）；有撤回的不看，那是人的事 fn wants_another_look( @@ -382,23 +387,24 @@ async fn apply_verdict( why: Option<&str>, ) -> anyhow::Result { // 有把握就动手，不再抽一成给人（0026 修订）：队列里等人的，只剩机器拿不准、 - // 或闸门说合了会送出图外的那些 + // 或闸门说合了会送出图外、会推翻来源身份声明的那些 let outcome = match same { Some(true) if conf >= AUTO_CONF => { // 执行闸门（0027）：合并会立刻送出图外的东西——违规、派生、答案——留给人， - // 把握再高也不动手。人看到的是留下的原因，不是「裁决器没把握」 - let impact = utopia_store::execution_gate::impact_of( + // 身份来源也留给人（0071），把握再高也不动手。人看到的是留下的原因， + // 不是「裁决器没把握」 + if let Some(hold) = utopia_store::execution_gate::hold_for_merge( &state.pool, kb_id, item.left.id, item.right.id, ) - .await?; - if let Some(hold) = utopia_store::execution_gate::hold(&impact) { + .await? + { utopia_store::resolution::escalate_review( &state.pool, item.id, - &format!("escalate_impact|{hold}"), + &hold.review_reason(), ) .await?; return Ok(Outcome::Escalated); @@ -494,6 +500,10 @@ async fn apply_verdict( Ok(outcome) } +#[cfg(test)] +#[path = "adjudication_identity_tests.rs"] +mod identity_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/crates/utopia-server/src/adjudication_identity_tests.rs b/crates/utopia-server/src/adjudication_identity_tests.rs new file mode 100644 index 000000000..1e13d5388 --- /dev/null +++ b/crates/utopia-server/src/adjudication_identity_tests.rs @@ -0,0 +1,252 @@ +use super::*; +use serde_json::{json, Value}; +use wiremock::{matchers::method, matchers::path, Mock, MockServer, Request, ResponseTemplate}; + +async fn decide_pair( + governance: bool, + same: bool, + identity: bool, + proposal: &str, +) -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = sqlx::PgPool::connect(&url).await?; + let server = MockServer::start().await; + let verdict = if same { "same" } else { "different" }; + Mock::given(method("POST")) + .and(path("/chat/completions")) + .respond_with(move |request: &Request| { + let body: Value = request.body_json().unwrap(); + let answer = json!({"verdict": verdict, "confidence": 0.99, "why": "source checked"}); + let message = if body["tools"].is_array() { + json!({"role":"assistant","content":null,"tool_calls":[{ + "id":"decide-1","type":"function","function":{ + "name":"decide","arguments":answer.to_string() + } + }]}) + } else { + json!({"role":"assistant","content":json!({"verdicts":[{ + "i":0,"verdict":verdict,"confidence":0.99,"why":"source checked" + }]}).to_string()}) + }; + ResponseTemplate::new(200).set_body_json(json!({ + "choices":[{"message":message,"finish_reason":"stop"}] + })) + }) + .mount(&server) + .await; + + let [org, ws, kb, source, doc, chunk, prose, prose_chunk, left, right]: [Uuid; 10] = + std::array::from_fn(|_| Uuid::now_v7()); + let result = async { + let similarity = utopia_core::review_reasons::similarity_proposed(Some(proposal)); + let right_name = if similarity { "Account" } else { "InvoiceExporter" }; + let config = if identity { + json!({"names":"identity"}) + } else { + json!({}) + }; + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','identity-merge'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','identity-merge'); + INSERT INTO knowledge_bases(id,workspace_id,name,governance) + VALUES ('{kb}','{ws}','identity-merge',{governance}); + INSERT INTO sources(id,kb_id,kind,name,config) + VALUES ('{source}','{kb}','statements','identities',$${config}$$::jsonb); + INSERT INTO documents(id,kb_id,filename,sha256,source_id) VALUES + ('{doc}','{kb}','records.json','{doc}','{source}'), + ('{prose}','{kb}','notes.md','{prose}',NULL); + INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES + ('{chunk}','{kb}','{doc}',0,'billing/export'), + ('{prose_chunk}','{kb}','{prose}',0,'billing/export'); + INSERT INTO entities(id,kb_id,canonical_name) VALUES + ('{left}','{kb}','InvoiceExporter'),('{right}','{kb}','{right_name}');" + )) + .execute(&pool) + .await?; + + // 别名同样算数，证据里散文和标识混在一起也算：名字带标识的是哪一侧都得拦下 + let named = if governance { left } else { right }; + for evidence in [prose_chunk, chunk] { + utopia_store::names::record( + &pool, + kb, + named, + "billing/export", + Some(utopia_store::names::NameSource { + chunk_id: evidence, + quote: "billing/export", + }), + None, + ) + .await?; + } + utopia_store::resolution::create_review( + &pool, + kb, + left, + right, + 0.9, + proposal, + utopia_store::resolution::ReviewStage::Adjudicating, + ) + .await?; + utopia_store::settings::upsert( + &pool, + ws, + Some(&server.uri()), + None, + Some("identity-merge"), + None, + None, + None, + None, + ) + .await?; + let dir = tempfile::tempdir()?; + let cfg = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open( + &dir.path().join("search"), + )?); + let state = AppState::new(pool.clone(), &cfg, search, "test-only".into()); + if governance { + crate::governance::govern(&state, kb).await?; + } else { + adjudicate_entities(&state, kb).await?; + } + let (review, status, stage, reason): (Uuid, String, String, String) = sqlx::query_as( + "SELECT id, status, stage, reason FROM resolution_reviews WHERE kb_id=$1", + ) + .bind(kb) + .fetch_one(&pool) + .await?; + let held = identity && same; + assert_eq!( + status, + if held { + "pending" + } else if same { + "merged" + } else { + "kept" + } + ); + if held { + assert_eq!(stage, "human"); + assert_eq!(reason, "escalate_identity"); + } + let live: i64 = sqlx::query_scalar( + "SELECT count(*) FROM entities WHERE kb_id=$1 AND merged_into IS NULL", + ) + .bind(kb) + .fetch_one(&pool) + .await?; + assert_eq!(live, if same && !identity { 1 } else { 2 }); + let second_look = same && similarity; + let calls = server.received_requests().await.unwrap().len(); + assert_eq!(calls, if second_look { 2 } else { 1 }); + if governance || second_look { + let (action, status, merge_id, reason): (String, String, Option, Option) = + sqlx::query_as("SELECT action, status, merge_id, reason FROM agent_decisions WHERE kb_id=$1") + .bind(kb) + .fetch_one(&pool) + .await?; + assert_eq!(action, if same { "merge" } else { "keep" }); + assert_eq!(status, if held { "proposed" } else { "applied" }); + if held { + assert!(merge_id.is_none()); + if governance { + assert_eq!( + reason.as_deref(), + Some("held for a person: a source treats one side's names as identifiers; source checked") + ); + } + } + } + if held && !governance { + // 缓存的模型判决重放时也得再查一次来源，不能直接沿用 + sqlx::query("UPDATE resolution_reviews SET stage='adjudicating' WHERE id=$1") + .bind(review) + .execute(&pool) + .await?; + adjudicate_entities(&state, kb).await?; + let replay: (String, String, String) = + sqlx::query_as("SELECT status, stage, reason FROM resolution_reviews WHERE id=$1") + .bind(review) + .fetch_one(&pool) + .await?; + assert_eq!( + replay, + ("pending".into(), "human".into(), "escalate_identity".into()) + ); + assert_eq!(server.received_requests().await.unwrap().len(), calls); + + // 关掉声明后，缓存的判决直接放行，不再调用模型 + sqlx::query("UPDATE sources SET config='{}'::jsonb WHERE id=$1") + .bind(source) + .execute(&pool) + .await?; + sqlx::query("UPDATE resolution_reviews SET stage='adjudicating' WHERE id=$1") + .bind(review) + .execute(&pool) + .await?; + adjudicate_entities(&state, kb).await?; + let status: String = + sqlx::query_scalar("SELECT status FROM resolution_reviews WHERE id=$1") + .bind(review) + .fetch_one(&pool) + .await?; + assert_eq!(status, "merged"); + let live: i64 = sqlx::query_scalar( + "SELECT count(*) FROM entities WHERE kb_id=$1 AND merged_into IS NULL", + ) + .bind(kb) + .fetch_one(&pool) + .await?; + assert_eq!(live, 1); + assert_eq!(server.received_requests().await.unwrap().len(), calls); + } + Ok::<_, anyhow::Error>(()) + } + .await; + sqlx::query("DELETE FROM jobs WHERE payload->>'kb_id'=$1") + .bind(kb.to_string()) + .execute(&pool) + .await?; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(org) + .execute(&pool) + .await?; + pool.close().await; + result +} + +#[tokio::test] +async fn a_same_verdict_on_an_identity_entity_waits_for_a_person() -> anyhow::Result<()> { + for governance in [false, true] { + for proposal in ["ambiguous_name|0.7", "name_vector|0.9"] { + decide_pair(governance, true, true, proposal).await?; + } + } + Ok(()) +} + +#[tokio::test] +async fn an_identity_source_still_allows_a_kept_verdict() -> anyhow::Result<()> { + for governance in [false, true] { + decide_pair(governance, false, true, "name_vector|0.9").await?; + } + Ok(()) +} + +#[tokio::test] +async fn a_source_without_identity_names_still_allows_a_merge() -> anyhow::Result<()> { + for governance in [false, true] { + decide_pair(governance, true, false, "name_vector|0.9").await?; + } + Ok(()) +} diff --git a/crates/utopia-server/src/api/documents_routes.rs b/crates/utopia-server/src/api/documents_routes.rs index 62746b758..f740cde67 100644 --- a/crates/utopia-server/src/api/documents_routes.rs +++ b/crates/utopia-server/src/api/documents_routes.rs @@ -227,11 +227,11 @@ pub async fn by_external_id( ) -> ApiResult { require_reader_kb(&state, &reader, kb_id).await?; let source = super::sources_routes::source_in_kb(&state, kb_id, source_id).await?; - if !super::sources_routes::is_push_kind(&source.kind) { + if !crate::pushed_keys::is_push_kind(&source.kind) { return Err(AppError::NotFound.into()); } let external_id = query.external_id.trim(); - let key = super::sources_routes::pushed_key(&source, external_id); + let key = crate::pushed_keys::pushed_key(&source, external_id); let found = utopia_store::documents::find_by_external_key(&state.pool, source.id, &key) .await? .ok_or(AppError::NotFound)?; diff --git a/crates/utopia-server/src/api/review_conflict_tests.rs b/crates/utopia-server/src/api/review_conflict_tests.rs new file mode 100644 index 000000000..c765b175e --- /dev/null +++ b/crates/utopia-server/src/api/review_conflict_tests.rs @@ -0,0 +1,221 @@ +use axum::body::{to_bytes, Body}; +use axum::http::Request; +use chrono::{DateTime, Utc}; +use serde_json::{json, Value}; +use sqlx::PgPool; +use std::sync::Arc; +use tower::ServiceExt; +use uuid::Uuid; + +async fn send( + app: &axum::Router, + token: &str, + kb: Uuid, + conflict: Uuid, + body: Value, +) -> anyhow::Result<(u16, Value)> { + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/v1/kbs/{kb}/conflicts/{conflict}")) + .header("authorization", format!("Bearer {token}")) + .header("content-type", "application/json") + .body(Body::from(body.to_string()))?, + ) + .await?; + let status = response.status().as_u16(); + Ok(( + status, + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await?)?, + )) +} + +async fn pair( + pool: &PgPool, + kb: Uuid, + predicate: Uuid, + marked: bool, +) -> anyhow::Result<(Uuid, Uuid, Uuid)> { + let subject = Uuid::now_v7(); + sqlx::query("INSERT INTO entities(id,kb_id,canonical_name) VALUES ($1,$2,'Lease')") + .bind(subject) + .bind(kb) + .execute(pool) + .await?; + let [old, new, conflict] = std::array::from_fn(|_| Uuid::now_v7()); + sqlx::query("INSERT INTO facts(id,kb_id,subject_id,predicate_id,object_value,layer,confidence, + valid_from,valid_from_precision,valid_to,valid_to_precision,end_after_period) + VALUES ($1,$3,$4,$5,'\"old\"','typed',0.9,'2024-01-01','year',NULL,NULL,FALSE), + ($2,$3,$4,$5,'\"new\"','typed',0.9,'2024-01-01','month', + CASE WHEN $6 THEN '2025-01-01'::timestamptz END, + CASE WHEN $6 THEN 'year' END,$6)") + .bind(old).bind(new).bind(kb).bind(subject).bind(predicate).bind(marked).execute(pool).await?; + sqlx::query("INSERT INTO fact_conflicts(id,kb_id,old_fact_id,new_fact_id,reason) VALUES ($1,$2,$3,$4,'simultaneous')") + .bind(conflict).bind(kb).bind(old).bind(new).execute(pool).await?; + Ok((old, new, conflict)) +} + +async fn counts(pool: &PgPool, kb: Uuid) -> anyhow::Result<(i64, i64, i64)> { + Ok(sqlx::query_as("SELECT (SELECT count(*) FROM facts WHERE kb_id=$1), + (SELECT count(*) FROM facts WHERE kb_id=$1 AND invalidated_at IS NOT NULL), + (SELECT count(*) FROM audit_events WHERE kb_id=$1)") + .bind(kb).fetch_one(pool).await?) +} + +#[tokio::test] +async fn human_conflict_dates_and_settlement_commit_with_the_actor() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + utopia_store::db::migrate(&pool).await?; + let [org, ws, kb, editor, viewer, predicate] = std::array::from_fn(|_| Uuid::now_v7()); + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','human-conflict'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','human-conflict'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','human-conflict'); + INSERT INTO users(id,org_id,email,display_name,password_hash) VALUES + ('{editor}','{org}','{editor}@conflict.test','editor','unused'), + ('{viewer}','{org}','{viewer}@conflict.test','viewer','unused'); + INSERT INTO kb_members(kb_id,user_id,role) VALUES ('{kb}','{editor}','editor'),('{kb}','{viewer}','viewer'); + INSERT INTO relation_types(id,kb_id,key,label,kind,datatype,temporal,functional) VALUES + ('{predicate}','{kb}','status','status','attribute','text','state',TRUE);" + )).execute(&pool).await?; + let result = async { + let dir = tempfile::tempdir()?; + let cfg = utopia_core::config::AppConfig { data_dir: dir.path().to_string_lossy().into_owned(), ..Default::default() }; + let search = Arc::new(utopia_search::SearchIndex::open(&dir.path().join("search"))?); + let state = crate::state::AppState::new(pool.clone(), &cfg, search, "conflict-test".into()); + let token = crate::auth::issue_token(&state, editor)?; + let viewer_token = crate::auth::issue_token(&state, viewer)?; + let app = crate::api::router(state, &cfg); + let (_, new, conflict) = pair(&pool, kb, predicate, true).await?; + let [statement, document, chunk] = std::array::from_fn(|_| Uuid::now_v7()); + sqlx::raw_sql(&format!( + "INSERT INTO documents(id,kb_id,filename,sha256,status) VALUES ('{document}','{kb}','amendment.md','{document}','ready'); + INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES ('{chunk}','{kb}','{document}',0,'The lease starts in June 2024.'); + INSERT INTO facts(id,kb_id,subject_id,object_value,layer,phrase,confidence) + SELECT '{statement}',kb_id,subject_id,object_value,'open','starts',confidence FROM facts WHERE id='{new}'; + INSERT INTO typed_fact_sources(fact_id,statement_id) VALUES ('{new}','{statement}'); + UPDATE facts SET from_statement_id='{statement}' WHERE id='{new}'; + INSERT INTO fact_evidence(fact_id,chunk_id,quote) VALUES ('{new}','{chunk}','The lease starts in June 2024.'); + INSERT INTO fact_qualifiers(fact_id,qualifier_type_id,value) VALUES ('{new}','{predicate}','\"approved\"');" + )).execute(&pool).await?; + let before = counts(&pool, kb).await?; + for (body, code) in [ + (json!({"action":"retime_new"}), "new_from_required"), + (json!({"action":"retime_new","new_from":"2026-01-01T00:00:00Z"}), "interval_inverted"), + (json!({"action":"retime_new","new_from":"2024-06-01T00:00:00Z","new_from_precision":"unknown"}), "bad_precision"), + ] { + let (status, reply) = send(&app, &token, kb, conflict, body).await?; + assert_eq!((status, reply["code"].as_str()), (422, Some(code))); + assert_eq!(counts(&pool, kb).await?, before); + } + assert_eq!(send(&app, &viewer_token, kb, conflict, json!({"action":"keep"})).await?.0, 403); + assert_eq!(send(&app, &token, Uuid::now_v7(), conflict, json!({"action":"keep"})).await?.0, 404); + // A rejected audit insert must roll back even a successful fact rewrite. + // Actors are deliberately not foreign keys in the append-only ledger; + // inject the failure only for this fixture's event instead. + let trigger = format!("fail_conflict_audit_{}", kb.simple()); + sqlx::raw_sql(&format!( + "CREATE FUNCTION {trigger}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.kb_id='{kb}' THEN RAISE EXCEPTION 'conflict audit failure'; END IF; + RETURN NEW; END $$; + CREATE TRIGGER {trigger} BEFORE INSERT ON audit_events FOR EACH ROW EXECUTE FUNCTION {trigger}();" + )).execute(&pool).await?; + let failed = utopia_store::temporal::resolve_review_conflict(&pool, kb, conflict, + utopia_store::temporal::ConflictDecision { action: "retime_new", close_at: None, close_precision: "day", + new_from: Some("2024-06-01T00:00:00Z".parse()?), new_precision: "month" }, editor).await; + sqlx::raw_sql(&format!("DROP TRIGGER {trigger} ON audit_events; DROP FUNCTION {trigger}();")) + .execute(&pool).await?; + assert!(failed.is_err()); + assert_eq!(counts(&pool, kb).await?, before); + let listed = utopia_store::temporal::list_conflicts(&pool, kb, 10, 0).await?; + assert_eq!(listed[0].new_valid_from_precision.as_deref(), Some("month")); + assert_eq!(send(&app, &token, kb, conflict, json!({"action":"retime_new", + "new_from":"2024-06-19T14:00:00Z","new_from_precision":"month"})).await?.0, 200); + let rewritten: (DateTime, String, DateTime, String, bool, bool, String) = sqlx::query_as( + "SELECT valid_from,valid_from_precision,valid_to,valid_to_precision,end_after_period,end_derived,corrected_ends + FROM facts WHERE supersedes=$1 AND invalidated_at IS NULL") + .bind(new).fetch_one(&pool).await?; + assert_eq!(rewritten, ("2024-06-01T00:00:00Z".parse()?, "month".into(), + "2025-01-01T00:00:00Z".parse()?, "year".into(), true, false, "start".into())); + let provenance: (Uuid, Uuid, Uuid, String, Value) = sqlx::query_as( + "SELECT f.from_statement_id,s.statement_id,e.chunk_id,e.quote,q.value FROM facts f + JOIN typed_fact_sources s ON s.fact_id=f.id JOIN fact_evidence e ON e.fact_id=f.id + JOIN fact_qualifiers q ON q.fact_id=f.id WHERE f.supersedes=$1 AND f.invalidated_at IS NULL") + .bind(new).fetch_one(&pool).await?; + assert_eq!(provenance, (statement, statement, chunk, "The lease starts in June 2024.".into(), json!("approved"))); + let (resolution, actor, detail): (String, Uuid, Value) = sqlx::query_as( + "SELECT c.resolution,a.actor_id,a.detail FROM fact_conflicts c JOIN audit_events a ON a.target_id=c.id + WHERE c.id=$1 AND a.action='conflict.retime_new'") + .bind(conflict).fetch_one(&pool).await?; + assert_eq!((resolution.as_str(), actor, &detail["new_from_precision"]), ("retimed_new", editor, &json!("month"))); + // 实体历史把重定时那一行记在人头上，不记给引擎 + let subject: Uuid = sqlx::query_scalar("SELECT subject_id FROM facts WHERE id=$1") + .bind(new).fetch_one(&pool).await?; + let retimed: Uuid = sqlx::query_scalar("SELECT id FROM facts WHERE supersedes=$1 AND invalidated_at IS NULL") + .bind(new).fetch_one(&pool).await?; + let (history, _) = utopia_store::graph::entity_history(&pool, kb, subject, 50, 0).await?; + for event in history.iter().filter(|e| e.kind == "corrected") { + if event.fact_id == Some(retimed) { + assert_eq!((event.actor_name.as_deref(), event.action.as_deref()), + (Some("editor"), Some("conflict.retime_new"))); + } else { + assert_ne!(event.action.as_deref(), Some("conflict.retime_new")); + } + } + assert!(history.iter().any(|e| e.kind == "corrected" && e.fact_id == Some(retimed))); + + for explicit in [false, true] { + let (old, _, c) = pair(&pool, kb, predicate, false).await?; + let body = if explicit { json!({"action":"close","close_at":"2024-08-13T00:00:00Z","close_at_precision":"month"}) } + else { json!({"action":"close"}) }; + // The simultaneous default is an empty state; an explicit date can answer it. + let (status, _) = send(&app, &token, kb, c, body).await?; + if !explicit { assert_eq!(status, 422); continue; } + assert_eq!(status, 200); + let (to, precision): (DateTime, String) = sqlx::query_as("SELECT valid_to,valid_to_precision FROM facts WHERE supersedes=$1 AND invalidated_at IS NULL") + .bind(old).fetch_one(&pool).await?; + assert_eq!((to, precision.as_str()), ("2024-08-01T00:00:00Z".parse()?, "month")); + } + let (_, new, c) = pair(&pool, kb, predicate, false).await?; + sqlx::query("UPDATE facts SET valid_to='2024-05-01',valid_to_precision='month',end_derived=TRUE WHERE id=$1") + .bind(new).execute(&pool).await?; + assert_eq!(send(&app, &token, kb, c, json!({"action":"retime_new","new_from":"2024-06-01T00:00:00Z","new_from_precision":"month"})).await?.0, 200); + let (derived, correction, to): (bool, Option, Option>) = sqlx::query_as("SELECT end_derived,corrected_ends,valid_to FROM facts + WHERE subject_id=(SELECT subject_id FROM facts WHERE id=$1) AND object_value='\"new\"' AND invalidated_at IS NULL") + .bind(new).fetch_one(&pool).await?; + assert!(!derived, "an end reopened by reconciliation is no longer derived"); + assert_eq!(to, None, "retiming lets the engine redraw its old end"); + assert_eq!(correction.as_deref(), Some("start")); + + let (old, new, c) = pair(&pool, kb, predicate, false).await?; + let sibling = Uuid::now_v7(); + sqlx::query("INSERT INTO fact_conflicts(id,kb_id,old_fact_id,new_fact_id,reason) VALUES ($1,$2,$3,$4,'no_time')") + .bind(sibling).bind(kb).bind(new).bind(old).execute(&pool).await?; + // Reversed sibling has old as successor, so it withdraws when new dies. + assert_eq!(send(&app, &token, kb, c, json!({"action":"reject_new"})).await?.0, 200); + let settled: (String, String) = sqlx::query_as("SELECT status,resolution FROM fact_conflicts WHERE id=$1") + .bind(c).fetch_one(&pool).await?; + assert_eq!(settled, ("resolved".into(), "rejected_new".into())); + let withdrawn: (String, Option) = sqlx::query_as("SELECT status,resolution FROM fact_conflicts WHERE id=$1") + .bind(sibling).fetch_one(&pool).await?; + assert_eq!(withdrawn, ("withdrawn".into(), None)); + let (_, _, c) = pair(&pool, kb, predicate, false).await?; + let before = counts(&pool, kb).await?; + let (a, b) = tokio::join!(send(&app, &token, kb, c, json!({"action":"keep"})), send(&app, &token, kb, c, json!({"action":"keep"}))); + let mut statuses = [a?.0, b?.0]; statuses.sort(); + assert_eq!(statuses[0], 200); + assert!(matches!(statuses[1], 404 | 409)); + assert_eq!(counts(&pool, kb).await?, (before.0, before.1, before.2 + 1)); + anyhow::Ok(()) + }.await; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(org) + .execute(&pool) + .await?; + result +} diff --git a/crates/utopia-server/src/api/review_routes.rs b/crates/utopia-server/src/api/review_routes.rs index 8f46a2fd9..fedd15138 100644 --- a/crates/utopia-server/src/api/review_routes.rs +++ b/crates/utopia-server/src/api/review_routes.rs @@ -16,12 +16,13 @@ use crate::state::AppState; #[path = "review_routes_phrase_tests.rs"] mod phrase_tests; +#[cfg(test)] +#[path = "review_conflict_tests.rs"] +mod conflict_tests; + /// 一页多少条。**服务端的默认，不是上限**——前端可以要更少，多则被 clamp 挡住 const REVIEW_PAGE: i64 = 10; -/// 冲突双方的三元组快照：(旧主语, 旧宾语, 新主语, 新宾语, 谓词标签)。 -type ConflictSnapshot = (String, Option, String, Option, String); - /// 事实快照（决策台账用）：reject 后事实从图里消失，台账必须自包含展示文本。 /// 一律在动作执行前取。 pub(crate) async fn fact_snapshot( @@ -226,14 +227,20 @@ pub async fn close_fact( #[derive(Deserialize)] pub struct ConflictBody { - /// close | keep | reject_new + /// close | retime_new | keep | reject_new pub action: String, /// close 且新事实无起点时必填 #[serde(default)] pub close_at: Option>, - /// `close_at` 的精度（year | month | day），不给按日 + /// `close_at` 的精度（year 到 second），不给按日 #[serde(default)] pub close_at_precision: Option, + /// retime_new 时必填：只改新事实起点，终点仍由原文或时间线定。 + #[serde(default)] + pub new_from: Option>, + /// `new_from` 的精度（year 到 second），不给按日。 + #[serde(default)] + pub new_from_precision: Option, } /// 时态冲突裁决（S3：自动闭合拿不准的那些）。 @@ -244,59 +251,22 @@ pub async fn resolve_conflict( Json(body): Json, ) -> ApiResult> { require_kb(&state, &user, kb_id, Role::Editor).await?; - // 快照双方三元组：裁决会作废/改写事实，先抄后动 - let snap: Option = sqlx::query_as( - "SELECT os.canonical_name, oo.canonical_name, ns.canonical_name, no_.canonical_name, - r.label - FROM fact_conflicts c - JOIN facts fo ON fo.id = c.old_fact_id - JOIN facts fn_ ON fn_.id = c.new_fact_id - JOIN entities os ON os.id = fo.subject_id - LEFT JOIN entities oo ON oo.id = fo.object_id - JOIN entities ns ON ns.id = fn_.subject_id - LEFT JOIN entities no_ ON no_.id = fn_.object_id - JOIN relation_types r ON r.id = fo.predicate_id - WHERE c.id = $1 AND c.kb_id = $2", - ) - .bind(conflict_id) - .bind(kb_id) - .fetch_optional(&state.pool) - .await - .ok() - .flatten(); - let close_precision = world_precision(body.close_at_precision.as_deref())?; - utopia_store::temporal::resolve_conflict( + let precision = world_precision(body.close_at_precision.as_deref())?; + let new_precision = world_precision(body.new_from_precision.as_deref())?; + utopia_store::temporal::resolve_review_conflict( &state.pool, kb_id, conflict_id, - &body.action, - body.close_at, - close_precision, + utopia_store::temporal::ConflictDecision { + action: &body.action, + close_at: body.close_at, + close_precision: precision, + new_from: body.new_from, + new_precision, + }, + user.id, ) .await?; - if let Some((os, oo, ns, no, pred)) = snap { - let action = match body.action.as_str() { - "close" => "conflict.close_old", - "keep" => "conflict.keep_both", - _ => "conflict.reject_new", - }; - let _ = utopia_store::audit::record( - &state.pool, - Some(kb_id), - user.id, - action, - "conflict", - Some(conflict_id), - json!({ - "predicate": pred, - "old_subject": os, "old_object": oo, - "new_subject": ns, "new_object": no, - "close_at": body.close_at.map(|t| t.to_rfc3339()), - "close_at_precision": body.close_at.map(|_| close_precision), - }), - ) - .await; - } state.emit_review(kb_id); Ok(Json(json!({ "ok": true }))) } diff --git a/crates/utopia-server/src/api/search_routes.rs b/crates/utopia-server/src/api/search_routes.rs index b7eb2a673..b9f3d7404 100644 --- a/crates/utopia-server/src/api/search_routes.rs +++ b/crates/utopia-server/src/api/search_routes.rs @@ -10,11 +10,7 @@ use crate::error::ApiResult; use crate::retrieval; use crate::state::AppState; -/// 前缀最多几条、每条多长：它们进的是一条 SQL 的数组参数，不是让人塞一份清单 -const MAX_PREFIXES: usize = 8; -const MAX_PREFIX_BYTES: usize = 256; - -/// 不认识的字段答 422：过滤条件还没做，悄悄忽略一个 `filters` 会让调用方以为它生效了 +/// 不认识的字段答 422：悄悄忽略一个拼错的过滤条件，调用方会以为它生效了 #[derive(Deserialize)] #[serde(deny_unknown_fields)] pub struct SearchReq { @@ -27,13 +23,13 @@ pub struct SearchReq { /// 命中的是当时活着的块、当时还没被删的文档 #[serde(default)] pub as_of: Option, - /// 只检索这个来源的文档（必须属于本库，否则 404） + /// 只检索本库这个来源的文档；别的库的来源答 404 #[serde(default)] pub source_id: Option, - /// 只检索 `external_id` 以其中之一开头的文档；前缀是相对 `source_id` 说的， - /// 所以没有 `source_id` 不收 + /// 只检索 `external_id` 以它开头的文档。identity 是推送来源里的，所以要 `source_id` + /// 且它是推送来源，否则 422 #[serde(default)] - pub external_id_prefixes: Option>, + pub external_id_prefix: Option, /// 这次检索算不算「用过」（RC-003）：默认算。探测、评测、健康检查传 false，不留痕 #[serde(default = "record_by_default")] pub record: bool, @@ -43,32 +39,35 @@ fn record_by_default() -> bool { true } -fn check_prefixes(req: &SearchReq) -> Result<(), AppError> { - let Some(prefixes) = &req.external_id_prefixes else { - return Ok(()); +async fn source_filter( + state: &AppState, + kb_id: Uuid, + req: &SearchReq, +) -> ApiResult> { + let source = match req.source_id { + Some(source_id) => { + Some(super::sources_routes::source_in_kb(state, kb_id, source_id).await?) + } + None => None, }; - if req.source_id.is_none() { - return Err(AppError::invalid( - "prefixes_need_source", - "external_id_prefixes requires source_id", - )); - } - if prefixes.is_empty() || prefixes.len() > MAX_PREFIXES { - return Err(AppError::invalid( - "bad_prefixes", - format!("external_id_prefixes takes 1 to {MAX_PREFIXES} entries"), - )); - } - if prefixes - .iter() - .any(|p| p.is_empty() || p.len() > MAX_PREFIX_BYTES) - { + let Some(prefix) = &req.external_id_prefix else { + return Ok(source.map(|source| retrieval::SourceFilter { + source_id: source.id, + key_prefix: None, + })); + }; + let Some(source) = source.filter(|source| crate::pushed_keys::is_push_kind(&source.kind)) + else { return Err(AppError::invalid( - "bad_prefixes", - format!("each external_id prefix is 1 to {MAX_PREFIX_BYTES} bytes"), - )); - } - Ok(()) + "prefix_needs_push_source", + "external_id_prefix needs the source_id of an API or Statements source", + ) + .into()); + }; + Ok(Some(retrieval::SourceFilter { + source_id: source.id, + key_prefix: Some(crate::pushed_keys::pushed_key(&source, prefix)), + })) } pub async fn search( @@ -91,29 +90,12 @@ pub async fn search( ) .into()); } - check_prefixes(&req)?; let as_of = super::graph_routes::parse_instant("as_of", req.as_of.as_deref())?; - let documents = match req.source_id { - Some(source_id) => { - super::sources_routes::source_in_kb(&state, kb_id, source_id).await?; - Some( - utopia_store::documents::ids_in_source( - &state.pool, - kb_id, - source_id, - req.external_id_prefixes.as_deref(), - as_of, - ) - .await?, - ) - } - None => None, - }; let opts = retrieval::SearchOpts { top_k, offset: req.offset, as_of, - documents, + filter: source_filter(&state, kb_id, &req).await?, }; let page = retrieval::hybrid_scored(&state, kb_id, kb.workspace_id, q, opts, None).await?; if req.record { diff --git a/crates/utopia-server/src/api/search_routes_tests.rs b/crates/utopia-server/src/api/search_routes_tests.rs index fd9a20ff6..3c3231061 100644 --- a/crates/utopia-server/src/api/search_routes_tests.rs +++ b/crates/utopia-server/src/api/search_routes_tests.rs @@ -20,7 +20,6 @@ struct Fixture { /// BM25 排第一的那块 best: Uuid, token: String, - search: Arc, _dir: tempfile::TempDir, } @@ -85,7 +84,7 @@ impl Fixture { let token = utopia_store::tokens::issue(&pool, user, "ci", "read", None, None) .await? .1; - let state = AppState::new(pool.clone(), &cfg, search.clone(), "test-only".into()); + let state = AppState::new(pool.clone(), &cfg, search, "test-only".into()); Ok(Some(Self { app: super::router(state, &cfg), pool, @@ -95,38 +94,11 @@ impl Fixture { doc, best: chunks[0].0.parse()?, token, - search, _dir: dir, })) } /// 本库 statements 来源里再推一篇一块的文档，进全文索引；交回那一块的 id - async fn push(&self, external_id: &str, text: &str) -> anyhow::Result { - let (doc, chunk) = (Uuid::now_v7(), Uuid::now_v7()); - sqlx::query( - "INSERT INTO documents(id,kb_id,source_id,external_key,filename,sha256) - VALUES ($1,$2,$3,$4,$5,$1::text)", - ) - .bind(doc) - .bind(self.kb) - .bind(self.source) - .bind(format!("statements:{external_id}")) - .bind(external_id) - .execute(&self.pool) - .await?; - sqlx::query("INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES($1,$2,$3,0,$4)") - .bind(chunk) - .bind(self.kb) - .bind(doc) - .bind(text) - .execute(&self.pool) - .await?; - let chunks = [(chunk.to_string(), text.to_string())]; - self.search - .reindex_document(&self.kb.to_string(), &doc.to_string(), &chunks)?; - Ok(chunk) - } - async fn search(&self, body: Value) -> anyhow::Result<(StatusCode, Value)> { let request = Request::post(format!("/api/v1/kbs/{}/search", self.kb)) .header("authorization", format!("Bearer {}", self.token)) @@ -221,74 +193,6 @@ async fn a_page_past_the_hundredth_hit_is_refused() -> anyhow::Result<()> { f.cleanup().await } -fn ids(body: &Value) -> Vec { - body["results"] - .as_array() - .into_iter() - .flatten() - .map(|hit| hit["id"].clone()) - .collect() -} - -/// 三十篇代码条目都比那条规格陈述更像 apple；每路只召回 24 条，召回完再筛就什么也 -/// 剩不下。过滤在召回之内，那条陈述才回得来 -#[tokio::test] -async fn a_prefix_finds_the_statement_that_thirty_better_matches_crowd_out() -> anyhow::Result<()> { - let Some(f) = Fixture::new().await? else { - return Ok(()); - }; - for n in 0..30 { - f.push(&format!("code:src/a_{n}.rs"), "apple apple apple apple") - .await?; - } - let spec = f - .push( - "doc:specs/x_y.md", - "one apple among many quiet words about the spring planting schedule", - ) - .await?; - - let (status, body) = f.search(json!({ "q": "apple", "top_k": 10 })).await?; - assert_eq!(status, StatusCode::OK, "{body}"); - assert!(!ids(&body).contains(&json!(spec)), "{body}"); - - let (status, body) = f - .search(json!({ - "q": "apple", - "top_k": 10, - "source_id": f.source, - "external_id_prefixes": ["doc:"], - })) - .await?; - assert_eq!(status, StatusCode::OK, "{body}"); - assert_eq!(ids(&body), vec![json!(spec)]); - - // `_` 与 `%` 是路径里的字，不是通配符：没有哪篇以 `doc:specs/x%` 开头 - let (status, body) = f - .search(json!({ - "q": "apple", - "source_id": f.source, - "external_id_prefixes": ["doc:specs/x%"], - })) - .await?; - assert_eq!(status, StatusCode::OK, "{body}"); - assert_eq!(ids(&body), Vec::::new()); - f.cleanup().await -} - -#[tokio::test] -async fn prefixes_without_a_source_are_refused() -> anyhow::Result<()> { - let Some(f) = Fixture::new().await? else { - return Ok(()); - }; - let (status, body) = f - .search(json!({ "q": "apple", "external_id_prefixes": ["doc:"] })) - .await?; - assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{body}"); - assert_eq!(body["code"], "prefixes_need_source"); - f.cleanup().await -} - #[tokio::test] async fn a_source_of_another_base_is_not_found() -> anyhow::Result<()> { let Some(f) = Fixture::new().await? else { diff --git a/crates/utopia-server/src/api/settings_routes.rs b/crates/utopia-server/src/api/settings_routes.rs index ff265d9ae..c1ca3834f 100644 --- a/crates/utopia-server/src/api/settings_routes.rs +++ b/crates/utopia-server/src/api/settings_routes.rs @@ -1,8 +1,8 @@ -use axum::extract::{Path, State}; +use axum::extract::{Path, Query, State}; use axum::Json; use serde::Deserialize; use serde_json::json; -use utopia_core::models::Role; +use utopia_core::models::{LlmSettings, Role}; use utopia_llm::ChatMessage; use uuid::Uuid; @@ -322,59 +322,86 @@ fn check(r: &anyhow::Result<()>) -> serde_json::Value { } } +/// 设置页的四张模型卡，各有一个「测试」 +#[derive(Deserialize, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum ModelCard { + Chat, + Embed, + Ocr, + Transcribe, +} + +#[derive(Deserialize)] +pub struct TestQuery { + /// 只测这一张卡。缺席 = 四张都测 + pub card: Option, +} + /// 连通性测试：对话按任务的三种请求形状各试一次（见 [`chat_checks`]）；embedding 试算一条并 -/// 返回维度；版面识别服务问一声健康。 +/// 返回维度；版面识别服务问一声健康。带 `?card=`（[`ModelCard`]）只测那一张卡，回来的也只有 +/// 那一项；不带就四张都测（#1424）。 pub async fn test( State(state): State, AuthUser(user): AuthUser, Path(workspace_id): Path, + Query(q): Query, ) -> ApiResult> { utopia_store::workspaces::require_role(&state.pool, user.id, workspace_id, Role::Admin).await?; - let Some(s) = utopia_store::settings::get(&state.pool, workspace_id).await? else { - return Ok(Json( - json!({ "chat": { "ok": false, "error": "Not configured" }, - "embed": { "ok": false, "error": "Not configured" }, - "ocr": { "ok": false, "error": "Not configured" }, - "transcribe": { "ok": false, "error": "Not configured" } }), - )); - }; + let s = utopia_store::settings::get(&state.pool, workspace_id).await?; + Ok(Json(probe(s.as_ref(), q.card).await)) +} - let chat_result = match llm_util::chat_probe_client(&s) { - None => json!({ "ok": false, "error": "Not configured" }), - Some(client) => chat_checks(&client).await, - }; +/// 只测点了的那张卡（#1424）。从前四张卡共用一次测试，按任意一张四套一起测；自 #1363 起 +/// 对话一测要发三问，测嵌入、读扫描件的卡也跟着白花这几问的 token。回来的只有测了的那几项 +async fn probe(s: Option<&LlmSettings>, only: Option) -> serde_json::Value { + let wants = |card| only.is_none_or(|c| c == card); + let not_configured = || json!({ "ok": false, "error": "Not configured" }); + let mut out = serde_json::Map::new(); - let embed_result = match llm_util::embed_client(&s) { - None => json!({ "ok": false, "error": "Not configured" }), - Some(client) => match client.embed(&["connectivity test".to_string()]).await { - Ok(v) if !v.is_empty() => json!({ "ok": true, "dim": v[0].len() }), - Ok(_) => json!({ "ok": false, "error": "Empty response" }), - Err(e) => json!({ "ok": false, "error": e.to_string() }), - }, - }; + if wants(ModelCard::Chat) { + let chat_result = match s.and_then(llm_util::chat_probe_client) { + None => not_configured(), + Some(client) => chat_checks(&client).await, + }; + out.insert("chat".into(), chat_result); + } - let ocr_result = match crate::readers::Ocr::from_settings(&s) { - None => json!({ "ok": false, "error": "Not configured" }), - Some(ocr) => match ocr.health().await { - Ok(h) => json!({ "ok": true, "version": h["version"] }), - Err(e) => json!({ "ok": false, "error": format!("{e:#}") }), - }, - }; + if wants(ModelCard::Embed) { + let embed_result = match s.and_then(llm_util::embed_client) { + None => not_configured(), + Some(client) => match client.embed(&["connectivity test".to_string()]).await { + Ok(v) if !v.is_empty() => json!({ "ok": true, "dim": v[0].len() }), + Ok(_) => json!({ "ok": false, "error": "Empty response" }), + Err(e) => json!({ "ok": false, "error": e.to_string() }), + }, + }; + out.insert("embed".into(), embed_result); + } - let transcribe_result = match crate::readers::Transcriber::from_settings(&s) { - None => json!({ "ok": false, "error": "Not configured" }), - Some(t) => match t.check().await { - Ok(()) => json!({ "ok": true }), - Err(e) => json!({ "ok": false, "error": format!("{e:#}") }), - }, - }; + if wants(ModelCard::Ocr) { + let ocr_result = match s.and_then(crate::readers::Ocr::from_settings) { + None => not_configured(), + Some(ocr) => match ocr.health().await { + Ok(h) => json!({ "ok": true, "version": h["version"] }), + Err(e) => json!({ "ok": false, "error": format!("{e:#}") }), + }, + }; + out.insert("ocr".into(), ocr_result); + } - Ok(Json(json!({ - "chat": chat_result, - "embed": embed_result, - "ocr": ocr_result, - "transcribe": transcribe_result, - }))) + if wants(ModelCard::Transcribe) { + let transcribe_result = match s.and_then(crate::readers::Transcriber::from_settings) { + None => not_configured(), + Some(t) => match t.check().await { + Ok(()) => json!({ "ok": true }), + Err(e) => json!({ "ok": false, "error": format!("{e:#}") }), + }, + }; + out.insert("transcribe".into(), transcribe_result); + } + + out.into() } #[cfg(test)] @@ -514,4 +541,42 @@ mod tests { "the streamed and tool shapes are not sent when the plain one fails" ); } + + /// #1424：按嵌入卡的「测试」，对话模型一问都收不到；不指定卡的旧调用照旧四项都回 + #[tokio::test] + async fn testing_one_card_sends_nothing_to_the_others() { + let chat = endpoint(false).await; + let s = utopia_core::models::LlmSettings { + workspace_id: uuid::Uuid::nil(), + chat_base_url: Some(chat.uri()), + chat_api_key: None, + chat_model: Some("scripted".into()), + embed_base_url: None, + embed_api_key: None, + embed_model: None, + embed_dim: None, + updated_at: chrono::Utc::now(), + ocr_base_url: None, + ocr_api_key: None, + ocr_backend: None, + ocr_provider: "mineru".into(), + ocr_model: None, + transcribe_base_url: None, + transcribe_api_key: None, + transcribe_model: None, + chat_reasoning_effort: None, + }; + + let out = super::probe(Some(&s), Some(super::ModelCard::Embed)).await; + assert_eq!( + out, + serde_json::json!({ "embed": { "ok": false, "error": "Not configured" } }) + ); + assert!(bodies(&chat).await.is_empty()); + + let all = super::probe(Some(&s), None).await; + let cards: Vec<_> = all.as_object().unwrap().keys().collect(); + assert_eq!(cards, ["chat", "embed", "ocr", "transcribe"], "{all}"); + assert_eq!(bodies(&chat).await.len(), 3); + } } diff --git a/crates/utopia-server/src/api/sources_checkpoint_tests.rs b/crates/utopia-server/src/api/sources_checkpoint_tests.rs new file mode 100644 index 000000000..b68d07222 --- /dev/null +++ b/crates/utopia-server/src/api/sources_checkpoint_tests.rs @@ -0,0 +1,187 @@ +//! 来源的增量游标在 API 里看得见（ADR 0072）。 +//! +//! **说的是「读到哪」，不是「这一轮什么时候跑」。** 一轮读不完是常事——同一个更新 +//! 时刻上可能挂着一千条工单——游标记的是「消费到了哪个更新时刻，且那一刻的整组都 +//! 读完了」。没有它，界面上只看得见「同步成功」，看不见这一轮停在了哪里；要排 +//! 「这一轮到底覆盖了多少」，只能去翻日志。 +//! +//! 用不到的来源那一格是 NULL，而不是 0 也不是别的时刻：没有游标与「游标在零点」 +//! 是两回事，前者是「还没读到哪」，后者是「读到 1970 年」。 +//! +//! 连库的测试，没有 `UTOPIA_DATABASE_URL` 就跳过（同 documents_routes_tests）。 + +use axum::body::{to_bytes, Body}; +use axum::http::{Request, StatusCode}; +use serde_json::{json, Value}; +use std::sync::Arc; +use tower::ServiceExt; +use uuid::Uuid; + +/// 游标落在这一刻：秒精度，正是工单更新时刻的粒度。 +/// +/// **响应里写的是 `Z` 结尾的那一种**（chrono 对 `DateTime` 的写法）， +/// 不是 `+00:00`——断言照着实际发出去的形式写，变了就是变了。 +const CHECKPOINT: &str = "2026-01-02T00:00:00Z"; + +fn checkpoint() -> chrono::DateTime { + chrono::DateTime::parse_from_rfc3339(CHECKPOINT) + .unwrap() + .with_timezone(&chrono::Utc) +} + +struct Fixture { + app: axum::Router, + kb: Uuid, + /// 工单来源：有游标 + ticket: Uuid, + /// 订阅来源：没有游标 + feed: Uuid, + token: String, + _dir: tempfile::TempDir, +} + +impl Fixture { + async fn new() -> anyhow::Result> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(None); + }; + let pool = sqlx::PgPool::connect(&url).await?; + utopia_store::db::migrate(&pool).await?; + let (org, ws, kb, user, ticket, feed) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + // Only locally generated UUIDs are interpolated into fixture SQL. + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','checkpoint-test'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','checkpoint-test'); + INSERT INTO users(id,org_id,email,display_name,password_hash) + VALUES ('{user}','{org}','{user}@checkpoint.test','checkpoint-test','unused'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','observations'); + INSERT INTO kb_members(kb_id,user_id,role) VALUES ('{kb}','{user}','editor'); + INSERT INTO sources(id,kb_id,kind,name,config) + VALUES ('{ticket}','{kb}','github_issues','tickets','{{\"repo\":\"o/r\"}}'); + INSERT INTO sources(id,kb_id,kind,name,config) + VALUES ('{feed}','{kb}','rss','feed','{{\"feed_url\":\"https://example.test/f\"}}');" + )) + .execute(&pool) + .await?; + // 工单来源读到了那一刻；订阅来源不按时刻分组，没有这一格 + sqlx::query("UPDATE sources SET sync_checkpoint = $2 WHERE id = $1") + .bind(ticket) + .bind(checkpoint()) + .execute(&pool) + .await?; + let dir = tempfile::tempdir()?; + let cfg = utopia_core::config::AppConfig { + data_dir: dir.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open( + &dir.path().join("search"), + )?); + let state = crate::state::AppState::new(pool.clone(), &cfg, search, "test-only".into()); + let token = crate::auth::issue_token(&state, user)?; + let app = super::super::router(state.clone(), &cfg); + Ok(Some(Self { + app, + kb, + ticket, + feed, + token, + _dir: dir, + })) + } + + async fn call( + &self, + method: &str, + uri: &str, + body: Option, + ) -> anyhow::Result<(StatusCode, Value)> { + let request = Request::builder() + .method(method) + .uri(uri) + .header("Authorization", format!("Bearer {}", self.token)); + let request = match body { + Some(body) => request + .header("Content-Type", "application/json") + .body(Body::from(body.to_string()))?, + None => request.body(Body::empty())?, + }; + let response = self.app.clone().oneshot(request).await?; + let status = response.status(); + let bytes = to_bytes(response.into_body(), 1 << 20).await?; + Ok(( + status, + serde_json::from_slice(&bytes).unwrap_or(Value::Null), + )) + } +} + +/// **列表里有这一格。** 工单来源带着游标，用不到游标的来源是 NULL。 +#[tokio::test] +async fn the_source_list_carries_the_checkpoint() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let (status, body) = fx + .call("GET", &format!("/api/v1/kbs/{}/sources", fx.kb), None) + .await?; + assert_eq!(status, StatusCode::OK, "{body}"); + let sources = body["sources"].as_array().expect("sources[]"); + let ticket = sources + .iter() + .find(|s| s["id"] == fx.ticket.to_string()) + .expect("工单来源在列表里"); + let feed = sources + .iter() + .find(|s| s["id"] == fx.feed.to_string()) + .expect("订阅来源在列表里"); + + // **时刻本身要给出来**，不是"有几个"：那一格说的是读到哪 + assert_eq!( + ticket["sync_checkpoint"], + json!(CHECKPOINT), + "工单来源的游标要原样给出" + ); + // 用不到的来源是 NULL——没有游标与"游标在零点"是两回事 + assert_eq!( + feed["sync_checkpoint"], + Value::Null, + "用不到游标的来源不该编一个时刻出来" + ); + // 游标与"这一轮什么时候跑"是两件事，两个都在 + assert!( + ticket["last_sync_at"].is_null(), + "还没同步过就没有这一轮的时刻" + ); + Ok(()) +} + +/// **改一个来源，回来的那一格也带着游标。** 列表与详情说的是同一件事， +/// 不能一处有一处没有。 +#[tokio::test] +async fn a_source_update_carries_the_checkpoint_too() -> anyhow::Result<()> { + let Some(fx) = Fixture::new().await? else { + return Ok(()); + }; + let (status, body) = fx + .call( + "PATCH", + &format!("/api/v1/kbs/{}/sources/{}", fx.kb, fx.ticket), + Some(json!({ "name": "renamed" })), + ) + .await?; + assert_eq!(status, StatusCode::OK, "{body}"); + assert_eq!( + body["source"]["sync_checkpoint"], + json!(CHECKPOINT), + "更新回来的那一格也要带游标" + ); + Ok(()) +} diff --git a/crates/utopia-server/src/api/sources_routes.rs b/crates/utopia-server/src/api/sources_routes.rs index dfd0f6ae2..c19e36754 100644 --- a/crates/utopia-server/src/api/sources_routes.rs +++ b/crates/utopia-server/src/api/sources_routes.rs @@ -12,6 +12,7 @@ use uuid::Uuid; use super::graph_routes::require_kb; use crate::auth::{AuthUser, SessionOrToken}; use crate::error::ApiResult; +use crate::pushed_keys::{is_push_kind, pushed_key}; use crate::state::AppState; /// 生成 api 来源的推送密钥。 @@ -188,19 +189,6 @@ pub async fn create( )) } -/// 有推送密钥的来源：`api` 推文档，`statements` 推陈述（0054）。两种的密钥同一套 -/// 生成、查看、轮换 -pub(crate) fn is_push_kind(kind: &str) -> bool { - matches!(kind, "api" | "statements") -} - -/// The `external_key` a push source files a pushed identity under (`api:…`, `statements:…`). -/// The push handlers write it and the read by `external_id` (#1276) looks it up, so the two -/// cannot drift apart. -pub(crate) fn pushed_key(source: &utopia_core::models::Source, identity: &str) -> String { - format!("{}:{identity}", source.kind) -} - /// 查看推送密钥（Editor；列表响应从不携带，查看走这里）。 /// /// 令牌要 write：密钥能往库里推东西，读令牌拿到它就等于绕过了自己的 scope @@ -444,6 +432,10 @@ pub async fn close_unconfirmed( #[path = "sources_close_unconfirmed_tests.rs"] mod close_unconfirmed_tests; +#[cfg(test)] +#[path = "sources_checkpoint_tests.rs"] +mod checkpoint_tests; + pub async fn delete( State(state): State, caller: SessionOrToken, @@ -1084,6 +1076,7 @@ mod tests { last_sync_status: "never".into(), last_sync_error: None, last_sync_added: 0, + sync_checkpoint: None, ingest_token: Some("utp_x".into()), created_at: chrono::Utc::now(), }; diff --git a/crates/utopia-server/src/api/token_read_tests.rs b/crates/utopia-server/src/api/token_read_tests.rs index 339e7654f..632405ae9 100644 --- a/crates/utopia-server/src/api/token_read_tests.rs +++ b/crates/utopia-server/src/api/token_read_tests.rs @@ -20,6 +20,7 @@ struct Fixture { entity: Uuid, fact: Uuid, doc: Uuid, + search: Arc, _dir: tempfile::TempDir, } @@ -63,7 +64,7 @@ impl Fixture { let search = Arc::new(utopia_search::SearchIndex::open( &dir.path().join("search"), )?); - let state = AppState::new(pool.clone(), &cfg, search, "test-only".into()); + let state = AppState::new(pool.clone(), &cfg, search.clone(), "test-only".into()); let app = super::router(state, &cfg); Ok(Some(Self { pool, @@ -76,6 +77,7 @@ impl Fixture { entity, fact, doc, + search, _dir: dir, })) } @@ -133,6 +135,22 @@ impl Fixture { Ok(self.app.clone().oneshot(request).await?.status()) } + async fn search( + &self, + token: &str, + body: serde_json::Value, + ) -> anyhow::Result { + let request = Request::post(format!("/api/v1/kbs/{}/search", self.kb)) + .header("authorization", format!("Bearer {token}")) + .header("content-type", "application/json") + .body(Body::from(body.to_string()))?; + let response = self.app.clone().oneshot(request).await?; + let status = response.status().as_u16(); + let bytes = axum::body::to_bytes(response.into_body(), usize::MAX).await?; + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap_or_default(); + Ok(json!({ "status": status, "body": body })) + } + async fn cleanup(self) -> anyhow::Result<()> { sqlx::query("DELETE FROM organizations WHERE id = $1") .bind(self.org) @@ -229,3 +247,158 @@ async fn a_search_past_depth_100_or_with_an_unknown_filters_field_gets_422() -> assert_eq!(statuses, [StatusCode::UNPROCESSABLE_ENTITY; 2]); Ok(()) } + +/// 一个 api 来源推了 `identities` 里的每一篇，都讲 harvest；另一个库有个来源 +async fn seed_pushed_and_foreign_sources( + f: &Fixture, + identities: &[&str], +) -> anyhow::Result<(Uuid, Uuid, Uuid)> { + let (pushed, custom, foreign) = (Uuid::now_v7(), Uuid::now_v7(), Uuid::now_v7()); + let (kb, other_kb) = (f.kb, f.other_kb); + sqlx::raw_sql(&format!( + "INSERT INTO sources(id,kb_id,kind,name) VALUES + ('{pushed}','{kb}','api','ci'), ('{custom}','{kb}','custom','poll'), + ('{foreign}','{other_kb}','api','ci');" + )) + .execute(&f.pool) + .await?; + for identity in identities { + let (doc, chunk) = (Uuid::now_v7(), Uuid::now_v7()); + sqlx::query( + "INSERT INTO documents(id,kb_id,source_id,external_key,filename,sha256) + VALUES($1,$2,$3,$4,$5,$5)", + ) + .bind(doc) + .bind(kb) + .bind(pushed) + .bind(format!("api:{identity}")) + .bind(identity) + .execute(&f.pool) + .await?; + sqlx::query( + "INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES($1,$2,$3,0,'harvest notes')", + ) + .bind(chunk) + .bind(kb) + .bind(doc) + .execute(&f.pool) + .await?; + f.search.reindex_document( + &kb.to_string(), + &doc.to_string(), + &[(chunk.to_string(), "harvest notes".into())], + )?; + } + Ok((pushed, custom, foreign)) +} + +fn found_external_ids(found: &serde_json::Value) -> serde_json::Value { + let mut ids: Vec<_> = found["body"]["results"] + .as_array() + .into_iter() + .flatten() + .map(|hit| hit["external_id"].clone()) + .collect(); + ids.sort_by_key(|id| id.to_string()); + json!({ "status": found["status"], "external_ids": ids }) +} + +#[tokio::test] +async fn a_search_by_source_and_external_id_prefix_docs_finds_only_docs_a_md() -> anyhow::Result<()> +{ + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let (pushed, _, _) = + seed_pushed_and_foreign_sources(&f, &["docs/a.md", "code/b.rs"]).await?; + let token = f.read_token(f.kb).await; + let all = f + .search(&token, json!({ "q": "harvest", "source_id": pushed })) + .await?; + let docs = f + .search( + &token, + json!({ "q": "harvest", "source_id": pushed, "external_id_prefix": "docs/" }), + ) + .await?; + anyhow::Ok((all, docs)) + } + .await; + f.cleanup().await?; + let (all, docs) = result?; + assert_eq!( + found_external_ids(&all), + json!({ "status": 200, "external_ids": ["code/b.rs", "docs/a.md"] }) + ); + assert_eq!( + found_external_ids(&docs), + json!({ "status": 200, "external_ids": ["docs/a.md"] }) + ); + Ok(()) +} + +#[tokio::test] +async fn a_search_by_another_bases_source_gets_404_and_a_prefix_without_a_push_source_gets_422( +) -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let (_, custom, foreign) = seed_pushed_and_foreign_sources(&f, &[]).await?; + let token = f.read_token(f.kb).await; + let mut codes = Vec::new(); + for body in [ + json!({ "q": "harvest", "source_id": foreign }), + json!({ "q": "harvest", "external_id_prefix": "docs/" }), + json!({ "q": "harvest", "source_id": custom, "external_id_prefix": "docs/" }), + ] { + let found = f.search(&token, body).await?; + codes.push(json!([found["status"], found["body"]["code"]])); + } + anyhow::Ok(codes) + } + .await; + f.cleanup().await?; + assert_eq!( + result?, + vec![ + json!([404, null]), + json!([422, "prefix_needs_push_source"]), + json!([422, "prefix_needs_push_source"]), + ] + ); + Ok(()) +} + +#[tokio::test] +async fn an_external_id_prefix_docs_underscore_finds_docs_a_md_but_not_docsxa_md( +) -> anyhow::Result<()> { + let Some(f) = Fixture::new().await? else { + return Ok(()); + }; + let result = async { + let (pushed, _, _) = seed_pushed_and_foreign_sources( + &f, + &["docs_a.md", "docsXa.md", "50%off.md", "50 percent off.md"], + ) + .await?; + let token = f.read_token(f.kb).await; + let mut found = Vec::new(); + for prefix in ["docs_", "50%"] { + let body = json!({ "q": "harvest", "source_id": pushed, "external_id_prefix": prefix }); + found.push(found_external_ids(&f.search(&token, body).await?)); + } + anyhow::Ok(found) + } + .await; + f.cleanup().await?; + assert_eq!( + result?, + [ + json!({ "status": 200, "external_ids": ["docs_a.md"] }), + json!({ "status": 200, "external_ids": ["50%off.md"] }), + ] + ); + Ok(()) +} diff --git a/crates/utopia-server/src/github_issues.rs b/crates/utopia-server/src/github_issues.rs index 2dee2a118..0d9aa0118 100644 --- a/crates/utopia-server/src/github_issues.rs +++ b/crates/utopia-server/src/github_issues.rs @@ -39,13 +39,16 @@ use chrono::{DateTime, Utc}; use serde::Deserialize; +use std::collections::BTreeMap; +use std::time::Duration; use crate::ticket::stamp; -/// 一次同步最多取多少页（每页 100）。GitHub 的分页没有天然终点， -/// 一个活跃仓库能翻很久；这里沿用单端点的分页预算。 -const MAX_PAGES: u32 = 10; +/// 列表预算可在完整时间组边界延长；单张工单的详情不受此预算限制。 +const MAX_LISTING_PAGES: u32 = 10; const PER_PAGE: u32 = 100; +const MAX_RATE_LIMIT_RETRIES: u32 = 3; +const MAX_RATE_LIMIT_WAIT: Duration = Duration::from_secs(30); #[derive(Debug, Deserialize)] pub struct Issue { @@ -190,16 +193,39 @@ pub(crate) async fn get_checked( auth: Option<&str>, accept: Option<&str>, ) -> anyhow::Result { - let mut req = http.get(url); - if let Some(a) = auth { - req = req.header(reqwest::header::AUTHORIZATION, a); - } - if let Some(a) = accept { - req = req.header(reqwest::header::ACCEPT, a); - } - let resp = req.send().await?; - let status = resp.status(); - if !status.is_success() { + let mut retries = 0; + loop { + let mut req = http.get(url.clone()); + if let Some(a) = auth { + req = req.header(reqwest::header::AUTHORIZATION, a); + } + if let Some(a) = accept { + req = req.header(reqwest::header::ACCEPT, a); + } + let resp = req.send().await?; + let status = resp.status(); + if status.is_success() { + return Ok(resp); + } + if matches!( + status, + reqwest::StatusCode::FORBIDDEN | reqwest::StatusCode::TOO_MANY_REQUESTS + ) && retries < MAX_RATE_LIMIT_RETRIES + { + if let Some(delay) = rate_limit_delay(resp.headers(), Utc::now()) { + // 等待留在当前请求内，否则超大时间组会在每次重试时重新耗尽前缀的配额。 + let delay = delay.max(Duration::from_secs(1 << retries)); + retries += 1; + tracing::info!( + retries, + wait_seconds = delay.as_secs(), + "GitHub rate limited; waiting before retry" + ); + drop(resp); + tokio::time::sleep(delay).await; + continue; + } + } // 限流说清楚是限流：未认证时每小时 60 次，一个中等仓库一次同步就能吃光。 // 报成通用 HTTP 错会让人去查网络，而正确的动作是配一个令牌 let remaining = resp @@ -214,10 +240,163 @@ pub(crate) async fn get_checked( } anyhow::bail!("HTTP {status} from GitHub"); } - Ok(resp) } -/// 分页取一个端点，直到空页或触顶。 +fn rate_limit_delay(headers: &reqwest::header::HeaderMap, now: DateTime) -> Option { + let retry_after = headers + .get(reqwest::header::RETRY_AFTER) + .and_then(|value| value.to_str().ok()) + .and_then(|value| { + value + .trim() + .parse::() + .ok() + .map(Duration::from_secs) + .or_else(|| { + DateTime::parse_from_rfc2822(value) + .ok() + .map(|at| at.signed_duration_since(now).to_std().unwrap_or_default()) + }) + }); + let delay = retry_after.or_else(|| { + if headers.get("x-ratelimit-remaining")?.to_str().ok()? != "0" { + return None; + } + let reset = headers + .get("x-ratelimit-reset")? + .to_str() + .ok()? + .parse::() + .ok()?; + let reset = DateTime::from_timestamp(reset, 0)?; + Some( + reset + .signed_duration_since(now) + .to_std() + .unwrap_or_default(), + ) + })?; + // 长等待不能占住 worker；拒绝本次重试，而不是早于服务端允许的时间重发。 + (delay <= MAX_RATE_LIMIT_WAIT).then_some(delay) +} + +pub struct IssueWindow { + pub issues: Vec, + pub checkpoint: Option>, +} + +/// offset 分页不是快照；复读同一前缀，变化时让调用者保留旧 checkpoint 后重试。 +pub async fn fetch_window( + http: &reqwest::Client, + base: &str, + since: Option>, + item_limit: usize, + include_prs: bool, + auth: Option<&str>, +) -> anyhow::Result { + let (window, positions) = + fetch_window_pass(http, base, since, item_limit, include_prs, auth, None).await?; + if let Some(checkpoint) = window + .checkpoint + .filter(|checkpoint| Some(*checkpoint) != since) + { + let (verified, verified_positions) = fetch_window_pass( + http, + base, + since, + item_limit, + include_prs, + auth, + Some(checkpoint), + ) + .await?; + anyhow::ensure!( + verified.checkpoint == Some(checkpoint) && positions == verified_positions, + "GitHub issues changed during pagination; retry the previous checkpoint" + ); + } + Ok(window) +} + +async fn fetch_window_pass( + http: &reqwest::Client, + base: &str, + since: Option>, + item_limit: usize, + include_prs: bool, + auth: Option<&str>, + through: Option>, +) -> anyhow::Result<(IssueWindow, BTreeMap>)> { + let mut query = vec![ + ("state", "all".to_owned()), + ("sort", "updated".to_owned()), + ("direction", "asc".to_owned()), + ]; + if let Some(since) = since { + query.push(("since", since.to_rfc3339())); + } + let mut window = IssueWindow { + issues: Vec::new(), + checkpoint: since, + }; + let mut positions = BTreeMap::new(); + let mut previous_time = None; + let mut stop_after = through; + let mut listing_pages = 0; + let mut page = 1; + loop { + let mut url = reqwest::Url::parse(base)?; + { + let mut parameters = url.query_pairs_mut(); + for (key, value) in &query { + parameters.append_pair(key, value); + } + parameters.append_pair("per_page", &PER_PAGE.to_string()); + parameters.append_pair("page", &page.to_string()); + } + let batch: Vec = get_checked(http, url, auth, None).await?.json().await?; + let count = batch.len(); + let mut page_counted = false; + for issue in batch { + let updated = issue.updated_at; + anyhow::ensure!( + previous_time.is_none_or(|previous| updated >= previous), + "GitHub issues are not ordered by update time; retry the previous checkpoint" + ); + previous_time = Some(updated); + if stop_after.is_some_and(|boundary| updated > boundary) { + return Ok((window, positions)); + } + // 已消费条目和排除的 PR 也能移动页边界，必须参加复读核对。 + anyhow::ensure!( + positions.insert(issue.number, updated).is_none(), + "GitHub issue repeated during pagination; retry the previous checkpoint" + ); + if since.is_some_and(|checkpoint| updated <= checkpoint) { + continue; + } + if !page_counted { + listing_pages += 1; + page_counted = true; + } + window.checkpoint = Some(updated); + if include_prs || issue.pull_request.is_none() { + window.issues.push(issue); + } + if through.is_none() + && (window.issues.len() >= item_limit || listing_pages >= MAX_LISTING_PAGES) + { + stop_after = Some(updated); + } + } + if count < PER_PAGE as usize { + return Ok((window, positions)); + } + page += 1; + } +} + +/// 详情必须读到末页，不能用截断的评论或事件覆盖已有完整正文。 pub async fn fetch_all Deserialize<'de>>( http: &reqwest::Client, base: &str, @@ -225,7 +404,8 @@ pub async fn fetch_all Deserialize<'de>>( auth: Option<&str>, ) -> anyhow::Result> { let mut out = Vec::new(); - for page in 1..=MAX_PAGES { + let mut page = 1; + loop { // 自己拼 query：这套 reqwest 特性组合里没有 RequestBuilder::query， // 而 sync_custom 本来也是这么拼的 let mut url = reqwest::Url::parse(base)?; @@ -244,6 +424,7 @@ pub async fn fetch_all Deserialize<'de>>( if n < PER_PAGE as usize { break; } + page += 1; } Ok(out) } @@ -251,6 +432,355 @@ pub async fn fetch_all Deserialize<'de>>( #[cfg(test)] mod tests { use super::*; + use serde_json::{json, Value}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate}; + + fn listing_row(number: i64, updated: &str) -> Value { + json!({ + "number": number, "title": format!("Issue {number}"), "state": "open", + "created_at": "2026-01-01T00:00:00Z", "updated_at": updated, + }) + } + + fn requested_page(request: &Request) -> usize { + request + .url + .query_pairs() + .find(|(key, _)| key == "page") + .unwrap() + .1 + .parse() + .unwrap() + } + + #[test] + fn rate_limit_headers_supply_a_bounded_wait_without_guessing() { + let now: DateTime = "2026-01-02T00:00:00Z".parse().unwrap(); + let mut headers = reqwest::header::HeaderMap::new(); + for (value, expected) in [ + ("2", Some(2)), + ("0", Some(0)), + ("30", Some(30)), + ("31", None), + ("-1", None), + ("broken", None), + ("86401", None), + ] { + headers.insert(reqwest::header::RETRY_AFTER, value.parse().unwrap()); + assert_eq!( + rate_limit_delay(&headers, now), + expected.map(Duration::from_secs) + ); + } + for (seconds, expected) in [(3, Some(3)), (31, None)] { + headers.insert( + reqwest::header::RETRY_AFTER, + (now + chrono::Duration::seconds(seconds)) + .to_rfc2822() + .parse() + .unwrap(), + ); + assert_eq!( + rate_limit_delay(&headers, now), + expected.map(Duration::from_secs) + ); + } + headers.remove(reqwest::header::RETRY_AFTER); + headers.insert("x-ratelimit-remaining", "0".parse().unwrap()); + headers.insert( + "x-ratelimit-reset", + (now.timestamp() + 30).to_string().parse().unwrap(), + ); + assert_eq!( + rate_limit_delay(&headers, now), + Some(Duration::from_secs(30)) + ); + headers.insert("x-ratelimit-remaining", "1".parse().unwrap()); + assert_eq!(rate_limit_delay(&headers, now), None); + headers.insert("x-ratelimit-remaining", "0".parse().unwrap()); + headers.insert( + "x-ratelimit-reset", + (now.timestamp() + 31).to_string().parse().unwrap(), + ); + assert_eq!(rate_limit_delay(&headers, now), None); + } + + #[tokio::test] + async fn a_rate_limited_request_waits_and_preserves_its_url_and_headers() { + let server = MockServer::start().await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + for status in [403, 429] { + server.reset().await; + let attempts = AtomicUsize::new(0); + Mock::given(method("GET")) + .respond_with(move |_: &Request| { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + ResponseTemplate::new(status).insert_header("Retry-After", "0") + } else { + ResponseTemplate::new(200) + } + }) + .mount(&server) + .await; + let url = reqwest::Url::parse(&format!("{}/resource?page=2", server.uri())).unwrap(); + let started = std::time::Instant::now(); + get_checked( + &http, + url.clone(), + Some("Bearer fixture"), + Some("application/vnd.github.raw+json"), + ) + .await + .unwrap(); + assert!(started.elapsed() >= Duration::from_secs(1)); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), 2); + for request in &requests { + // wiremock 为 origin-form URI 补 localhost；实际目标主机保留在 Host 头中。 + assert_eq!(request.url.path(), url.path()); + assert_eq!(request.url.query(), url.query()); + assert_eq!( + request.headers.get("host").unwrap().to_str().unwrap(), + server.address().to_string() + ); + assert_eq!( + request.headers.get("authorization").unwrap(), + "Bearer fixture" + ); + assert_eq!( + request.headers.get("accept").unwrap(), + "application/vnd.github.raw+json" + ); + } + } + } + + #[tokio::test] + async fn persistent_rate_limits_stop_after_a_bounded_number_of_requests() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(429).insert_header("Retry-After", "0")) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + assert!( + get_checked(&http, server.uri().parse().unwrap(), None, None) + .await + .is_err() + ); + assert_eq!(server.received_requests().await.unwrap().len(), 4); + } + + #[tokio::test] + async fn long_rate_limit_waits_fail_without_sleeping_or_retrying() { + let server = MockServer::start().await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + for status in [403, 429] { + let responses = [ + ResponseTemplate::new(status).insert_header("Retry-After", "31"), + ResponseTemplate::new(status).insert_header( + "Retry-After", + (Utc::now() + chrono::Duration::seconds(60)).to_rfc2822(), + ), + ResponseTemplate::new(status) + .insert_header("x-ratelimit-remaining", "0") + .insert_header( + "x-ratelimit-reset", + (Utc::now().timestamp() + 60).to_string(), + ), + ]; + for response in responses { + server.reset().await; + Mock::given(method("GET")) + .respond_with(response) + .mount(&server) + .await; + let result = tokio::time::timeout( + Duration::from_secs(5), + get_checked(&http, server.uri().parse().unwrap(), None, None), + ) + .await + .expect("a long rate-limit wait must fail immediately"); + assert!(result.is_err()); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + } + } + + #[tokio::test] + async fn an_empty_window_retains_the_previous_checkpoint_without_verification() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!([]))) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let since = Some("2026-01-02T00:00:00Z".parse().unwrap()); + let window = fetch_window(&http, &server.uri(), since, 200, false, None) + .await + .unwrap(); + assert!(window.issues.is_empty()); + assert_eq!(window.checkpoint, since); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn a_large_timestamp_group_finishes_and_the_next_run_passes_its_old_pages() { + let server = MockServer::start().await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let group_time = "2026-01-02T00:00:00Z"; + let later_time = "2026-01-03T00:00:00Z"; + // PR 多时先耗尽十页预算；全是工单时先耗尽 200 条预算。 + for issue_interval in [1, 100] { + server.reset().await; + let mut rows: Vec<_> = (1..=1001) + .map(|number| { + let mut row = listing_row(number, group_time); + if (number - 1) % issue_interval as i64 != 0 { + row["pull_request"] = json!({}); + } + row + }) + .collect(); + rows.push(listing_row(1002, later_time)); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let page = requested_page(request); + let rows: Vec<_> = rows + .iter() + .skip((page - 1) * 100) + .take(100) + .cloned() + .collect(); + ResponseTemplate::new(200).set_body_json(rows) + }) + .mount(&server) + .await; + let first = fetch_window(&http, &server.uri(), None, 200, false, None) + .await + .unwrap(); + let numbers: Vec<_> = first.issues.iter().map(|issue| issue.number).collect(); + assert_eq!( + numbers, + (1..=1001).step_by(issue_interval).collect::>() + ); + assert_eq!(first.checkpoint, Some(group_time.parse().unwrap())); + + // 模拟端点仍返回 inclusive 下界；十页旧记录不应耗尽新一轮的任一预算。 + let next = fetch_window(&http, &server.uri(), first.checkpoint, 200, false, None) + .await + .unwrap(); + assert_eq!(next.issues.len(), 1); + assert_eq!(next.issues[0].number, 1002); + assert_eq!(next.checkpoint, Some(later_time.parse().unwrap())); + let requests = server.received_requests().await.unwrap(); + assert!(requests.iter().all(|request| { + request + .url + .query_pairs() + .any(|(key, value)| key == "sort" && value == "updated") + && request + .url + .query_pairs() + .any(|(key, value)| key == "direction" && value == "asc") + })); + assert!(requests.iter().any(|request| requested_page(request) == 11)); + } + } + + #[tokio::test] + async fn a_moved_issue_rejects_the_window_and_is_read_after_pagination_stabilizes() { + let server = MockServer::start().await; + let moved = AtomicBool::new(false); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let page = requested_page(request); + if page == 2 { + moved.store(true, Ordering::SeqCst); + } + let mut rows: Vec<_> = (1..=201) + .map(|number| listing_row(number, "2026-01-02T00:00:00Z")) + .collect(); + if moved.load(Ordering::SeqCst) { + rows.remove(0); + rows.push(listing_row(1, "2026-01-03T00:00:00Z")); + } + let rows: Vec<_> = rows.into_iter().skip((page - 1) * 100).take(100).collect(); + ResponseTemplate::new(200).set_body_json(rows) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_window(&http, &server.uri(), None, 200, false, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("changed during pagination")); + + let recovered = fetch_window(&http, &server.uri(), None, 200, false, None) + .await + .unwrap(); + let recovered_numbers: Vec<_> = recovered.issues.iter().map(|issue| issue.number).collect(); + assert_eq!(recovered_numbers, (2..=201).collect::>()); + assert_eq!( + recovered.checkpoint, + Some("2026-01-02T00:00:00Z".parse().unwrap()) + ); + let next = fetch_window(&http, &server.uri(), recovered.checkpoint, 200, false, None) + .await + .unwrap(); + assert_eq!(next.issues.len(), 1); + assert_eq!(next.issues[0].number, 1); + assert_eq!( + next.checkpoint, + Some("2026-01-03T00:00:00Z".parse().unwrap()) + ); + } + + #[tokio::test] + async fn a_complete_discussion_is_read_beyond_ten_pages() { + let server = MockServer::start().await; + for count in [1000, 1001, 1100] { + server.reset().await; + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let page = requested_page(request); + let rows: Vec<_> = (0..count).skip((page - 1) * 100).take(100).collect(); + ResponseTemplate::new(200).set_body_json(rows) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_all::(&http, &server.uri(), &[], None) + .await + .unwrap(); + assert_eq!(result, (0..count).collect::>()); + let requests = server.received_requests().await.unwrap(); + assert_eq!(requests.len(), count as usize / 100 + 1); + } + } + + #[tokio::test] + async fn a_failed_discussion_page_does_not_return_a_partial_history() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(|request: &Request| { + let page = requested_page(request); + if page == 11 { + return ResponseTemplate::new(503); + } + let rows: Vec<_> = (0..1001).skip((page - 1) * 100).take(100).collect(); + ResponseTemplate::new(200).set_body_json(rows) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_all::(&http, &server.uri(), &[], None).await; + assert!(result.unwrap_err().to_string().contains("503")); + assert_eq!(server.received_requests().await.unwrap().len(), 11); + } fn issue(json: serde_json::Value) -> Issue { serde_json::from_value(json).unwrap() diff --git a/crates/utopia-server/src/github_sync_tests.rs b/crates/utopia-server/src/github_sync_tests.rs index 39b38db2b..025acb57e 100644 --- a/crates/utopia-server/src/github_sync_tests.rs +++ b/crates/utopia-server/src/github_sync_tests.rs @@ -260,14 +260,15 @@ async fn incremental_github_sync_preserves_the_complete_discussion() -> anyhow:: .iter() .filter(|request| request.url.path() == "/repos/test/repo/issues") .collect(); - anyhow::ensure!(issue_requests.len() == 6); + // 每轮先选窗口，再复读检查分页期间是否发生移动。 + anyhow::ensure!(issue_requests.len() == 12); for (index, request) in issue_requests.iter().enumerate() { let lower_bound = request .url .query_pairs() .find(|(key, _)| key == "since") .map(|(_, value)| time(&value)); - anyhow::ensure!(lower_bound == if index == 0 { None } else { since }); + anyhow::ensure!(lower_bound == if index < 2 { None } else { since }); } anyhow::ensure!( requests diff --git a/crates/utopia-server/src/governance.rs b/crates/utopia-server/src/governance.rs index fdba8ae0f..2e8eb9482 100644 --- a/crates/utopia-server/src/governance.rs +++ b/crates/utopia-server/src/governance.rs @@ -570,15 +570,11 @@ async fn apply(ctx: &Ctx<'_>, item: &ReviewItem, p: &Precedents, look: Look) -> return Ok(()); } // 执行闸门（0027）：合并会立刻送出图外的东西——违规、派生、答案——留给人， - // 把握再高也不动手。agent 的看法照记成建议，理由前面写明是闸门留下的 - let impact = execution_gate::impact_of(pool, kb_id, l, r).await?; - if let Some(hold) = execution_gate::hold(&impact) { - utopia_store::resolution::escalate_review( - pool, - item.id, - &format!("escalate_impact|{hold}"), - ) - .await?; + // 身份来源也留给人（0071），把握再高也不动手。agent 的看法照记成建议， + // 理由前面写明是闸门留下的 + if let Some(hold) = execution_gate::hold_for_merge(pool, kb_id, l, r).await? { + utopia_store::resolution::escalate_review(pool, item.id, &hold.review_reason()) + .await?; let held = match look.why.as_deref() { Some(why) => format!("held for a person: {}; {why}", hold.explain()), None => format!("held for a person: {}", hold.explain()), diff --git a/crates/utopia-server/src/ingest_sources.rs b/crates/utopia-server/src/ingest_sources.rs index 035d194b6..e2ec48d77 100644 --- a/crates/utopia-server/src/ingest_sources.rs +++ b/crates/utopia-server/src/ingest_sources.rs @@ -16,6 +16,10 @@ use uuid::Uuid; #[path = "source_checkpoint_tests.rs"] mod source_checkpoint_tests; +#[cfg(test)] +#[path = "timestamp_checkpoint_tests.rs"] +mod timestamp_checkpoint_tests; + /// 单次同步的新文档上限（防超长 feed/URL 列表拖垮任务） const MAX_NEW_PER_SYNC: usize = 200; const MAX_FEED_BYTES: usize = 4 * 1024 * 1024; @@ -39,6 +43,7 @@ pub struct SyncStats { pub discovered: usize, pub queued_for_content: usize, pub content_terminal: usize, + pub checkpoint: Option>, } #[derive(Debug)] @@ -69,9 +74,12 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() let source = utopia_store::sources::get(&state.pool, source_id).await?; let kind = SourceKind::parse(&source.kind); let since = match kind { - Some(SourceKind::Custom | SourceKind::GithubIssues | SourceKind::JiraIssues) => { + Some(SourceKind::Custom) => { utopia_store::sources::last_successful_sync_start(&state.pool, source_id).await? } + Some(SourceKind::GithubIssues | SourceKind::JiraIssues) => { + utopia_store::sources::sync_checkpoint(&state.pool, source_id).await? + } _ => None, }; utopia_store::sources::mark_running(&state.pool, source_id).await?; @@ -104,19 +112,51 @@ pub async fn sync_source(state: &AppState, source_id: Uuid) -> anyhow::Result<() None => Err(anyhow::anyhow!("unknown source kind `{}`", source.kind)), }; - match outcome { - Ok(stats) => { - utopia_store::sources::finish_run( + let outcome = match outcome { + Ok(stats) + if matches!( + kind, + Some(SourceKind::GithubIssues | SourceKind::JiraIssues) + ) => + { + utopia_store::sources::finish_timestamp_sync( &state.pool, run_id, source_id, - None, + since, + stats.checkpoint, stats.created as i32, stats.updated as i32, ) - .await?; - utopia_store::sources::finish_sync(&state.pool, source_id, None, stats.total() as i32) + .await + .map(|()| stats) + .map_err(anyhow::Error::from) + } + other => other, + }; + match outcome { + Ok(stats) => { + if !matches!( + kind, + Some(SourceKind::GithubIssues | SourceKind::JiraIssues) + ) { + utopia_store::sources::finish_run( + &state.pool, + run_id, + source_id, + None, + stats.created as i32, + stats.updated as i32, + ) + .await?; + utopia_store::sources::finish_sync( + &state.pool, + source_id, + None, + stats.total() as i32, + ) .await?; + } state.emit_source(source.kb_id); tracing::info!( %source_id, @@ -844,20 +884,20 @@ async fn sync_github_issues_with_client( let base = format!("{api_base}/repos/{repo}"); - // 增量：GitHub 的 since 是"这之后更新过的" - let mut issue_q: Vec<(&str, String)> = vec![("state", "all".into())]; - if let Some(t) = since { - issue_q.push(("since", t.to_rfc3339())); - } - - let issues: Vec = - crate::github_issues::fetch_all(http, &format!("{base}/issues"), &issue_q, auth).await?; - let mut stats = SyncStats::default(); - for issue in issues - .iter() - .filter(|i| include_prs || i.pull_request.is_none()) - .take(MAX_NEW_PER_SYNC) - { + let window = crate::github_issues::fetch_window( + http, + &format!("{base}/issues"), + since, + MAX_NEW_PER_SYNC, + include_prs, + auth, + ) + .await?; + let mut stats = SyncStats { + checkpoint: window.checkpoint, + ..SyncStats::default() + }; + for issue in &window.issues { // 只重建本轮选中的工单，但每张要读完整评论历史。仓库级评论即使 // 去掉 since，也可能被无关工单和 PR 的评论挤出分页上限。 let comments = crate::github_issues::fetch_all::( @@ -1217,21 +1257,22 @@ async fn sync_jira_issues( ) .await?; - let jql = crate::jira_issues::jql(project, since); - let (issues, total) = crate::jira_issues::fetch_all(&http, base_url, &jql, auth).await?; - // **截断了就说出来。** 一个跑了多年的项目动辄上万张工单，翻页上限意味着 - // 这一轮只覆盖了一段；不报的话界面上"同步完成"就是一句误导 - if total > issues.len() as i64 { - tracing::warn!( - source_id = %source.id, - fetched = issues.len(), - total, - "Jira 结果被翻页上限截断，本轮只覆盖了一部分；下一轮的 JQL 窗口会接上" - ); - } - - let mut stats = SyncStats::default(); - for issue in issues.iter().take(MAX_NEW_PER_SYNC) { + let cloud = crate::jira_issues::is_jira_cloud(base_url); + let window = crate::jira_issues::fetch_window( + &http, + base_url, + cloud, + project, + since, + MAX_NEW_PER_SYNC, + auth, + ) + .await?; + let mut stats = SyncStats { + checkpoint: window.checkpoint, + ..SyncStats::default() + }; + for issue in &window.issues { let body = crate::jira_issues::render(issue); // 逻辑身份带上站点：一个知识库接两个 Jira 时，PROJ-1 不会互相覆盖 let host = reqwest::Url::parse(base_url) diff --git a/crates/utopia-server/src/jira_issues.rs b/crates/utopia-server/src/jira_issues.rs index 4ade51412..6cea99774 100644 --- a/crates/utopia-server/src/jira_issues.rs +++ b/crates/utopia-server/src/jira_issues.rs @@ -37,16 +37,22 @@ //! 评论正文换成了 ADF（一棵 JSON 树而非字符串）——那需要一个渲染器，是另一件事。 //! v2 在 Cloud 上通常仍可用且返回字符串，所以先只做 v2；真遇到只有 v3 的实例 //! 再补，那时才知道 ADF 的哪些节点是必须处理的。 +//! +//! 但 Cloud 撤掉了 `search`（410 Gone，CHANGE-2046），只剩 `search/jql`： +//! 按 `nextPageToken` 翻页、`isLast` 收尾，没有 `startAt` 也没有 `total`。 +//! Server/DC 仍只有旧端点，所以只对 Cloud 站点换。 use chrono::{DateTime, Utc}; use serde::Deserialize; +use std::collections::BTreeMap; +use std::time::Duration; use crate::ticket::stamp; -/// 一次同步最多翻多少页。Jira 的 `total` 常常是几万，全量拉回来没有意义—— -/// 增量窗口之外的等下一次 JQL 取。 +/// 达到列表预算后仍读完当前时间组，下次从完整组之后继续。 const MAX_PAGES: u32 = 10; const PAGE_SIZE: u32 = 50; +const MAX_RATE_LIMIT_RETRIES: u32 = 3; /// 想要的字段。**必须显式列**：不列 `comment` 就不返回评论， /// 而默认返回全部字段会把响应撑到几百 KB 一条。 @@ -57,10 +63,30 @@ const FIELDS: &str = "summary,status,issuetype,priority,created,updated,resoluti pub struct SearchPage { #[serde(default)] pub issues: Vec, - #[serde(default)] + // total 用来识别提前结束的分页，缺失时不能把空页当作完整结果。 pub total: i64, } +/// Cloud `search/jql` 的一页：没有 total，只能靠 token 与 isLast 判断是否读完。 +#[derive(Debug, Deserialize)] +pub struct JqlSearchPage { + #[serde(default)] + pub issues: Vec, + #[serde(rename = "nextPageToken")] + pub next_page_token: Option, + #[serde(rename = "isLast", default)] + pub is_last: bool, +} + +/// 只有 Cloud 撤掉了旧 `search`；Server/DC 站点换过去反而会 404。 +pub fn is_jira_cloud(base_url: &str) -> bool { + reqwest::Url::parse(base_url).is_ok_and(|url| match url.host_str() { + Some("api.atlassian.com") => url.path().starts_with("/ex/jira"), + Some(host) => host.ends_with(".atlassian.net"), + None => false, + }) +} + #[derive(Debug, Deserialize)] pub struct Issue { pub key: String, @@ -243,67 +269,612 @@ pub fn jql(project: &str, since: Option>) -> String { t.format("%Y-%m-%d %H:%M") )); } - q.push_str(" ORDER BY updated ASC"); + q.push_str(" ORDER BY updated ASC, key ASC"); q } -/// 分页取。Jira 用 `startAt`/`maxResults`，`total` 常常是几万—— -/// 靠 MAX_PAGES 封顶，剩下的等下一次增量窗口。 -/// -/// **把 `total` 一起返回**：截断了就得说出来。取回 500 条而服务端有 14506 条时， -/// 界面上"同步完成"是一句误导——真实情况是"这一轮只覆盖了一小段"。 -pub async fn fetch_all( +pub struct IssueWindow { + pub issues: Vec, + pub checkpoint: Option>, +} + +/// 复读固定的已选前缀，检测更新导致的 offset 位移；变化时不确认本轮 checkpoint。 +/// `cloud` 由调用方按 [`is_jira_cloud`] 给出，测试的本地 mock 才能走 Cloud 的翻页。 +pub async fn fetch_window( http: &reqwest::Client, base_url: &str, - jql: &str, + cloud: bool, + project: &str, + since: Option>, + item_limit: usize, auth: Option<&str>, -) -> anyhow::Result<(Vec, i64)> { - let mut out = Vec::new(); - let mut total = 0i64; - for page in 0..MAX_PAGES { +) -> anyhow::Result { + let query = jql(project, since); + let (window, positions) = + fetch_window_pass(http, base_url, cloud, &query, since, item_limit, auth, None).await?; + if let Some(checkpoint) = window + .checkpoint + .filter(|checkpoint| Some(*checkpoint) != since) + { + let (verified, verified_positions) = fetch_window_pass( + http, + base_url, + cloud, + &query, + since, + item_limit, + auth, + Some(checkpoint), + ) + .await?; + anyhow::ensure!( + verified.checkpoint == Some(checkpoint) && positions == verified_positions, + "Jira issues changed during pagination; retry the previous checkpoint" + ); + } + Ok(window) +} + +#[allow(clippy::too_many_arguments)] +async fn fetch_window_pass( + http: &reqwest::Client, + base_url: &str, + cloud: bool, + query: &str, + since: Option>, + item_limit: usize, + auth: Option<&str>, + through: Option>, +) -> anyhow::Result<(IssueWindow, BTreeMap>)> { + let mut window = IssueWindow { + issues: Vec::new(), + checkpoint: since, + }; + let mut positions = BTreeMap::new(); + let mut previous_time = None; + let mut stop_after = through; + let mut listing_pages = 0; + let mut start_at = 0; + let mut next_page_token: Option = None; + let path = if cloud { "search/jql" } else { "search" }; + loop { let mut url = reqwest::Url::parse(&format!( - "{}/rest/api/2/search", + "{}/rest/api/2/{path}", base_url.trim_end_matches('/') ))?; { let mut q = url.query_pairs_mut(); - q.append_pair("jql", jql); + q.append_pair("jql", query); q.append_pair("expand", "changelog"); q.append_pair("fields", FIELDS); q.append_pair("maxResults", &PAGE_SIZE.to_string()); - q.append_pair("startAt", &(page * PAGE_SIZE).to_string()); + if !cloud { + q.append_pair("startAt", &start_at.to_string()); + } else if let Some(token) = &next_page_token { + q.append_pair("nextPageToken", token); + } + } + let (issues, more) = if cloud { + let page_data: JqlSearchPage = fetch_page(http, url, auth).await?; + // 没有 total 可对账：未到末页却给空页或不给 token，都不能当作读完。 + anyhow::ensure!( + page_data.is_last + || (!page_data.issues.is_empty() && page_data.next_page_token.is_some()), + "Jira returned no next page before its last page; retry the previous checkpoint" + ); + next_page_token = page_data.next_page_token; + (page_data.issues, !page_data.is_last) + } else { + let page_data: SearchPage = fetch_page(http, url, auth).await?; + let count = page_data.issues.len(); + anyhow::ensure!( + page_data.total >= 0 && (start_at + count) as i64 <= page_data.total, + "Jira returned an inconsistent page total; retry the previous checkpoint" + ); + anyhow::ensure!( + count != 0 || start_at as i64 >= page_data.total, + "Jira returned an empty page before its total; retry the previous checkpoint" + ); + start_at += count; + ( + page_data.issues, + count != 0 && (start_at as i64) < page_data.total, + ) + }; + let mut page_counted = false; + for issue in issues { + let updated = issue + .fields + .updated + .ok_or_else(|| { + anyhow::anyhow!("Jira issue {} is missing its update time", issue.key) + })? + .0; + anyhow::ensure!( + previous_time.is_none_or(|previous| updated >= previous), + "Jira issues are not ordered by update time; retry the previous checkpoint" + ); + previous_time = Some(updated); + if stop_after.is_some_and(|boundary| updated > boundary) { + return Ok((window, positions)); + } + // 分钟取整返回的旧记录也可能改变 offset，复读时不能只比较新记录。 + anyhow::ensure!( + positions.insert(issue.key.clone(), updated).is_none(), + "Jira issue repeated during pagination; retry the previous checkpoint" + ); + if since.is_some_and(|checkpoint| updated <= checkpoint) { + continue; + } + if !page_counted { + listing_pages += 1; + page_counted = true; + } + window.checkpoint = Some(updated); + window.issues.push(issue); + if through.is_none() + && (window.issues.len() >= item_limit || listing_pages >= MAX_PAGES) + { + stop_after = Some(updated); + } + } + if !more { + return Ok((window, positions)); } + } +} + +async fn fetch_page( + http: &reqwest::Client, + url: reqwest::Url, + auth: Option<&str>, +) -> anyhow::Result { + let mut retries = 0; + loop { let mut req = http - .get(url) + .get(url.clone()) .header(reqwest::header::ACCEPT, "application/json"); - if let Some(a) = auth { - req = req.header(reqwest::header::AUTHORIZATION, a); + if let Some(auth) = auth { + req = req.header(reqwest::header::AUTHORIZATION, auth); } let resp = req.send().await?; let status = resp.status(); - if !status.is_success() { - // Jira 把 JQL 语法错误也报成 400，正文里才是原因。 - // 只说 "HTTP 400" 会让人去查网络，而真正该改的是 project key 或时间格式 - let detail = resp.text().await.unwrap_or_default(); - anyhow::bail!( - "HTTP {status} from Jira: {}", - detail.chars().take(200).collect::() - ); + if status.is_success() { + return Ok(resp.json().await?); } - let page_data: SearchPage = resp.json().await?; - total = page_data.total; - let n = page_data.issues.len(); - out.extend(page_data.issues); - if n < PAGE_SIZE as usize { - break; + if status == reqwest::StatusCode::TOO_MANY_REQUESTS && retries < MAX_RATE_LIMIT_RETRIES { + if let Some(delay) = rate_limit_delay(resp.headers(), Utc::now()) { + // 保留已读取的组和当前页，不能把限流变成从头扫描的循环。 + let delay = delay.max(Duration::from_secs(1 << retries)); + retries += 1; + tracing::info!( + retries, + wait_seconds = delay.as_secs(), + "Jira rate limited; waiting before retry" + ); + drop(resp); + tokio::time::sleep(delay).await; + continue; + } } + // Jira 把 JQL 语法错误也报成 400，正文里才是原因。 + let detail = resp.text().await.unwrap_or_default(); + anyhow::bail!( + "HTTP {status} from Jira: {}", + detail.chars().take(200).collect::() + ); } - Ok((out, total)) +} + +fn rate_limit_delay(headers: &reqwest::header::HeaderMap, now: DateTime) -> Option { + let value = headers + .get(reqwest::header::RETRY_AFTER)? + .to_str() + .ok()? + .trim(); + let delay = value + .parse::() + .ok() + .map(Duration::from_secs) + .or_else(|| { + DateTime::parse_from_rfc2822(value) + .ok() + .map(|at| at.signed_duration_since(now).to_std().unwrap_or_default()) + })?; + // shortcut: 长等待走普通失败重试；严格遵循跨任务的 Retry-After 需要持久化调度时间。 + (delay <= Duration::from_secs(30)).then_some(delay) } #[cfg(test)] mod tests { use super::*; + use serde_json::{json, Value}; + use std::sync::atomic::{AtomicBool, Ordering}; + use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate}; + + fn listing_row(number: i64, updated: &str) -> Value { + json!({"key": format!("X-{number}"), "fields": {"summary": "Issue", "updated": updated}}) + } + + fn requested_offset(request: &Request) -> usize { + request + .url + .query_pairs() + .find(|(key, _)| key == "startAt") + .unwrap() + .1 + .parse() + .unwrap() + } + + #[test] + fn rate_limit_headers_supply_a_bounded_wait_without_guessing() { + let now: DateTime = "2026-01-02T00:00:00Z".parse().unwrap(); + let mut headers = reqwest::header::HeaderMap::new(); + assert_eq!(rate_limit_delay(&headers, now), None); + for (value, expected) in [ + ("2", Some(2)), + ("0", Some(0)), + ("30", Some(30)), + ("31", None), + ("-1", None), + ("broken", None), + ("86401", None), + ] { + headers.insert(reqwest::header::RETRY_AFTER, value.parse().unwrap()); + assert_eq!( + rate_limit_delay(&headers, now), + expected.map(Duration::from_secs) + ); + } + for (seconds, expected) in [(3, Some(3)), (30, Some(30)), (31, None)] { + headers.insert( + reqwest::header::RETRY_AFTER, + (now + chrono::Duration::seconds(seconds)) + .to_rfc2822() + .parse() + .unwrap(), + ); + assert_eq!( + rate_limit_delay(&headers, now), + expected.map(Duration::from_secs) + ); + } + } + + #[tokio::test] + async fn a_long_rate_limit_wait_fails_without_holding_the_sync_slot() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(429).insert_header("Retry-After", "31")) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let url = reqwest::Url::parse(&server.uri()).unwrap(); + let result = tokio::time::timeout( + Duration::from_secs(1), + fetch_page::(&http, url, None), + ) + .await + .expect("a long Retry-After must return an error without waiting"); + assert!(result.unwrap_err().to_string().contains("HTTP 429")); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn a_rate_limited_page_resumes_without_rereading_the_previous_page() { + let server = MockServer::start().await; + let limited = AtomicBool::new(false); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let offset = requested_offset(request); + if offset == 50 && !limited.swap(true, Ordering::SeqCst) { + return ResponseTemplate::new(429).insert_header("Retry-After", "0"); + } + let rows: Vec<_> = (1..=51) + .skip(offset) + .take(50) + .map(|number| listing_row(number, "2026-01-02T00:00:00Z")) + .collect(); + ResponseTemplate::new(200).set_body_json(json!({"issues": rows, "total": 51})) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let started = std::time::Instant::now(); + let window = fetch_window( + &http, + &server.uri(), + false, + "X", + None, + 200, + Some("Bearer fixture"), + ) + .await + .unwrap(); + assert_eq!(window.issues.len(), 51); + assert!(started.elapsed() >= Duration::from_secs(1)); + let requests = server.received_requests().await.unwrap(); + assert_eq!( + requests.iter().map(requested_offset).collect::>(), + [0, 50, 50, 0, 50] + ); + assert!(requests + .iter() + .all( + |request| request.headers.get("authorization").unwrap() == "Bearer fixture" + && request.headers.get("accept").unwrap() == "application/json" + )); + } + + #[tokio::test] + async fn an_empty_window_retains_the_previous_checkpoint_without_verification() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with( + ResponseTemplate::new(200).set_body_json(json!({"issues": [], "total": 0})), + ) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let since = Some("2026-01-02T00:00:00Z".parse().unwrap()); + let window = fetch_window(&http, &server.uri(), false, "X", since, 200, None) + .await + .unwrap(); + assert!(window.issues.is_empty()); + assert_eq!(window.checkpoint, since); + assert_eq!(server.received_requests().await.unwrap().len(), 1); + } + + #[tokio::test] + async fn a_large_timestamp_group_finishes_and_the_next_run_passes_its_old_pages() { + // Jira may return fewer items per page than the requested maxResults. + for (group_size, page_size) in [(501, 50), (101, 10)] { + let server = MockServer::start().await; + let group_time = "2026-01-02T12:34:56Z"; + let later_time = "2026-01-02T12:34:57Z"; + let mut rows: Vec<_> = (1..=group_size) + .map(|number| listing_row(number, group_time)) + .collect(); + rows.push(listing_row(group_size + 1, later_time)); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let offset = requested_offset(request); + let page: Vec<_> = rows.iter().skip(offset).take(page_size).cloned().collect(); + ResponseTemplate::new(200).set_body_json( + json!({"issues": page, "total": rows.len(), "maxResults": page_size}), + ) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let first = fetch_window(&http, &server.uri(), false, "X", None, 200, None) + .await + .unwrap(); + assert_eq!(first.issues.len(), group_size as usize); + assert_eq!( + first + .issues + .iter() + .map(|issue| issue.key.clone()) + .collect::>(), + (1..=group_size) + .map(|number| format!("X-{number}")) + .collect::>() + ); + assert_eq!(first.checkpoint, Some(group_time.parse().unwrap())); + let next = fetch_window( + &http, + &server.uri(), + false, + "X", + first.checkpoint, + 200, + None, + ) + .await + .unwrap(); + assert_eq!(next.issues.len(), 1); + assert_eq!(next.issues[0].key, format!("X-{}", group_size + 1)); + assert_eq!(next.checkpoint, Some(later_time.parse().unwrap())); + let requests = server.received_requests().await.unwrap(); + assert!(requests.iter().any(|request| { + requested_offset(request) == 10 * page_size + && request.url.query_pairs().any(|(key, value)| { + key == "jql" + && value + == "project = X AND updated >= \"2026-01-02 12:34\" ORDER BY updated ASC, key ASC" + }) + })); + } + } + + #[tokio::test] + async fn a_moved_issue_rejects_the_window_and_a_stable_retry_recovers_every_issue() { + let server = MockServer::start().await; + let moved = AtomicBool::new(false); + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let offset = requested_offset(request); + if offset == 50 { + moved.store(true, Ordering::SeqCst); + } + let mut rows: Vec<_> = (1..=201) + .map(|number| listing_row(number, "2026-01-02T00:00:00Z")) + .collect(); + if moved.load(Ordering::SeqCst) { + rows.remove(0); + rows.push(listing_row(1, "2026-01-03T00:00:00Z")); + } + let page: Vec<_> = rows.into_iter().skip(offset).take(50).collect(); + ResponseTemplate::new(200).set_body_json(json!({"issues": page, "total": 201})) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_window(&http, &server.uri(), false, "X", None, 200, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("changed during pagination")); + + let recovered = fetch_window(&http, &server.uri(), false, "X", None, 200, None) + .await + .unwrap(); + let mut keys: Vec<_> = recovered.issues.iter().map(|issue| &issue.key).collect(); + keys.sort(); + let mut expected_keys: Vec<_> = (2..=201).map(|number| format!("X-{number}")).collect(); + expected_keys.sort(); + assert_eq!(keys, expected_keys.iter().collect::>()); + assert_eq!( + recovered.checkpoint, + Some("2026-01-02T00:00:00Z".parse().unwrap()) + ); + + let next = fetch_window( + &http, + &server.uri(), + false, + "X", + recovered.checkpoint, + 200, + None, + ) + .await + .unwrap(); + assert_eq!(next.issues.len(), 1); + assert_eq!(next.issues[0].key, "X-1"); + assert_eq!( + next.checkpoint, + Some("2026-01-03T00:00:00Z".parse().unwrap()) + ); + } + + #[tokio::test] + async fn an_issue_without_an_update_time_cannot_advance_a_checkpoint() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "issues": [{"key": "X-1", "fields": {"summary": "Missing time"}}], "total": 1, + }))) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_window(&http, &server.uri(), false, "X", None, 200, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("missing its update time")); + } + + #[tokio::test] + async fn an_empty_page_cannot_finish_a_group_when_jira_reports_unread_items() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + let rows: Vec<_> = if requested_offset(request) == 0 { + (1..=50) + .map(|number| listing_row(number, "2026-01-02T00:00:00Z")) + .collect() + } else { + Vec::new() + }; + ResponseTemplate::new(200).set_body_json(json!({"issues": rows, "total": 51})) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let result = fetch_window(&http, &server.uri(), false, "X", None, 200, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("empty page before its total")); + assert!(serde_json::from_value::(json!({"issues": []})).is_err()); + for response in [ + json!({"issues": [], "total": -1}), + json!({"issues": [listing_row(1, "2026-01-02T00:00:00Z")], "total": 0}), + ] { + server.reset().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_json(response)) + .mount(&server) + .await; + let result = fetch_window(&http, &server.uri(), false, "X", None, 200, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("inconsistent page total")); + } + } + + #[test] + fn only_cloud_sites_use_the_jql_search() { + assert!(is_jira_cloud("https://acme.atlassian.net")); + assert!(is_jira_cloud("https://api.atlassian.com/ex/jira/abc-123")); + assert!(!is_jira_cloud("https://issues.apache.org/jira")); + assert!(!is_jira_cloud( + "https://api.atlassian.com/ex/confluence/abc" + )); + } + + #[tokio::test] + async fn a_cloud_site_pages_the_jql_search_by_token_until_its_last_page() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .respond_with(move |request: &Request| { + assert_eq!(request.url.path(), "/rest/api/2/search/jql"); + assert!(!request.url.query_pairs().any(|(key, _)| key == "startAt")); + let token = request + .url + .query_pairs() + .find(|(key, _)| key == "nextPageToken") + .map(|(_, value)| value.into_owned()); + let body = match token.as_deref() { + None => json!({"issues": [listing_row(1, "2026-01-02T00:00:00Z")], + "nextPageToken": "p2", "isLast": false}), + Some("p2") => json!({"issues": [listing_row(2, "2026-01-03T00:00:00Z")], + "isLast": true}), + other => panic!("unexpected token {other:?}"), + }; + ResponseTemplate::new(200).set_body_json(body) + }) + .mount(&server) + .await; + let http = reqwest::Client::builder().no_proxy().build().unwrap(); + let window = fetch_window(&http, &server.uri(), true, "X", None, 200, None) + .await + .unwrap(); + let keys: Vec<_> = window + .issues + .iter() + .map(|issue| issue.key.as_str()) + .collect(); + assert_eq!(keys, ["X-1", "X-2"]); + assert_eq!( + window.checkpoint, + Some("2026-01-03T00:00:00Z".parse().unwrap()) + ); + + // 末页之前不给 token，不能当作读完而确认 checkpoint + server.reset().await; + Mock::given(method("GET")) + .respond_with(ResponseTemplate::new(200).set_body_json( + json!({"issues": [listing_row(1, "2026-01-02T00:00:00Z")], "isLast": false}), + )) + .mount(&server) + .await; + let result = fetch_window(&http, &server.uri(), true, "X", None, 200, None).await; + assert!(result + .err() + .unwrap() + .to_string() + .contains("no next page before its last page")); + } /// **Jira 的时间戳不是 RFC3339。** `+0000` 没有冒号，chrono 的默认实现解不了。 /// 这一条是整个模块最容易悄悄坏掉的地方：解不出来就是整页工单丢掉。 @@ -322,13 +893,14 @@ mod tests { let t = DateTime::parse_from_rfc3339("2026-08-30T12:34:56Z") .unwrap() .with_timezone(&Utc); - let q = jql("KAFKA", Some(t)); - assert!(q.contains("project = KAFKA"), "{q}"); - assert!(q.contains("updated >= \"2026-08-30 12:34\""), "{q}"); - assert!(q.contains("ORDER BY updated ASC"), "{q}"); - // 没有 since 时不该带上 updated 子句。**查子句不查词**—— - // ORDER BY updated ASC 里也有 "updated"，第一版断言就栽在这上面 - assert!(!jql("KAFKA", None).contains("updated >=")); + assert_eq!( + jql("KAFKA", Some(t)), + "project = KAFKA AND updated >= \"2026-08-30 12:34\" ORDER BY updated ASC, key ASC" + ); + assert_eq!( + jql("KAFKA", None), + "project = KAFKA ORDER BY updated ASC, key ASC" + ); } /// **拿真实响应钉住字段形状。** diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rs index 0879a4935..a86ac387e 100644 --- a/crates/utopia-server/src/main.rs +++ b/crates/utopia-server/src/main.rs @@ -34,6 +34,7 @@ mod owl_import; mod pack_alignment; mod phrase_alignment; mod pipeline; +mod pushed_keys; mod query_engine; mod rdf; mod readers; diff --git a/crates/utopia-server/src/namesake_decision_measurements.rs b/crates/utopia-server/src/namesake_decision_measurements.rs new file mode 100644 index 000000000..6a08217cc --- /dev/null +++ b/crates/utopia-server/src/namesake_decision_measurements.rs @@ -0,0 +1,376 @@ +//! #1193 的裁决路径测量，与原八项案例分开。来源身份决定金标准，脚本回复只推动真实调用方。 +//! 除证据归属和图/审核状态一致性外，逐名锁定现有金标准错误：名单内必须仍错，名单外必须正确。 +//! 修复要同时移除对应名字；错误意外消失与新回归都不能被一个总数掩盖。 +use super::*; + +const KNOWN_GOLD_ERRORS: &[&str] = &[ + "Batch/ordinary/forward/distinct", + "Batch/ordinary/reverse/distinct", + "Batch/governance/forward/distinct", + "Batch/governance/reverse/distinct", + "BatchHigh/ordinary/forward/distinct", + "BatchHigh/ordinary/reverse/distinct", + "BatchHigh/governance/forward/distinct", + "BatchHigh/governance/reverse/distinct", + "Cached/ordinary/forward/distinct", + "Cached/ordinary/reverse/distinct", + "Cached/governance/forward/distinct", + "Cached/governance/reverse/distinct", + "Facts/ordinary/forward/distinct", + "Facts/ordinary/reverse/distinct", + "Facts/governance/forward/distinct", + "Facts/governance/reverse/distinct", + "Quotes/ordinary/forward/distinct", + "Quotes/ordinary/reverse/distinct", + "Quotes/governance/forward/distinct", + "Quotes/governance/reverse/distinct", + "QuotesSplit/ordinary/forward/same", + "QuotesSplit/ordinary/reverse/same", + "QuotesSplit/governance/forward/same", + "QuotesSplit/governance/reverse/same", +]; + +#[derive(Clone, Copy, Debug)] +enum Path { + Batch, + BatchHigh, + Cached, + Facts, + Quotes, + QuotesSplit, +} + +fn tool(name: &str, args: Value, id: usize) -> Value { + json!({"id":format!("call_{id}"),"type":"function", + "function":{"name":name,"arguments":args.to_string()}}) +} + +impl Fx { + async fn measure_decision( + &self, + path: Path, + governance: bool, + gold_same: bool, + reverse: bool, + ) -> anyhow::Result { + let order = if reverse { [1, 0] } else { [0, 1] }; + let mut holders = [Uuid::nil(); 2]; + // 登记册原文明说员工号与人一一对应；金标准来自来源身份，不由模型判决反推， + // 也不靠本体声明唯一关系——这个开放图谱没有这样的声明。 + let numbers = if gold_same { ["E17", "E17"] } else { EMPLOYEES }; + let quotes = numbers.map(|n| { + format!("Register R assigns one number per person and one person per number: Alex Lee is employee {n}.") + }); + for i in order { + holders[i] = self.seed(i, Check::Adjudicator { literal: true }).await?; + sqlx::query( + "UPDATE facts SET object_value=$1 WHERE subject_id=$2 AND phrase='employee number'", + ) + .bind(json!({"value":numbers[i]})) + .bind(holders[i]) + .execute(&self.pool) + .await?; + sqlx::query("UPDATE chunks SET text=$1 WHERE id=$2") + .bind("es[i]) + .bind(self.chunks[i]) + .execute(&self.pool) + .await?; + sqlx::query("UPDATE fact_evidence SET quote=$1 WHERE chunk_id=$2") + .bind("es[i]) + .bind(self.chunks[i]) + .execute(&self.pool) + .await?; + } + sqlx::query("UPDATE knowledge_bases SET governance=$1 WHERE id=$2") + .bind(governance) + .bind(self.kb) + .execute(&self.pool) + .await?; + utopia_store::resolution::create_review( + &self.pool, + self.kb, + holders[order[0]], + holders[order[1]], + 0.7, + "scripted_same_name", + ReviewStage::Adjudicating, + ) + .await?; + if matches!(path, Path::Cached) { + let items = + utopia_store::resolution::pending_adjudications(&self.pool, self.kb, 1).await?; + anyhow::ensure!(items.len() == 1, "missing pending pair"); + let p = + utopia_store::governance::precedents_for(&self.pool, self.kb, &items[0]).await?; + let key = crate::adjudication::seed_measurement_cache_key( + &items[0], + &utopia_store::governance::render_lines(&p), + ); + utopia_store::resolution::put_verdict( + &self.pool, + self.kb, + &key, + Some(true), + 0.8, + "scripted", + ) + .await?; + } + // 治理不读普通裁决缓存：预种缓存的治理案例仍然要问一次攒批模型。 + let asks_batch = governance || !matches!(path, Path::Cached); + let reads_evidence = matches!(path, Path::Facts | Path::Quotes | Path::QuotesSplit); + let confidence = if reads_evidence { + 0.6 + } else if matches!(path, Path::BatchHigh) { + 0.9 + } else { + 0.8 + }; + let investigates = reads_evidence || (governance && confidence < 0.85); + if asks_batch { + self.model.steps.lock().unwrap().push_back(Step { + prompt_has: "Alex Lee".into(), + stream: false, + text: json!({"verdicts":[{"i":0,"verdict":"same", + "confidence":confidence, + "why":"scripted same, independent of source gold"}]}) + .to_string(), + }); + } + if investigates { + if reads_evidence { + let name = if matches!(path, Path::Quotes | Path::QuotesSplit) { + "quotes" + } else { + "facts" + }; + let calls = ["A", "B"] + .into_iter() + .enumerate() + .map(|(i, side)| tool(name, json!({"side":side}), i)) + .collect::>(); + self.model.steps.lock().unwrap().push_back(Step { + prompt_has: "Alex Lee".into(), + stream: false, + text: json!({"role":"assistant","content":null,"tool_calls":calls}).to_string(), + }); + } + self.model.steps.lock().unwrap().push_back(Step { + prompt_has: "Alex Lee".into(), + stream: false, + text: json!({"role":"assistant","content":null,"tool_calls":[tool("decide", json!({ + "verdict":if matches!(path, Path::QuotesSplit) {"different"} else {"same"}, + "confidence":if matches!(path, Path::QuotesSplit) {0.5} else {0.95}, + "why":"scripted conclusion is independent of source gold" + }), 2)]}) + .to_string(), + }); + } + if governance { + crate::governance::govern(&self.state, self.kb).await?; + } else { + crate::adjudication::adjudicate_entities(&self.state, self.kb).await?; + } + anyhow::ensure!( + self.model.errors.lock().unwrap().is_empty(), + "transport errors: {:?}", + self.model.errors.lock().unwrap() + ); + anyhow::ensure!( + self.model.steps.lock().unwrap().is_empty(), + "unconsumed script" + ); + let requests = self.model.requests.lock().unwrap().clone(); + // 检查实际 HTTP 请求里各自那一侧，包括工具结果；只查整段 contains 会放过 A/B 交换。 + if asks_batch { + let messages = requests[0]["messages"].as_array().unwrap(); + let user = messages.iter().find(|m| m["role"] == "user").unwrap()["content"] + .as_str() + .unwrap(); + let (a, b) = user + .split_once("\nRecord B: ") + .ok_or_else(|| anyhow::anyhow!("missing Record B"))?; + anyhow::ensure!( + a.contains(numbers[order[0]]) && b.contains(numbers[order[1]]), + "facts not delivered to their own sides" + ); + if !gold_same { + anyhow::ensure!( + !a.contains(numbers[order[1]]) && !b.contains(numbers[order[0]]), + "cross-owned initial evidence" + ); + } + } + let mut read = Vec::new(); + if reads_evidence { + for m in requests.last().unwrap()["messages"].as_array().unwrap() { + if m["role"] != "tool" { + continue; + } + let id = m["tool_call_id"].as_str().unwrap(); + let i = match id { + "call_0" => 0, + "call_1" => 1, + _ => continue, + }; + let content = m["content"].as_str().unwrap(); + anyhow::ensure!( + content.contains(numbers[order[i]]), + "missing owned tool evidence" + ); + if !gold_same { + anyhow::ensure!( + !content.contains(numbers[order[1 - i]]), + "cross-owned tool evidence" + ); + } + if matches!(path, Path::Quotes | Path::QuotesSplit) { + anyhow::ensure!( + content.contains("es[order[i]]), + "quote was not read verbatim" + ); + } + read.push(json!({"side":if i==0 {"A"} else {"B"},"content":content})); + } + anyhow::ensure!( + read.len() == 2, + "both tool sides must reach the final request" + ); + } + let l = utopia_store::resolution::survivor(&self.pool, self.kb, holders[0]).await?; + let r = utopia_store::resolution::survivor(&self.pool, self.kb, holders[1]).await?; + let review: (String, String, Option) = + sqlx::query_as("SELECT status,stage,reason FROM resolution_reviews WHERE kb_id=$1") + .bind(self.kb) + .fetch_one(&self.pool) + .await?; + anyhow::ensure!( + (l == r) == (review.0 == "merged"), + "graph and review disagree" + ); + let decisions: Vec<(i32, Value)> = sqlx::query_as( + "SELECT calls,trace FROM agent_decisions WHERE kb_id=$1 ORDER BY created_at,id", + ) + .bind(self.kb) + .fetch_all(&self.pool) + .await?; + let human: i64 = sqlx::query_scalar("SELECT count(*) FROM resolution_reviews WHERE kb_id=$1 AND stage='human' AND status='pending'") + .bind(self.kb).fetch_one(&self.pool).await?; + // 终态再执行一次不能悄悄新增调查和模型调用。 + if governance { + crate::governance::govern(&self.state, self.kb).await?; + } else { + crate::adjudication::adjudicate_entities(&self.state, self.kb).await?; + } + anyhow::ensure!( + self.model.requests.lock().unwrap().len() == requests.len(), + "terminal pair was investigated again" + ); + let id = format!( + "{path:?}/{}/{}/{}", + if governance { "governance" } else { "ordinary" }, + if reverse { "reverse" } else { "forward" }, + if gold_same { "same" } else { "distinct" }, + ); + Ok( + json!({"id":id,"path":format!("{path:?}"),"governance":governance, + "order":if reverse {"reverse"} else {"forward"},"source_gold_same":gold_same,"batch_confidence":confidence, + "source_quotes":quotes,"observed_same":l==r,"gold_error":(l==r)!=gold_same, + "model_calls":requests.len(),"investigation_calls":decisions.iter().map(|d|d.0).sum::(), + "source_passages_read":matches!(path,Path::Quotes | Path::QuotesSplit) && !read.is_empty(), + "tool_results":read,"decisions":decisions,"review":review,"pending_human":human}), + ) + } +} + +#[tokio::test] +async fn namesake_decision_measurements_have_only_the_named_gold_errors() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = sqlx::PgPool::connect(&url).await?; + let mut rows = Vec::new(); + let mut seen = Vec::new(); + let mut mismatches = Vec::new(); + let case = Case { + id: "decision_measurement", + check: Check::Adjudicator { literal: true }, + }; + for governance in [false, true] { + for path in [ + Path::Batch, + Path::BatchHigh, + Path::Cached, + Path::Facts, + Path::Quotes, + Path::QuotesSplit, + ] { + for gold_same in [false, true] { + for reverse in [false, true] { + let org = Uuid::now_v7(); + let f = match Fx::new(&pool, &case, org).await { + Ok(f) => f, + Err(error) => { + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(org) + .execute(&pool) + .await?; + return Err(error); + } + }; + let run = tokio::time::timeout( + Duration::from_secs(15), + f.measure_decision(path, governance, gold_same, reverse), + ) + .await; + f.cleanup().await?; + let row = + run.map_err(|_| anyhow::anyhow!("decision measurement timed out"))??; + println!("NAMESAKE_DECISION_MEASUREMENT {row}"); + let id = row["id"].as_str().unwrap().to_string(); + let error = row["observed_same"].as_bool().unwrap() + != row["source_gold_same"].as_bool().unwrap(); + anyhow::ensure!(row["gold_error"] == error, "{id}: inconsistent gold error"); + if seen.contains(&id) { + mismatches.push(format!("{id}: duplicate row")); + } + if KNOWN_GOLD_ERRORS.contains(&id.as_str()) != error { + mismatches.push(format!( + "{id}: {}", + if error { + "unlisted gold error" + } else { + "unexpected pass: remove this name with its fix" + } + )); + } + seen.push(id); + rows.push(row); + } + } + } + } + // 名单必须被真实运行消费：删掉一行或改了名字，不能留下无人检查的旧豁免。 + for &name in KNOWN_GOLD_ERRORS { + if !seen.iter().any(|id| id.as_str() == name) { + mismatches.push(format!("{name}: unused known-gold-error name")); + } + } + if rows.len() != 48 { + mismatches.push(format!("expected 48 rows, got {}", rows.len())); + } + println!( + "NAMESAKE_DECISION_SUMMARY {}", + json!({"cases":rows.len(), + "source_gold_errors":rows.iter().filter(|r|r["gold_error"]==true).count(), + "scope":"named gold errors and evidence transport; scripted verdicts do not measure model accuracy", + "known_gold_errors":KNOWN_GOLD_ERRORS,"mismatches":mismatches, + "original_known_failure_list_unchanged":true}) + ); + anyhow::ensure!( + mismatches.is_empty(), + "namesake gold expectation mismatch: {}", + mismatches.join("; ") + ); + Ok(()) +} diff --git a/crates/utopia-server/src/namesake_scripted_cases.rs b/crates/utopia-server/src/namesake_scripted_cases.rs index b5be0f345..d26b7758b 100644 --- a/crates/utopia-server/src/namesake_scripted_cases.rs +++ b/crates/utopia-server/src/namesake_scripted_cases.rs @@ -166,10 +166,18 @@ async fn reply(State(m): State, Json(body): Json) -> axum::respons } let step = step.unwrap(); if !step.stream { - return Json( - json!({"choices":[{"message":{"role":"assistant","content":step.text}, - "finish_reason":"stop"}]}), - ) + // 工具回合要走真实调查传输；普通脚本仍是文本回复，内容里的 JSON 判决照旧传递。 + let message = serde_json::from_str::(&step.text) + .ok() + .filter(|v| v["tool_calls"].is_array()) + .unwrap_or_else(|| json!({"role":"assistant","content":step.text})); + let finish = if message["tool_calls"].is_array() { + "tool_calls" + } else { + "stop" + }; + return Json(json!({"choices":[{"message":message, + "finish_reason":finish}]})) .into_response(); } let frame = json!({"choices":[{"delta":{"content":step.text}}]}); @@ -181,6 +189,9 @@ async fn reply(State(m): State, Json(body): Json) -> axum::respons .into_response() } +#[path = "namesake_decision_measurements.rs"] +mod decision_measurements; + struct Fx { pool: sqlx::PgPool, state: AppState, diff --git a/crates/utopia-server/src/ontology_agent.rs b/crates/utopia-server/src/ontology_agent.rs index fe844de1d..6e7400b8a 100644 --- a/crates/utopia-server/src/ontology_agent.rs +++ b/crates/utopia-server/src/ontology_agent.rs @@ -91,7 +91,7 @@ pub async fn propose_with( tracing::info!(%kb_id, "本体代理已有一份在跑，这次跳过"); return Ok(()); } - let result = propose_locked(state, kb_id, &settings, &client, full_glossary).await; + let result = propose_locked(state, &kb, &settings, &client, full_glossary).await; let _ = sqlx::query("SELECT pg_advisory_unlock(hashtext('propose_ontology'), hashtext($1))") .bind(kb_id.to_string()) .execute(&mut *guard) @@ -116,11 +116,12 @@ struct Merged { async fn propose_locked( state: &AppState, - kb_id: Uuid, + kb: &utopia_core::models::KnowledgeBase, settings: &utopia_core::models::LlmSettings, client: &utopia_llm::LlmClient, full_glossary: bool, ) -> anyhow::Result<()> { + let kb_id = kb.id; let pool = &state.pool; let classes = utopia_store::graph::entity_types(pool, kb_id).await?; let props = utopia_store::ontology::relation_type_views(pool, kb_id).await?; @@ -430,7 +431,13 @@ async fn propose_locked( let sig_ids: HashSet = sig_batch.iter().map(|s| s.id).collect(); let word_ids: HashSet = word_batch.iter().map(|w| w.id).collect(); let glossary = glossary_for(b * SIGNATURES_PER_CALL..(b + 1) * SIGNATURES_PER_CALL); - let messages = agent::build_proposal_messages(sig_batch, word_batch, &glossary, &q_items); + let messages = agent::build_proposal_messages( + sig_batch, + word_batch, + &glossary, + &q_items, + &kb.ontology_lang, + ); let reply = match chat_retrying_rate_limits_at(state, settings, client, &messages, Some(0.0)).await { diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rs index 656a6a2d1..8aaa6b9aa 100644 --- a/crates/utopia-server/src/pipeline_tests.rs +++ b/crates/utopia-server/src/pipeline_tests.rs @@ -7,7 +7,7 @@ //! 1. **每个分块拿到自己正文的向量。** 40 条、三批、四路并发，批次乱序完成，读回来逐条对。 //! 附带：末尾不足一批的余数照样送，空文档一个请求都不发。 //! 2. **数量对不上整批放弃**：少回一条，那一批一条都不写。 -//! 3. **上限守得住**：十二批、每批延时 150ms，同一时刻在飞的从不超过 EMBED_JOBS， +//! 3. **上限守得住**：十二批、每批延时 400ms，同一时刻在飞的从不超过 EMBED_JOBS， //! 总耗时明显短于串行。 //! 4. **一批失败文档不悬着**：走完整的 process_document，第二批回 500，文档落在 failed //! 并带原因，不是停在 embedding。 @@ -306,7 +306,7 @@ async fn a_batch_that_answers_with_the_wrong_count_is_abandoned_whole() -> anyho #[tokio::test] async fn the_embedding_gate_is_never_held_beyond_its_ceiling() -> anyhow::Result<()> { - let delay = Duration::from_millis(150); + let delay = Duration::from_millis(400); let Some(f) = fixture(FakeEmbed::new(delay)).await? else { return Ok(()); }; @@ -336,9 +336,9 @@ async fn the_embedding_gate_is_never_held_beyond_its_ceiling() -> anyhow::Result ); assert!(peak >= 2, "batches actually overlap; peak was {peak}"); // 总耗时只用来兜底「闸门把批次完全串行化了」，真正守上限和重叠的是上面两条。 - // 这条不能卡得太紧：理想 450ms，可 CI 的 runner 只有 4 个 vCPU，同一进程里还有 - // 三百多个测试在并行，dev 上有一次跑到 1.03s——超过串行的一半就红了，而那次 - // 峰值并发完全正常。放到串行的四分之三：完全串行是 1.8s，仍能一眼分辨 + // 延时要盖过数据库写入的固定开销：150ms 在两核、proot PostgreSQL 上跑到 1.51s， + // 峰值和重叠都正常，却超过了 1.35s 门槛。400ms 让网络等待主导耗时；仍用串行 + // 的四分之三作上限（3.6s 对 4.8s），真正串行的回归仍会失败。 let serial = delay * batches as u32; assert!( elapsed < serial * 3 / 4, diff --git a/crates/utopia-server/src/pushed_keys.rs b/crates/utopia-server/src/pushed_keys.rs new file mode 100644 index 000000000..a4c449d4c --- /dev/null +++ b/crates/utopia-server/src/pushed_keys.rs @@ -0,0 +1,23 @@ +//! 推送来源（`api`、`statements`）的文档怎么按推送方给的 identity 存 `external_key`、 +//! 怎么从它取回来。推送的处理器、按 `external_id` 读回和检索都用这一份（#1271） + +/// 有推送密钥的来源：`api` 推文档，`statements` 推陈述（0054）。两种的密钥同一套 +/// 生成、查看、轮换 +pub fn is_push_kind(kind: &str) -> bool { + matches!(kind, "api" | "statements") +} + +/// The `external_key` a push source files a pushed identity under (`api:…`, `statements:…`). +/// The push handlers write it and the read by `external_id` (#1276) looks it up, so the two +/// cannot drift apart. +pub fn pushed_key(source: &utopia_core::models::Source, identity: &str) -> String { + format!("{}:{identity}", source.kind) +} + +/// `pushed_key` 倒过来：推送来源的文档交回推送方给的 identity，别的来源没有（#1271） +pub fn pushed_identity<'k>(kind: &str, key: &'k str) -> Option<&'k str> { + if !is_push_kind(kind) { + return None; + } + key.strip_prefix(kind)?.strip_prefix(':') +} diff --git a/crates/utopia-server/src/query_engine/mod.rs b/crates/utopia-server/src/query_engine/mod.rs index 532352ee2..e4ce6bb88 100644 --- a/crates/utopia-server/src/query_engine/mod.rs +++ b/crates/utopia-server/src/query_engine/mod.rs @@ -219,7 +219,7 @@ pub(crate) fn coerce(type_name: &str, raw: &serde_json::Value) -> serde_json::Va if let Ok(n) = s.parse::() { return n.into(); } - if let Ok(f) = s.parse::() { + if let Some(f) = s.parse::().ok().filter(|_| fits_f64(&ty, s)) { if let Some(n) = serde_json::Number::from_f64(f) { return serde_json::Value::Number(n); } @@ -235,6 +235,20 @@ pub(crate) fn coerce(type_name: &str, raw: &serde_json::Value) -> serde_json::Va raw.clone() } +/// 定点数（DECIMAL / NUMERIC / NUMBER / FIXED）有效数字超过 15 位时 f64 装不下， +/// 转成数就改了值：`12345678901234567.89` 会变成 `12345678901234568`。那样留字符串 +fn fits_f64(ty: &str, s: &str) -> bool { + if !["DECIMAL", "NUMERIC", "NUMBER", "FIXED"] + .iter() + .any(|k| ty.contains(k)) + { + return true; + } + let mantissa = s.split(['e', 'E']).next().unwrap_or_default(); + let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); + digits.trim_start_matches('0').trim_end_matches('0').len() <= 15 +} + /// 单引号字面量的转义：schema 名进 information_schema 的 WHERE 子句 pub(crate) fn sql_literal(s: &str) -> String { format!("'{}'", s.replace('\'', "''")) @@ -414,4 +428,25 @@ mod tests { assert_eq!(coerce("INTERVAL", &json!("1 day")), json!("1 day")); assert_eq!(coerce("DOUBLE", &json!(null)), json!(null)); } + + #[test] + fn an_exact_decimal_longer_than_a_double_stays_exact() { + assert_eq!( + coerce("DECIMAL(20,2)", &json!("12345678901234567.89")), + json!("12345678901234567.89") + ); + assert_eq!( + coerce("fixed", &json!("12345678901234567890")), + json!("12345678901234567890") + ); + assert_eq!(coerce("DECIMAL", &json!("1234.50")), json!(1234.5)); + assert_eq!( + coerce("fixed", &json!("100000000000000000000")), + json!(1e20) + ); + assert_eq!( + coerce("DOUBLE", &json!("0.30000000000000004")), + json!(0.30000000000000004) + ); + } } diff --git a/crates/utopia-server/src/query_engine/mysql.rs b/crates/utopia-server/src/query_engine/mysql.rs index df25f7d42..38ffc3928 100644 --- a/crates/utopia-server/src/query_engine/mysql.rs +++ b/crates/utopia-server/src/query_engine/mysql.rs @@ -471,6 +471,27 @@ mod tests { assert!(super::super::guard_sql_for("mysql", "DELETE FROM sales.orders").is_err()); } + /// 超出 f64 精度的 DECIMAL 留字符串，分毫不差；装得下的仍是数 + #[tokio::test] + async fn a_decimal_longer_than_a_double_keeps_every_digit_mysql() { + let Some(fx) = Fx::new(1).await else { return }; + let a = fx.schemas[0].clone(); + fx.exec(&format!( + "CREATE TABLE {a}.ledger (total DECIMAL(20,2), fee DECIMAL(10,2)); + INSERT INTO {a}.ledger VALUES (12345678901234567.89, 1234.50);" + )) + .await; + let r = MysqlEngine::new(&fx.url) + .execute(&format!("SELECT total, fee FROM {a}.ledger")) + .await + .expect("execute"); + assert_eq!( + r.rows, + vec![r#"{"total":"12345678901234567.89","fee":1234.5}"#] + ); + fx.cleanup().await; + } + /// YEAR 与 BIT 只有无符号整数读得出：字符串、字节都不收它们，落进兜底档就整列 null #[tokio::test] async fn a_year_or_bit_column_reads_as_its_number_mysql() { diff --git a/crates/utopia-server/src/query_engine/snowflake.rs b/crates/utopia-server/src/query_engine/snowflake.rs index dce0e707c..8972e42c5 100644 --- a/crates/utopia-server/src/query_engine/snowflake.rs +++ b/crates/utopia-server/src/query_engine/snowflake.rs @@ -54,6 +54,67 @@ struct PartitionInfo { row_count: u64, } +/// SQL API 把日期与时刻给成从纪元起的数：DATE 是天数（`18262`），TIME 与 +/// TIMESTAMP_* 是秒数带九位小数（`1616173619.000000000`），TIMESTAMP_TZ 再跟一个 +/// 空格和 1440 加时区分钟数。原样交出去，模型看到的只是一个大数。这里写回 ISO， +/// 其余类型照 `coerce` +fn cell(ty: &str, raw: &serde_json::Value) -> serde_json::Value { + use chrono::SecondsFormat::AutoSi; + let serde_json::Value::String(s) = raw else { + return raw.clone(); + }; + let ty = ty.to_ascii_lowercase(); + let text = match ty.as_str() { + "date" => s + .parse::() + .ok() + .and_then(|days| chrono::DateTime::from_timestamp(days.checked_mul(86_400)?, 0)) + .map(|t| t.date_naive().to_string()), + "time" => epoch(s) + .and_then(|(secs, nanos)| { + chrono::NaiveTime::from_num_seconds_from_midnight_opt( + u32::try_from(secs).ok()?, + nanos, + ) + }) + .map(|t| t.to_string()), + "timestamp_ntz" => epoch(s) + .and_then(|(secs, nanos)| chrono::DateTime::from_timestamp(secs, nanos)) + .map(|t| t.naive_utc().to_string()), + "timestamp_ltz" => epoch(s) + .and_then(|(secs, nanos)| chrono::DateTime::from_timestamp(secs, nanos)) + .map(|t| t.to_rfc3339_opts(AutoSi, true)), + "timestamp_tz" => s.split_once(' ').and_then(|(at, offset)| { + let minutes = offset.trim().parse::().ok()? - 1440; + let zone = chrono::FixedOffset::east_opt(minutes * 60)?; + let (secs, nanos) = epoch(at)?; + let t = chrono::DateTime::from_timestamp(secs, nanos)?; + Some(t.with_timezone(&zone).to_rfc3339_opts(AutoSi, false)) + }), + _ => return coerce(&ty, raw), + }; + text.map_or_else(|| raw.clone(), serde_json::Value::String) +} + +/// `-1.500000000` → (-2, 500000000)：秒向下取整，纳秒总是正的 +fn epoch(s: &str) -> Option<(i64, u32)> { + let (whole, frac) = s.split_once('.').unwrap_or((s, "")); + let secs = whole.parse::().ok()?; + if frac.len() > 9 || !frac.chars().all(|c| c.is_ascii_digit()) { + return None; + } + let nanos = if frac.is_empty() { + 0 + } else { + format!("{frac:0<9}").parse::().ok()? + }; + if nanos > 0 && whole.starts_with('-') { + Some((secs - 1, 1_000_000_000 - nanos)) + } else { + Some((secs, nanos)) + } +} + impl SnowflakeEngine { pub fn new(conn: SnowflakeConn) -> Self { Self { conn } @@ -141,7 +202,7 @@ impl SnowflakeEngine { .map(|row| { row.iter() .enumerate() - .map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v)) + .map(|(i, v)| cell(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v)) .collect() }) .collect(); @@ -168,7 +229,7 @@ impl SnowflakeEngine { rows.extend(chunk_rows.into_iter().map(|row| { row.iter() .enumerate() - .map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v)) + .map(|(i, v)| cell(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v)) .collect() })); } @@ -291,6 +352,43 @@ mod tests { assert_eq!(out.rows, vec![r#"{"REGION":"east","TOTAL":42.1}"#]); } + /// SQL API 的日期与时刻是从纪元起的数：DATE 是天，TIME 与 TIMESTAMP_* 是秒， + /// TIMESTAMP_TZ 后面再跟 1440 加上时区的分钟 + #[tokio::test] + async fn dates_and_times_read_as_dates_and_times() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/api/v2/statements")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "resultSetMetaData": { "numRows": 2, "rowType": [ + { "name": "D", "type": "date" }, + { "name": "T", "type": "time" }, + { "name": "NTZ", "type": "timestamp_ntz" }, + { "name": "LTZ", "type": "timestamp_ltz" }, + { "name": "TZ", "type": "timestamp_tz" } + ] }, + "data": [ + ["18262", "82919.000000000", "1616173619.000000000", "1616173619.500000000", "1616173619.000000000 1500"], + ["-1", "0.000000000", "-1.500000000", null, "1616173619.000000000 1080"] + ], + "code": "090001", + "statementHandle": "h1" + }))) + .mount(&server) + .await; + let out = SnowflakeEngine::new(conn(&server)) + .execute("SELECT d, t, ntz, ltz, tz FROM events") + .await + .unwrap(); + assert_eq!( + out.rows, + vec![ + r#"{"D":"2020-01-01","T":"23:01:59","NTZ":"2021-03-19 17:06:59","LTZ":"2021-03-19T17:06:59.500Z","TZ":"2021-03-19T18:06:59+01:00"}"#, + r#"{"D":"1969-12-31","T":"00:00:00","NTZ":"1969-12-31 23:59:58.500","LTZ":null,"TZ":"2021-03-19T11:06:59-06:00"}"#, + ] + ); + } + #[tokio::test] async fn a_202_is_polled_and_its_partitions_keep_the_first_rows_types() { let server = MockServer::start().await; diff --git a/crates/utopia-server/src/query_engine/trino.rs b/crates/utopia-server/src/query_engine/trino.rs index 771a86973..d6c484a92 100644 --- a/crates/utopia-server/src/query_engine/trino.rs +++ b/crates/utopia-server/src/query_engine/trino.rs @@ -7,7 +7,7 @@ use super::conn::TrinoConn; use super::{ - rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult, + coerce, rows_to_json_lines, sql_literal, truncate_rows, wrap_limit, QueryEngine, QueryResult, SchemaColumn, HTTP_POLL_BUDGET, STATEMENT_TIMEOUT_SECS, }; use base64::Engine as _; @@ -22,6 +22,8 @@ pub struct TrinoEngine { #[derive(Deserialize)] struct Column { name: String, + #[serde(rename = "type", default)] + ty: String, } #[derive(Deserialize)] @@ -86,7 +88,7 @@ impl TrinoEngine { .json() .await?; let started = Instant::now(); - let mut columns: Option> = None; + let mut columns: Option> = None; let mut rows = Vec::new(); loop { if let Some(e) = page.error { @@ -94,10 +96,7 @@ impl TrinoEngine { anyhow::bail!("{name}{}", e.message); } if columns.is_none() { - columns = page - .columns - .take() - .map(|cs| cs.into_iter().map(|c| c.name).collect()); + columns = page.columns.take(); } if let Some(d) = page.data.take() { rows.extend(d); @@ -120,7 +119,16 @@ impl TrinoEngine { .json() .await?; } - Ok((columns.unwrap_or_default(), rows)) + let columns = columns.unwrap_or_default(); + // 协议把 decimal 写成字符串（其余的数都是 JSON 数），按列类型还原 + for row in &mut rows { + for (v, c) in row.iter_mut().zip(&columns) { + if c.ty.starts_with("decimal") { + *v = coerce("DECIMAL", v); + } + } + } + Ok((columns.into_iter().map(|c| c.name).collect(), rows)) } } @@ -311,6 +319,45 @@ mod tests { assert!(!out.truncated); } + /// Trino 的 REST 协议把 decimal 写成字符串（`SqlDecimal` 走 `writeString`）， + /// 别的数是 JSON 数 + #[tokio::test] + async fn a_decimal_reads_as_a_number() { + let server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/v1/statement")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({ + "id": "q3", + "columns": [ + { "name": "region", "type": "varchar(5)" }, + { "name": "total", "type": "decimal(12,2)" }, + { "name": "orders", "type": "bigint" } + ], + "data": [ ["east", "1234.50", 7], ["west", null, 0] ], + "stats": { "state": "FINISHED" } + }))) + .mount(&server) + .await; + let conn = TrinoConn::parse(&format!( + "trino://alice@{}/hive?ssl=false", + server.uri().trim_start_matches("http://") + )) + .unwrap(); + let out = TrinoEngine::new(conn) + .execute( + "SELECT region, SUM(amount) AS total, COUNT(*) AS orders FROM orders GROUP BY 1", + ) + .await + .unwrap(); + assert_eq!( + out.rows, + vec![ + r#"{"region":"east","total":1234.5,"orders":7}"#, + r#"{"region":"west","total":null,"orders":0}"# + ] + ); + } + #[tokio::test] async fn a_trino_error_page_becomes_an_error() { let server = MockServer::start().await; diff --git a/crates/utopia-server/src/retrieval.rs b/crates/utopia-server/src/retrieval.rs index 51c377fe6..769681cb9 100644 --- a/crates/utopia-server/src/retrieval.rs +++ b/crates/utopia-server/src/retrieval.rs @@ -31,9 +31,14 @@ pub struct SearchOpts { pub top_k: usize, pub offset: usize, pub as_of: Option>, - /// 只在这些文档里检索。两路各自在召回**之内**过滤：每路只取前 `recall` 条， - /// 召回完再筛，要的文档早被库里别的东西挤出名额了。None 即全库 - pub documents: Option>, + pub filter: Option, +} + +/// 只检索一个来源的文档，给了 `key_prefix` 就只要 `external_key` 以它开头的（#1271）。 +/// 两路各在召回**之内**过滤：融合完再筛，库里别的命中早占满了每路的名额，页就短了 +pub struct SourceFilter { + pub source_id: Uuid, + pub key_prefix: Option, } /// 一条命中连同它的分数（#1271）。`score` 是 RRF 总分，只在同一次查询里能排先后， @@ -42,6 +47,8 @@ pub struct SearchOpts { pub struct ScoredChunk { #[serde(flatten)] pub chunk: SourcedChunk, + /// 推送来源的文档才有：推送方给的那个 id，好让它拿回去对自己的记录 + pub external_id: Option, pub score: f64, /// 来自一个会衰减的源、且久没被检索：分已经乘过 [`utopia_store::decay::STALE_SEARCH_WEIGHT`]。 /// 不衰减的库这一格永远是 false，不进响应 @@ -68,19 +75,16 @@ pub async fn hybrid( as_of: Option>, embeddings: Option<&mut llm_util::EmbedCache>, ) -> AppResult> { - let opts = SearchOpts { - top_k, - offset: 0, - as_of, - documents: None, - }; let (hits, _) = fused_hits( state, kb_id, workspace_id, query, - RECALL_PER_CHANNEL, - &opts, + Recall { + per_channel: RECALL_PER_CHANNEL, + as_of, + documents: None, + }, embeddings, ) .await?; @@ -102,20 +106,29 @@ pub async fn hybrid_scored( opts: SearchOpts, embeddings: Option<&mut llm_util::EmbedCache>, ) -> AppResult { - if opts.documents.as_ref().is_some_and(Vec::is_empty) { - return Ok(SearchPage { - results: Vec::new(), - has_more: false, - vector_used: false, - }); - } + let documents = match &opts.filter { + Some(filter) => Some( + utopia_store::documents::ids_in_source( + &state.pool, + kb_id, + filter.source_id, + filter.key_prefix.as_deref(), + opts.as_of, + ) + .await?, + ), + None => None, + }; let (hits, vector_used) = fused_hits( state, kb_id, workspace_id, query, - MAX_DEPTH, - &opts, + Recall { + per_channel: MAX_DEPTH, + as_of: opts.as_of, + documents: documents.as_deref(), + }, embeddings, ) .await?; @@ -133,41 +146,38 @@ pub async fn hybrid_scored( }) } -/// 两路各取 `per_channel` 条、RRF 融合、按记录轴过滤、陈旧的降过分的整张表，连同向量那一路跑了没有 +/// 每一路怎么召回：取几条、按哪一刻、只在哪些文档里（None 即全库） +#[derive(Clone, Copy)] +struct Recall<'a> { + per_channel: usize, + as_of: Option>, + documents: Option<&'a [Uuid]>, +} + +/// 两路按 `recall` 各取一份、RRF 融合、按记录轴过滤后的整张表，连同向量那一路跑了没有 async fn fused_hits( state: &AppState, kb_id: Uuid, workspace_id: Uuid, query: &str, - per_channel: usize, - opts: &SearchOpts, + recall: Recall<'_>, embeddings: Option<&mut llm_util::EmbedCache>, ) -> AppResult<(Vec, bool)> { let bm25 = { let search = state.search.clone(); let kb = kb_id.to_string(); let query = query.to_string(); - let documents: Option> = opts + let documents: Option> = recall .documents - .as_ref() .map(|ids| ids.iter().map(Uuid::to_string).collect()); - tokio::task::spawn_blocking(move || { - search.search(&kb, &query, per_channel, documents.as_deref()) - }) + let limit = recall.per_channel; + tokio::task::spawn_blocking(move || search.search(&kb, &query, limit, documents.as_deref())) }; // `join!` 不是 `try_join!`：向量那一路失败是降级不是错误（见 vector_channel）， // `try_join!` 会把模型故障变成检索报错 let (bm25, vector) = tokio::join!( bm25, - vector_channel( - state, - kb_id, - workspace_id, - query, - per_channel, - opts, - embeddings - ) + vector_channel(state, kb_id, workspace_id, query, recall, embeddings) ); let bm25 = bm25 .map_err(|e| utopia_core::AppError::Other(anyhow::anyhow!("BM25 检索线程退出：{e}")))? @@ -185,14 +195,22 @@ async fn fused_hits( .filter_map(|f| Some((f.id.parse().ok()?, f))) .collect(); let ids: Vec = fused.iter().map(|(id, _)| *id).collect(); - let live = utopia_store::documents::chunks_by_ids(&state.pool, kb_id, &ids, opts.as_of).await?; + let live = + utopia_store::documents::chunks_by_ids(&state.pool, kb_id, &ids, recall.as_of).await?; let mut fused: HashMap = fused.into_iter().collect(); let mut hits: Vec = live .into_iter() .filter_map(|chunk| { let f = fused.remove(&chunk.chunk.id)?; + let external_id = chunk + .source_kind + .as_deref() + .zip(chunk.external_key.as_deref()) + .and_then(|(kind, key)| crate::pushed_keys::pushed_identity(kind, key)) + .map(str::to_string); Some(ScoredChunk { chunk, + external_id, score: f.score, stale: false, bm25_rank: position(&f.ranks, 0), @@ -255,8 +273,7 @@ async fn vector_channel( kb_id: Uuid, workspace_id: Uuid, query: &str, - per_channel: usize, - opts: &SearchOpts, + recall: Recall<'_>, cache: Option<&mut llm_util::EmbedCache>, ) -> AppResult>> { let vector = match cache.as_deref().and_then(|c| c.get(query)).cloned() { @@ -287,9 +304,9 @@ async fn vector_channel( &state.pool, kb_id, &vector, - per_channel as i64, - opts.as_of, - opts.documents.as_deref(), + recall.per_channel as i64, + recall.as_of, + recall.documents, ) .await?; Ok(Some(ids.into_iter().map(|id| id.to_string()).collect())) diff --git a/crates/utopia-server/src/retrieval_tests.rs b/crates/utopia-server/src/retrieval_tests.rs index 7e71f8317..a50c541d6 100644 --- a/crates/utopia-server/src/retrieval_tests.rs +++ b/crates/utopia-server/src/retrieval_tests.rs @@ -127,7 +127,7 @@ impl Fixture { top_k: 8, offset: 0, as_of: None, - documents: None, + filter: None, }; Ok( super::hybrid_scored(&self.state, self.kb, self.ws, "apple harvest", opts, None) @@ -207,7 +207,7 @@ async fn both_channels_fuse_with_bm25_first() -> anyhow::Result<()> { top_k: 8, offset: 0, as_of: None, - documents: None, + filter: None, }; let page = super::hybrid_scored(&f.state, f.kb, f.ws, "apple harvest", opts, None).await?; let hits: Vec<_> = page @@ -219,7 +219,7 @@ async fn both_channels_fuse_with_bm25_first() -> anyhow::Result<()> { h.bm25_rank, h.vector_rank, h.chunk.source_id, - h.chunk.external_id.as_deref(), + h.external_id.as_deref(), ) }) .collect(); @@ -325,8 +325,10 @@ fn hit(document_id: Uuid, score: f64) -> super::ScoredChunk { filename: String::new(), }, source_id: None, - external_id: None, + source_kind: None, + external_key: None, }, + external_id: None, score, stale: false, bm25_rank: None, @@ -439,7 +441,7 @@ fn page(offset: usize, top_k: usize) -> super::SearchOpts { top_k, offset, as_of: None, - documents: None, + filter: None, } } @@ -486,3 +488,80 @@ async fn a_page_ending_at_depth_100_of_101_hits_has_no_more() -> anyhow::Result< assert_eq!((last.results.len(), last.has_more), (50, false)); Ok(()) } + +/// A source filter works inside both channels (#1271): a hundred better hits from +/// elsewhere fill each channel's candidates, so filtering after fusion would leave +/// nothing, and the six weak hits of the source must still page like any other list. +#[tokio::test] +async fn a_source_filter_pages_through_its_six_hits_under_a_hundred_better_ones( +) -> anyhow::Result<()> { + let Some(f) = fixture().await? else { + return Ok(()); + }; + let _server = seed_ranked_chunks(&f, 100).await?; + let (source, doc) = (Uuid::now_v7(), Uuid::now_v7()); + sqlx::raw_sql(&format!( + "INSERT INTO sources(id,kb_id,kind,name) VALUES ('{source}','{kb}','api','ci'); + INSERT INTO documents(id,kb_id,source_id,external_key,filename,sha256) + VALUES ('{doc}','{kb}','{source}','api:weak','weak.md','w')", + kb = f.kb + )) + .execute(&f.pool) + .await?; + let mut weak = Vec::new(); + for seq in 0..6 { + let id = Uuid::now_v7(); + let text = format!("apple {}", "pear ".repeat(200 + seq)); + sqlx::query( + "INSERT INTO chunks(id,kb_id,document_id,seq,text,embedding) + VALUES($1,$2,$3,$4,$5,$6::text::vector)", + ) + .bind(id) + .bind(f.kb) + .bind(doc) + .bind(seq as i32) + .bind(&text) + .bind(format!("[{},100,0,0]", seq + 1)) + .execute(&f.pool) + .await?; + weak.push((id.to_string(), text)); + } + f.state + .search + .reindex_document(&f.kb.to_string(), &doc.to_string(), &weak)?; + let walk = |offset, top_k| { + let opts = super::SearchOpts { + filter: Some(super::SourceFilter { + source_id: source, + key_prefix: None, + }), + ..page(offset, top_k) + }; + super::hybrid_scored(&f.state, f.kb, f.ws, "apple", opts, None) + }; + let (first, second, whole) = (walk(0, 3).await, walk(3, 3).await, walk(0, 10).await); + f.cleanup().await?; + let (first, second, whole) = (first?, second?, whole?); + let ids = + |hits: &[super::ScoredChunk]| hits.iter().map(|h| h.chunk.chunk.id).collect::>(); + let mut found: Vec<_> = ids(&whole.results).iter().map(Uuid::to_string).collect(); + found.sort(); + let mut expected: Vec<_> = weak.into_iter().map(|(id, _)| id).collect(); + expected.sort(); + assert_eq!(found, expected); + let channels: Vec<_> = whole + .results + .iter() + .map(|h| (h.bm25_rank.is_some(), h.vector_rank.is_some())) + .collect(); + assert_eq!(channels, [(true, true); 6]); + assert_eq!( + [ids(&first.results), ids(&second.results)].concat(), + ids(&whole.results) + ); + assert_eq!( + (first.has_more, second.has_more, whole.has_more), + (true, false, false) + ); + Ok(()) +} diff --git a/crates/utopia-server/src/timestamp_checkpoint_tests.rs b/crates/utopia-server/src/timestamp_checkpoint_tests.rs new file mode 100644 index 000000000..f97ae4561 --- /dev/null +++ b/crates/utopia-server/src/timestamp_checkpoint_tests.rs @@ -0,0 +1,495 @@ +//! ADR 0072: persisted progress describes complete source timestamp groups. +use super::{sync_github_issues_with_client, sync_source}; +use chrono::{DateTime, Utc}; +use serde_json::{json, Value}; +use std::sync::{Arc, Mutex}; +use utopia_store::sources; +use uuid::Uuid; +use wiremock::{matchers::method, Mock, MockServer, Request, ResponseTemplate}; + +const FIRST: &str = "2026-01-02T12:34:20Z"; +const NEXT: &str = "2026-01-02T12:34:40Z"; + +fn time(value: &str) -> DateTime { + value.parse().unwrap() +} + +#[derive(Clone, Copy)] +enum Provider { + Github, + Jira, +} + +impl Provider { + fn kind(self) -> &'static str { + match self { + Self::Github => "github_issues", + Self::Jira => "jira_issues", + } + } + + fn row(self, number: usize, updated: &str) -> Value { + match self { + Self::Github => json!({"number":number, "title":format!("Issue {number}"), + "state":"open", "body":format!("Body {number}"), + "created_at":"2026-01-01T00:00:00Z", "updated_at":updated}), + Self::Jira => json!({"key":format!("TEST-{number}"), "fields":{ + "summary":format!("Issue {number}"), "description":format!("Body {number}"), + "created":"2026-01-01T00:00:00Z", "updated":updated}}), + } + } +} + +#[derive(Default)] +struct Feed { + rows: Vec, + fail_page: Option, + oversized_discussion: Option<(&'static str, usize)>, + fail_detail_page: Option, +} + +struct Fixture { + provider: Provider, + state: crate::state::AppState, + source_id: Uuid, + org_id: Uuid, + server: MockServer, + feed: Arc>, + _directory: tempfile::TempDir, +} + +impl Fixture { + async fn new(provider: Provider) -> anyhow::Result> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(None); + }; + let pool = sqlx::PgPool::connect(&url).await?; + let (org_id, workspace_id, kb_id, source_id) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::query("INSERT INTO organizations(id,name) VALUES($1,'timestamp-test')") + .bind(org_id) + .execute(&pool) + .await?; + sqlx::query("INSERT INTO workspaces(id,org_id,name) VALUES($1,$2,'timestamp-test')") + .bind(workspace_id) + .bind(org_id) + .execute(&pool) + .await?; + sqlx::query( + "INSERT INTO knowledge_bases(id,workspace_id,name) VALUES($1,$2,'timestamp-test')", + ) + .bind(kb_id) + .bind(workspace_id) + .execute(&pool) + .await?; + let server = MockServer::start().await; + let config = match provider { + Provider::Github => json!({"repo":"test/repo"}), + Provider::Jira => json!({"base_url":server.uri(), "project":"TEST"}), + }; + sqlx::query( + "INSERT INTO sources(id,kb_id,kind,name,config) VALUES($1,$2,$3,'timestamp-test',$4)", + ) + .bind(source_id) + .bind(kb_id) + .bind(provider.kind()) + .bind(config) + .execute(&pool) + .await?; + let directory = tempfile::tempdir()?; + let config = utopia_core::config::AppConfig { + data_dir: directory.path().to_string_lossy().into_owned(), + ..Default::default() + }; + let search = Arc::new(utopia_search::SearchIndex::open( + &directory.path().join("search"), + )?); + let state = crate::state::AppState::new(pool, &config, search, "test-only".into()); + let feed = Arc::new(Mutex::new(Feed::default())); + let current = feed.clone(); + Mock::given(method("GET")).respond_with(move |request: &Request| { + let feed = current.lock().unwrap(); + let query_number = |key: &str, default: usize| request.url.query_pairs() + .find(|(name, _)| name == key) + .map(|(_, value)| value.parse::().unwrap()).unwrap_or(default); + let path = request.url.path(); + if path.ends_with("/comments") || path.ends_with("/events") { + if let Some((kind, count)) = feed.oversized_discussion { + if path.ends_with(kind) && path.contains("/issues/1/") { + let page = query_number("page", 1); + if feed.fail_detail_page == Some(page) { + return ResponseTemplate::new(503); + } + let rows: Vec<_> = (1..=count).skip((page - 1) * 100).take(100) + .map(|number| if kind == "comments" { + json!({"issue_url":"https://api.github.com/repos/test/repo/issues/1", + "created_at":FIRST, "body":format!("Discussion item {number}")}) + } else { + json!({"event":"closed", "created_at":FIRST, + "actor":{"login":format!("Discussion item {number}")}}) + }).collect(); + return ResponseTemplate::new(200).set_body_json(rows); + } + } + return ResponseTemplate::new(200).set_body_json(json!([])); + } + let (offset, limit) = match provider { + Provider::Github => ((query_number("page", 1) - 1) * 100, 100), + Provider::Jira => (query_number("startAt", 0), query_number("maxResults", 50)), + }; + if feed.fail_page == Some(offset / limit + 1) { + return ResponseTemplate::new(503); + } + // Return the consumed prefix deliberately: provider bounds are inclusive/rounded. + let rows: Vec<_> = feed.rows.iter().skip(offset).take(limit).cloned().collect(); + let body = match provider { + Provider::Github => json!(rows), + Provider::Jira => json!({"issues":rows,"total":feed.rows.len(),"startAt":offset,"maxResults":limit}), + }; + ResponseTemplate::new(200).set_body_json(body) + }).mount(&server).await; + Ok(Some(Self { + provider, + state, + source_id, + org_id, + server, + feed, + _directory: directory, + })) + } + + fn set_rows(&self, first_count: usize, next_count: usize) { + self.feed.lock().unwrap().rows = (1..=first_count + next_count) + .map(|number| { + self.provider + .row(number, if number <= first_count { FIRST } else { NEXT }) + }) + .collect(); + } + + async fn sync(&self) -> anyhow::Result<()> { + if matches!(self.provider, Provider::Jira) { + sync_source(&self.state, self.source_id).await?; + return Ok(()); + } + let previous = sources::sync_checkpoint(&self.state.pool, self.source_id).await?; + let source = sources::get(&self.state.pool, self.source_id).await?; + let run = sources::start_run(&self.state.pool, self.source_id).await?; + let http = reqwest::Client::builder().no_proxy().build()?; + let stats = sync_github_issues_with_client( + &self.state, + &source, + previous, + &http, + &self.server.uri(), + ) + .await?; + sources::finish_timestamp_sync( + &self.state.pool, + run, + self.source_id, + previous, + stats.checkpoint, + stats.created as i32, + stats.updated as i32, + ) + .await?; + Ok(()) + } + + async fn checkpoint(&self) -> anyhow::Result>> { + Ok(sources::sync_checkpoint(&self.state.pool, self.source_id).await?) + } + + async fn counts(&self) -> anyhow::Result<(i64, i64, i64)> { + Ok(sqlx::query_as("SELECT + (SELECT count(*) FROM documents WHERE source_id=$1), + (SELECT count(*) FROM document_versions v JOIN documents d ON d.id=v.document_id WHERE d.source_id=$1), + (SELECT count(*) FROM jobs j JOIN documents d ON d.id::text=j.payload->>'document_id' WHERE d.source_id=$1)") + .bind(self.source_id).fetch_one(&self.state.pool).await?) + } + + async fn cleanup(self) -> anyhow::Result<()> { + sqlx::query("DELETE FROM jobs WHERE payload->>'document_id' IN (SELECT id::text FROM documents WHERE source_id=$1)") + .bind(self.source_id).execute(&self.state.pool).await?; + sqlx::query("DELETE FROM organizations WHERE id=$1") + .bind(self.org_id) + .execute(&self.state.pool) + .await?; + Ok(()) + } +} + +async fn large_group_progress(provider: Provider, group_size: usize) -> anyhow::Result<()> { + let Some(fixture) = Fixture::new(provider).await? else { + return Ok(()); + }; + fixture.set_rows(group_size, 1); + fixture.sync().await?; + assert_eq!(fixture.counts().await?.0, group_size as i64); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + fixture.sync().await?; + let counts = fixture.counts().await?; + assert_eq!( + counts, + ( + (group_size + 1) as i64, + (group_size + 1) as i64, + (group_size + 1) as i64 + ) + ); + assert_eq!(fixture.checkpoint().await?, Some(time(NEXT))); + fixture.sync().await?; + assert_eq!(fixture.counts().await?, counts); + assert_eq!(fixture.checkpoint().await?, Some(time(NEXT))); + let requests = fixture.server.received_requests().await.unwrap(); + let incremental = requests.iter().any(|request| match provider { + Provider::Github => request.url.query_pairs().any(|(key, value)| { + key == "since" + && (time(&value) == time(FIRST) + || time(&value) == time(FIRST) + chrono::Duration::seconds(1)) + }), + Provider::Jira => request + .url + .query_pairs() + .any(|(key, value)| key == "jql" && value.contains("2026-01-02 12:34")), + }); + assert!( + incremental, + "the next run must send the persisted source bound" + ); + fixture.cleanup().await +} + +#[tokio::test] +async fn github_imports_1001_equal_time_items_then_skips_the_old_prefix() -> anyhow::Result<()> { + large_group_progress(Provider::Github, 1001).await +} + +#[tokio::test] +async fn jira_imports_501_equal_time_items_then_skips_the_rounded_prefix() -> anyhow::Result<()> { + large_group_progress(Provider::Jira, 501).await +} + +#[tokio::test] +async fn processing_budget_finishes_item_240_and_leaves_the_next_group() -> anyhow::Result<()> { + for provider in [Provider::Github, Provider::Jira] { + large_group_progress(provider, 240).await?; + } + Ok(()) +} + +#[tokio::test] +async fn failed_listing_does_not_advance_and_retry_deduplicates() -> anyhow::Result<()> { + for provider in [Provider::Github, Provider::Jira] { + let Some(fixture) = Fixture::new(provider).await? else { + return Ok(()); + }; + fixture.set_rows(240, 0); + fixture.feed.lock().unwrap().fail_page = Some(2); + assert!(fixture.sync().await.is_err()); + assert_eq!(fixture.checkpoint().await?, None); + fixture.feed.lock().unwrap().fail_page = None; + fixture.sync().await?; + assert_eq!(fixture.counts().await?, (240, 240, 240)); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + fixture.cleanup().await?; + } + Ok(()) +} + +#[tokio::test] +async fn github_failed_discussion_tail_preserves_the_document_and_recovers() -> anyhow::Result<()> { + for (endpoint, count) in [("comments", 1001), ("events", 1100)] { + let Some(fixture) = Fixture::new(Provider::Github).await? else { + return Ok(()); + }; + fixture.set_rows(1, 0); + fixture.sync().await?; + let original: String = + sqlx::query_scalar("SELECT sha256 FROM documents WHERE source_id=$1") + .bind(fixture.source_id) + .fetch_one(&fixture.state.pool) + .await?; + { + let mut feed = fixture.feed.lock().unwrap(); + feed.rows = vec![Provider::Github.row(1, NEXT), Provider::Github.row(2, NEXT)]; + feed.oversized_discussion = Some((endpoint, count)); + feed.fail_detail_page = Some(11); + } + assert!( + fixture.sync().await.is_err(), + "failed {endpoint} tail must not replace a document" + ); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + let current: String = sqlx::query_scalar("SELECT sha256 FROM documents WHERE source_id=$1") + .bind(fixture.source_id) + .fetch_one(&fixture.state.pool) + .await?; + assert_eq!(current, original); + assert_eq!(fixture.counts().await?, (1, 1, 1)); + fixture.feed.lock().unwrap().fail_detail_page = None; + fixture.sync().await?; + assert_eq!(fixture.checkpoint().await?, Some(time(NEXT))); + assert_eq!(fixture.counts().await?, (2, 3, 3)); + let updated: String = sqlx::query_scalar( + "SELECT sha256 FROM documents WHERE source_id=$1 AND external_key='github:test/repo#1'", + ) + .bind(fixture.source_id) + .fetch_one(&fixture.state.pool) + .await?; + let body = String::from_utf8(fixture.state.blob.get(&updated).await?)?; + assert!(body.contains(&format!("Discussion item {count}"))); + fixture.sync().await?; + assert_eq!(fixture.counts().await?, (2, 3, 3)); + fixture.cleanup().await?; + } + Ok(()) +} + +#[tokio::test] +async fn recovery_ignores_old_success_and_retains_history_after_audit_pruning() -> anyhow::Result<()> +{ + for provider in [Provider::Github, Provider::Jira] { + let Some(fixture) = Fixture::new(provider).await? else { + return Ok(()); + }; + fixture.set_rows(1, 0); + fixture.sync().await?; + let before = fixture.counts().await?; + sqlx::query("UPDATE sources SET sync_checkpoint=NULL WHERE id=$1") + .bind(fixture.source_id) + .execute(&fixture.state.pool) + .await?; + assert_eq!(fixture.checkpoint().await?, None); + fixture.sync().await?; + assert_eq!( + fixture.counts().await?, + before, + "recovery must retain versions and jobs" + ); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + for _ in 0..51 { + let run = sources::start_run(&fixture.state.pool, fixture.source_id).await?; + sources::finish_run( + &fixture.state.pool, + run, + fixture.source_id, + Some("fixture failure"), + 0, + 0, + ) + .await?; + } + assert_eq!( + sources::list_runs(&fixture.state.pool, fixture.source_id, 100) + .await? + .len(), + 50 + ); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + fixture.feed.lock().unwrap().rows.clear(); + fixture.sync().await?; + assert_eq!( + fixture.checkpoint().await?, + Some(time(FIRST)), + "empty reads must not advance to wall clock" + ); + assert_eq!(fixture.counts().await?, before); + fixture.cleanup().await?; + } + Ok(()) +} + +#[tokio::test] +async fn checkpoint_and_run_success_commit_atomically() -> anyhow::Result<()> { + let Some(fixture) = Fixture::new(Provider::Github).await? else { + return Ok(()); + }; + let pool = &fixture.state.pool; + assert!(sources::finish_timestamp_sync( + pool, + Uuid::now_v7(), + fixture.source_id, + None, + Some(time(FIRST)), + 1, + 0 + ) + .await + .is_err()); + assert_eq!( + fixture.checkpoint().await?, + None, + "missing run must roll back source checkpoint" + ); + let run = sources::start_run(pool, fixture.source_id).await?; + sources::finish_timestamp_sync(pool, run, fixture.source_id, None, Some(time(FIRST)), 1, 0) + .await?; + let stale_run = sources::start_run(pool, fixture.source_id).await?; + assert!(sources::finish_timestamp_sync( + pool, + stale_run, + fixture.source_id, + None, + Some(time(NEXT)), + 1, + 0 + ) + .await + .is_err()); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + let status: String = sqlx::query_scalar("SELECT status FROM source_sync_runs WHERE id=$1") + .bind(stale_run) + .fetch_one(pool) + .await?; + assert_eq!( + status, "running", + "checkpoint conflict must not mark the stale run successful" + ); + fixture.cleanup().await +} + +#[tokio::test] +async fn a_document_write_failure_keeps_the_checkpoint_and_retry_deduplicates() -> anyhow::Result<()> +{ + for provider in [Provider::Github, Provider::Jira] { + let Some(fixture) = Fixture::new(provider).await? else { + return Ok(()); + }; + fixture.set_rows(3, 0); + // The constraint affects only this fixture's second document, including on shared test databases. + let constraint = format!("checkpoint_write_failure_{}", fixture.source_id.simple()); + let second_key = match provider { + Provider::Github => "github:test/repo#2".to_owned(), + Provider::Jira => "jira:127.0.0.1/TEST-2".to_owned(), + }; + sqlx::query(&format!("ALTER TABLE documents ADD CONSTRAINT {constraint} CHECK \ + (source_id IS DISTINCT FROM '{}'::uuid OR external_key IS DISTINCT FROM '{}') NOT VALID", + fixture.source_id, second_key)) + .execute(&fixture.state.pool).await?; + let failed = fixture.sync().await; + sqlx::query(&format!( + "ALTER TABLE documents DROP CONSTRAINT {constraint}" + )) + .execute(&fixture.state.pool) + .await?; + assert!(failed.is_err(), "the second document write must fail"); + assert_eq!(fixture.checkpoint().await?, None); + assert_eq!( + fixture.counts().await?, + (1, 1, 1), + "the first item was committed before the failure" + ); + fixture.sync().await?; + assert_eq!(fixture.counts().await?, (3, 3, 3)); + assert_eq!(fixture.checkpoint().await?, Some(time(FIRST))); + fixture.cleanup().await?; + } + Ok(()) +} diff --git a/crates/utopia-store/src/audit.rs b/crates/utopia-store/src/audit.rs index 64e6532aa..0692d913f 100644 --- a/crates/utopia-store/src/audit.rs +++ b/crates/utopia-store/src/audit.rs @@ -1,5 +1,5 @@ //! 审计日志：谁在何时对什么做了什么。纯审计——只记录与展示，不承载回滚等衍生功能。 -//! 记录失败绝不影响业务操作（调用方一律 `let _ =`）。 +//! 普通事件由调用方尽力记录；审核裁决用 `record_on` 与图修改一起提交。 use sqlx::PgPool; use utopia_core::models::AuditEventView; @@ -68,17 +68,40 @@ pub async fn record_opt( target_kind: &str, target_id: Option, detail: serde_json::Value, +) -> AppResult<()> { + let mut conn = pool.acquire().await?; + record_on( + &mut conn, + kb_id, + actor_id, + action, + target_kind, + target_id, + detail, + ) + .await +} + +/// Reviewed graph changes must not commit without their actor's decision. +pub async fn record_on( + conn: &mut sqlx::PgConnection, + kb_id: Option, + actor_id: Option, + action: &str, + target_kind: &str, + target_id: Option, + detail: serde_json::Value, ) -> AppResult<()> { let ctx = client_context(); // 身份快照：用户日后被删除时（0025 之后行会留下），台账仍认得出是谁，而不是 // 只剩一串 UUID。此刻查是可靠的——操作正在发生，人还在。 let actor_label: Option = match actor_id { - Some(id) => sqlx::query_scalar("SELECT email FROM users WHERE id = $1") - .bind(id) - .fetch_optional(pool) - .await - .ok() - .flatten(), + Some(id) => { + sqlx::query_scalar("SELECT email FROM users WHERE id = $1") + .bind(id) + .fetch_optional(&mut *conn) + .await? + } None => None, }; sqlx::query( @@ -97,7 +120,7 @@ pub async fn record_opt( .bind(ctx.ip) .bind(ctx.user_agent) .bind(actor_label) - .execute(pool) + .execute(&mut *conn) .await?; Ok(()) } diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs index 81d314a20..c9a4434e2 100644 --- a/crates/utopia-store/src/documents.rs +++ b/crates/utopia-store/src/documents.rs @@ -1896,8 +1896,8 @@ async fn set_embeddings_for_input( /// 只是大致按距离，而距离并列时精确路径和 HNSW 各排各的——同一问两种计划回的 /// 居首不同（#652）。并列由 id 定 /// -/// `documents`：只在这些文档的块里找。条件写在近邻 CTE 里面，`LIMIT` 数的就只是 -/// 允许的块——放到外层，库里别的块会先占满名额 +/// `documents`：只在这些文档的块里找。条件在近邻 CTE 里面，`LIMIT` 数的就只是 +/// 允许的块；放到外层，库里别的块会先占满名额 pub async fn vector_search( pool: &PgPool, kb_id: Uuid, @@ -1944,27 +1944,24 @@ pub async fn vector_search( Ok(rows.into_iter().map(|(id,)| id).collect()) } -/// 检索的文档过滤：库里某个来源当时活着的文档，`prefixes` 给了就只要 -/// `external_id` 以其中之一开头的。`external_key` 是 `{kind}:{external_id}`， -/// 比前缀用 `starts_with`——路径里的 `_` 和 `%` 进了 LIKE 就成了通配符 +/// 检索按来源过滤时允许的文档：库里这个来源当时活着的文档，给了 `key_prefix` 就只要 +/// `external_key` 以它开头的。比较用 `starts_with`：路径里的 `_` 和 `%` 进了 LIKE 就是通配符 pub async fn ids_in_source( pool: &PgPool, kb_id: Uuid, source_id: Uuid, - prefixes: Option<&[String]>, + key_prefix: Option<&str>, as_of: Option>, ) -> AppResult> { let rows: Vec<(Uuid,)> = sqlx::query_as(&format!( - "SELECT d.id FROM documents d JOIN sources s ON s.id = d.source_id + "SELECT d.id FROM documents d WHERE d.kb_id = $1 AND d.source_id = $2 AND {doc_live} - AND ($3::text[] IS NULL OR EXISTS ( - SELECT 1 FROM unnest($3::text[]) p - WHERE starts_with(d.external_key, s.kind || ':' || p)))", + AND ($3::text IS NULL OR starts_with(d.external_key, $3))", doc_live = crate::record_axis::document_live_at("d", as_of.map(|_| 4)), )) .bind(kb_id) .bind(source_id) - .bind(prefixes) + .bind(key_prefix) .bind(as_of) .fetch_all(pool) .await?; @@ -1983,8 +1980,7 @@ pub async fn chunks_by_ids( ) -> AppResult> { let rows: Vec = sqlx::query_as(&format!( "SELECT c.id, c.document_id, c.seq, c.text, d.filename, d.source_id, - CASE WHEN s.kind IN ('api', 'statements') - THEN substr(d.external_key, length(s.kind) + 2) END AS external_id + s.kind AS source_kind, d.external_key FROM chunks c JOIN documents d ON d.id = c.document_id LEFT JOIN sources s ON s.id = d.source_id WHERE c.kb_id = $1 AND c.id = ANY($2) diff --git a/crates/utopia-store/src/execution_gate.rs b/crates/utopia-store/src/execution_gate.rs index cb44c2c79..ad95a4005 100644 --- a/crates/utopia-store/src/execution_gate.rs +++ b/crates/utopia-store/src/execution_gate.rs @@ -5,10 +5,12 @@ //! 撤不回的是它活着那几天离开图的东西——推理据它推出的派生、据它开出的违规与告警、 //! 进了导出的三元组、带引用给出去的答案。撤销还原的是图，不是图被读进去的那个世界。 //! -//! 这里问的只有一件事：**这一次合并会不会立刻把什么送出图外**。会，就留给人， -//! 把握再高也不动手。只管合并：分开撤起来不费什么，下一次合并就是撤销。 +//! **这一次合并会不会立刻把什么送出图外**。会，就留给人，把握再高也不动手。 +//! 来源已声明名字是身份标识时，也要由人决定合并（0071），不能让模型推翻来源的声明。 +//! 只管合并：分开撤起来不费什么，下一次合并就是撤销。 //! -//! `hold` 是纯函数，`impact_of` 只取数。攒批裁决器与治理 agent 走同一个闸门； +//! `hold` 是纯函数，`impact_of` 只取数；`hold_for_merge` 先看身份来源，再看影响。 +//! 攒批裁决器与治理 agent 走同一个闸门； //! 路线图上那道「执行闸门」（检查 agent 的调用）以后也接这一个，不长第二个。 use sqlx::PgPool; @@ -28,10 +30,10 @@ pub struct Impact { pub answered: i64, } -/// 留给人的理由。`Display` 写成 `kind value`，直接接在 `escalate_impact|` 后面， -/// 界面按 kind 措辞 +/// 留给人的理由。`review_reason` 写卡片的 code，`explain` 写决策台账的句子。 #[derive(Debug, Clone, PartialEq, Eq)] pub enum Hold { + Identity, Contradiction(String), Derived(i64), Answered(i64), @@ -40,6 +42,7 @@ pub enum Hold { impl std::fmt::Display for Hold { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + Hold::Identity => write!(f, "identity"), Hold::Contradiction(p) => write!(f, "contradiction {p}"), Hold::Derived(n) => write!(f, "derived {n}"), Hold::Answered(n) => write!(f, "answered {n}"), @@ -48,9 +51,17 @@ impl std::fmt::Display for Hold { } impl Hold { + pub fn review_reason(&self) -> String { + match self { + Hold::Identity => "escalate_identity".into(), + _ => format!("escalate_impact|{self}"), + } + } + /// 写给 `agent_decisions.reason` 的一句：人在 Agent 队列里看到 agent 为什么没动手 pub fn explain(&self) -> String { match self { + Hold::Identity => "a source treats one side's names as identifiers".into(), Hold::Contradiction(p) => { format!("merging would put two \"{p}\" facts on one entity") } @@ -107,6 +118,38 @@ pub fn hold(impact: &Impact) -> Option { None } +/// 身份来源的名字证据属于实体，不只属于它现在显示的本名；混有散文证据也不撤销声明。 +/// 每次要自动合并时重读来源的开关，包括缓存判决和同簇里已合过一侧的后继判决。 +pub async fn hold_for_merge( + pool: &PgPool, + kb_id: Uuid, + a: Uuid, + b: Uuid, +) -> AppResult> { + let identity: bool = sqlx::query_scalar( + "SELECT EXISTS ( + SELECT 1 FROM facts f + JOIN relation_types r ON r.id = f.predicate_id + JOIN fact_evidence fe ON fe.fact_id = f.id + JOIN documents d ON d.id = fe.document_id + JOIN sources s ON s.id = d.source_id + WHERE f.kb_id = $1 AND f.subject_id IN ($2, $3) + AND f.invalidated_at IS NULL AND r.builtin AND r.key = $4 + AND s.kind = 'statements' AND s.config->>'names' = 'identity' + )", + ) + .bind(kb_id) + .bind(a) + .bind(b) + .bind(crate::names::KNOWN_AS) + .fetch_one(pool) + .await?; + if identity { + return Ok(Some(Hold::Identity)); + } + Ok(hold(&impact_of(pool, kb_id, a, b).await?)) +} + /// 取一对实体合并会牵动什么。三条查询，只在裁决器有把握要合的时候才跑。 /// /// 写路径的守卫看的是「当前那一行」（`invalidated_at IS NULL`），与 `resolve_mention` diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs index e0a126eff..aafb6bf07 100644 --- a/crates/utopia-store/src/graph.rs +++ b/crates/utopia-store/src/graph.rs @@ -2408,15 +2408,21 @@ pub async fn entity_history( -- 人工改区间（302）。少了这一条，人做的修正在 -- 这条轴上归给「engine」——一个决策账本把人的 -- 决定记成机器的，比不记还坏 - 'fact.time_corrected'] + 'fact.time_corrected', 'conflict.retime_new'] -- 并入只可能由采纳造成，不会是 Review 里的拒绝 WHEN 'merged' THEN ARRAY['ontology.predicate_adopted'] ELSE ARRAY['fact.reject', 'conflict.reject_new', 'ontology.adoption_reverted'] END) AND (a.target_id = COALESCE(ev.supersedes, ev.id) - OR a.target_id IN (SELECT c.id FROM fact_conflicts c - WHERE c.old_fact_id = COALESCE(ev.supersedes, ev.id) - OR c.new_fact_id = ev.id) + OR (a.action <> 'conflict.retime_new' + AND a.target_id IN (SELECT c.id FROM fact_conflicts c + WHERE c.old_fact_id = COALESCE(ev.supersedes, ev.id) + OR c.new_fact_id = ev.id)) + -- 重定时改的是冲突里的新事实，修正行替的是它，上面那一跳对不上； + -- 审计里记着它写下的那一行，直接认。旧事实若因此被对账重画， + -- 那是引擎做的，不该算到这次决定头上 + OR (a.action = 'conflict.retime_new' + AND a.detail->>'corrected_fact_id' = ev.id::text) -- 采纳与撤销都记在关系类型上、一次动作改一批事实， -- 靠 fact_adoptions 精确关联到具体哪几条（corrected 事件 -- 是新行、merged 是旧行，两头都认） diff --git a/crates/utopia-store/src/jobs_claim_tests.rs b/crates/utopia-store/src/jobs_claim_tests.rs index e1822a3fe..48eb038d4 100644 --- a/crates/utopia-store/src/jobs_claim_tests.rs +++ b/crates/utopia-store/src/jobs_claim_tests.rs @@ -1,30 +1,100 @@ -//! 认领顺序（#1300）：不调对话模型的任务排在模型任务前面，哪怕它来得晚。 +//! 认领顺序（#1300、#1351）：不调对话模型的任务排在模型任务前面，哪怕它来得晚。 //! //! `claim_one` 认领的是全库的任务，所以这两条放在一百年前，排在库里别的一切前面。 use super::claim_one; -use sqlx::PgPool; +use sqlx::{Connection, PgConnection, PgPool}; #[tokio::test] -async fn a_later_sync_is_claimed_before_an_earlier_extraction() -> anyhow::Result<()> { +async fn a_later_intake_job_is_claimed_before_an_earlier_extraction() -> anyhow::Result<()> { let Some(url) = crate::test_db::url() else { return Ok(()); }; let pool = PgPool::connect(&url).await?; - let ids: Vec = sqlx::query_scalar( - r#"INSERT INTO jobs (kind, payload, run_at) VALUES + for kind in ["sync_source", crate::phrase_bindings::MATERIALIZE_KIND] { + let ids: Vec = sqlx::query_scalar( + r#"INSERT INTO jobs (kind, payload, run_at) VALUES ('extract_document', '{"test": "claim-order"}', now() - interval '101 years'), - ('sync_source', '{"test": "claim-order"}', now() - interval '100 years') + ($1, '{"test": "claim-order"}', now() - interval '100 years') RETURNING id"#, - ) - .fetch_all(&pool) - .await?; + ) + .bind(kind) + .fetch_all(&pool) + .await?; + + let claimed = claim_one(&pool, false).await?.map(|job| job.kind); + sqlx::query("DELETE FROM jobs WHERE id = ANY($1)") + .bind(&ids) + .execute(&pool) + .await?; + assert_eq!(claimed.as_deref(), Some(kind)); + } + Ok(()) +} + +#[tokio::test] +async fn an_upgrade_reindexes_already_queued_materialization() -> anyhow::Result<()> { + let Some(url) = crate::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let schema = format!("claim_upgrade_{}", uuid::Uuid::now_v7().simple()); + let result: anyhow::Result> = async { + let mut tx = pool.begin().await?; + // 旧函数和旧索引只装进这个 schema，不改并发测试正在用的队列。 + sqlx::raw_sql(&format!( + "CREATE SCHEMA {schema}; + SET LOCAL search_path TO {schema}; + CREATE TABLE jobs (kind text, run_at timestamptz, status text DEFAULT 'queued');" + )) + .execute(&mut *tx) + .await?; + sqlx::raw_sql(include_str!( + "../../../migrations/0116_intake_jobs_are_claimed_before_model_work.sql" + )) + .execute(&mut *tx) + .await?; + sqlx::query( + "INSERT INTO jobs (kind, run_at) VALUES + ('extract_document', now() - interval '1 day'), + ($1, now())", + ) + .bind(crate::phrase_bindings::MATERIALIZE_KIND) + .execute(&mut *tx) + .await?; + // 旧数据在升级前已经提交；新进程迁移时没有旧函数的内联计划缓存。 + tx.commit().await?; + let mut upgrade = PgConnection::connect(&url).await?; + sqlx::raw_sql(&format!( + "SET search_path TO {schema}; {}", + include_str!("../../../migrations/0123_human_phrase_materialization_is_intake.sql") + )) + .execute(&mut upgrade) + .await?; + upgrade.close().await?; + + // 认领由升级后的新 worker 连接执行，直接验证已有队列的索引键。 + let mut fresh = PgConnection::connect(&url).await?; + sqlx::raw_sql(&format!( + "SET search_path TO {schema}; + SET enable_seqscan = off; SET enable_bitmapscan = off;" + )) + .execute(&mut fresh) + .await?; + let eligible = sqlx::query_scalar( + "SELECT kind FROM jobs WHERE status = 'queued' AND job_priority(kind) = 0 + ORDER BY job_priority(kind), run_at", + ) + .fetch_all(&mut fresh) + .await?; + fresh.close().await?; + Ok(eligible) + } + .await; - let claimed = claim_one(&pool, false).await?.map(|job| job.kind); - sqlx::query("DELETE FROM jobs WHERE id = ANY($1)") - .bind(&ids) + sqlx::query(&format!("DROP SCHEMA IF EXISTS {schema} CASCADE")) .execute(&pool) .await?; - assert_eq!(claimed.as_deref(), Some("sync_source")); + assert_eq!(result?, vec![crate::phrase_bindings::MATERIALIZE_KIND]); Ok(()) } diff --git a/crates/utopia-store/src/name_vectors.rs b/crates/utopia-store/src/name_vectors.rs index 92d68c7db..a367e8eb6 100644 --- a/crates/utopia-store/src/name_vectors.rs +++ b/crates/utopia-store/src/name_vectors.rs @@ -79,30 +79,48 @@ pub async fn pending( Ok(rows) } +const DROP_FOR_SOURCE_SQL: &str = "DELETE FROM name_vectors v + USING fact_evidence fe JOIN documents d ON d.id = fe.document_id + WHERE fe.fact_id = v.fact_id AND d.source_id = $1"; + +const DROP_FOR_DOCUMENT_SQL: &str = "DELETE FROM name_vectors v USING fact_evidence fe + WHERE fe.fact_id = v.fact_id AND fe.document_id = $1"; + /// 来源刚开了身份名（0071 决定 6）：删掉有它文档证据的名字向量，与改配置同一个事务 pub async fn drop_for_source( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, source_id: Uuid, ) -> AppResult<()> { - sqlx::query( - "DELETE FROM name_vectors v - USING fact_evidence fe JOIN documents d ON d.id = fe.document_id - WHERE fe.fact_id = v.fact_id AND d.source_id = $1", - ) - .bind(source_id) - .execute(&mut **tx) - .await?; + sqlx::query(DROP_FOR_SOURCE_SQL) + .bind(source_id) + .execute(&mut **tx) + .await?; Ok(()) } /// 身份来源的文档抽完（0071 决定 5）：散文先写下的名字向量，这篇给了它身份证据就删 pub async fn drop_for_document(pool: &PgPool, document_id: Uuid) -> AppResult<()> { - sqlx::query( - "DELETE FROM name_vectors v USING fact_evidence fe - WHERE fe.fact_id = v.fact_id AND fe.document_id = $1", - ) - .bind(document_id) - .execute(pool) + sqlx::query(DROP_FOR_DOCUMENT_SQL) + .bind(document_id) + .execute(pool) + .await?; + Ok(()) +} + +/// 合并把重复名字的证据抄到了存活的名字上（#1426）：其中有了身份证据的，删掉向量 +pub async fn drop_with_identity_evidence( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + facts: &[Uuid], +) -> AppResult<()> { + if facts.is_empty() { + return Ok(()); + } + sqlx::query(concat!( + "DELETE FROM name_vectors v WHERE v.fact_id = ANY($1) AND ", + has_identity_evidence!("v.fact_id") + )) + .bind(facts) + .execute(&mut **tx) .await?; Ok(()) } diff --git a/crates/utopia-store/src/name_vectors_plan_tests.rs b/crates/utopia-store/src/name_vectors_plan_tests.rs index 651522f77..37983dc30 100644 --- a/crates/utopia-store/src/name_vectors_plan_tests.rs +++ b/crates/utopia-store/src/name_vectors_plan_tests.rs @@ -4,7 +4,7 @@ //! 规划器按统计选索引：库是空的时候几个索引代价一样，选哪个看运气（别的测试同时在写 facts， //! 选择就会翻）。所以这里在回滚的事务里造一个两千条事实、十分之一是名字的库并 ANALYZE， //! 比的是有数据时它走哪条路。 -use super::PENDING_SQL; +use super::{DROP_FOR_DOCUMENT_SQL, DROP_FOR_SOURCE_SQL, PENDING_SQL}; use sqlx::PgPool; use uuid::Uuid; @@ -68,3 +68,80 @@ async fn pending_reads_the_name_facts_through_the_predicate_index() -> anyhow::R assert!(!plan.contains("Sort"), "{plan}"); Ok(()) } + +/// 0071 的清向量每篇身份文档都走一次，不能读完别的文档的证据才挑中这一篇（#1408）。 +/// 临时表复制迁移后的列和索引，统计只来自本例，避免别的测试的写入改变查询计划。 +/// 这里约束选择性高的小来源；一次覆盖很多文档时，顺扫可能更便宜，应由规划器选择。 +#[tokio::test] +async fn dropping_vectors_reads_only_the_selected_documents_evidence() -> anyhow::Result<()> { + let Some(url) = crate::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + let mut tx = pool.begin().await?; + sqlx::raw_sql( + "CREATE TEMP TABLE documents (LIKE public.documents INCLUDING ALL) ON COMMIT DROP; + CREATE TEMP TABLE fact_evidence (LIKE public.fact_evidence INCLUDING ALL) ON COMMIT DROP; + CREATE TEMP TABLE name_vectors (LIKE public.name_vectors INCLUDING ALL) ON COMMIT DROP; + INSERT INTO documents (id, kb_id, source_id, filename, sha256, external_key) + SELECT gen_random_uuid(), md5('kb')::uuid, + md5(CASE WHEN n <= 2 THEN 'selected' ELSE 'other' END)::uuid, + 'document-' || n, md5(n::text), n::text + FROM generate_series(1, 1000) n; + INSERT INTO fact_evidence (fact_id, chunk_id, document_id) + SELECT gen_random_uuid(), gen_random_uuid(), d.id + FROM documents d CROSS JOIN generate_series(1, 20) n; + INSERT INTO name_vectors (fact_id, kb_id, entity_id, embedding) + SELECT fact_id, md5('kb')::uuid, md5('entity')::uuid, '[1,0,0]'::vector + FROM fact_evidence; + ANALYZE pg_temp.documents; + ANALYZE pg_temp.fact_evidence; + ANALYZE pg_temp.name_vectors;", + ) + .execute(&mut *tx) + .await?; + let (document, source): (Uuid, Uuid) = + sqlx::query_as("SELECT id, source_id FROM documents WHERE external_key = '1'") + .fetch_one(&mut *tx) + .await?; + + for (query, id, expected_rows) in [ + (DROP_FOR_DOCUMENT_SQL, document, 20), + (DROP_FOR_SOURCE_SQL, source, 40), + ] { + // 真正执行删除，但每次都回到相同的向量集合。比访问行数，不把机器快慢变成断言。 + sqlx::query("SAVEPOINT plan_case").execute(&mut *tx).await?; + let plan: serde_json::Value = + sqlx::query_scalar(&format!("EXPLAIN (ANALYZE, FORMAT JSON) {query}")) + .bind(id) + .fetch_one(&mut *tx) + .await?; + sqlx::query("ROLLBACK TO SAVEPOINT plan_case") + .execute(&mut *tx) + .await?; + let visited = evidence_rows_visited(&plan[0]["Plan"]); + assert!(visited > 0 && visited <= expected_rows, "{plan}"); + } + tx.rollback().await?; + Ok(()) +} + +fn evidence_rows_visited(node: &serde_json::Value) -> u64 { + let own_rows = if node["Relation Name"] == "fact_evidence" + && node["Node Type"] + .as_str() + .is_some_and(|kind| kind.ends_with("Scan")) + { + (node["Actual Rows"].as_u64().unwrap_or(0) + + node["Rows Removed by Filter"].as_u64().unwrap_or(0) + + node["Rows Removed by Index Recheck"].as_u64().unwrap_or(0)) + * node["Actual Loops"].as_u64().unwrap_or(0) + } else { + 0 + }; + own_rows + + node["Plans"] + .as_array() + .map(|children| children.iter().map(evidence_rows_visited).sum::()) + .unwrap_or(0) +} diff --git a/crates/utopia-store/src/resolution.rs b/crates/utopia-store/src/resolution.rs index 355b124e4..9847126b8 100644 --- a/crates/utopia-store/src/resolution.rs +++ b/crates/utopia-store/src/resolution.rs @@ -2231,6 +2231,7 @@ pub async fn merge_entities( .fetch_all(&mut *tx) .await?; let mut dup_ids = Vec::new(); + let (mut copied_to, mut copied_chunks) = (Vec::new(), Vec::new()); for (ids,) in dups { // 先给存活行接上来源，否则下一次物化可能作废它，再按原文重建，丢掉人的修正。 // 旧行的链接暂留：撤回合并还用得上；物化按它原有的规则清理失效链接。 @@ -2243,6 +2244,25 @@ pub async fn merge_entities( .bind(&ids[1..]) .execute(&mut *tx) .await?; + // 证据也抄一份（#1426）：身份来源的名字证据要是只留在作废的那条上，存活方就不再是 + // 身份实体，精确名挂靠和合并前的人工把关（0071 决定 2、7）都认不出它。 + // 同一分块存活行已有的不动；抄过去的记进账本，撤回合并时只删这些 + let copied: Vec = sqlx::query_scalar( + "INSERT INTO fact_evidence (fact_id, chunk_id, quote, document_id, doc_version, + proposed_predicate, quote_start, quote_end, attested_at, attested_by) + SELECT DISTINCT ON (chunk_id) $1, chunk_id, quote, document_id, doc_version, + proposed_predicate, quote_start, quote_end, attested_at, attested_by + FROM fact_evidence WHERE fact_id = ANY($2) + ORDER BY chunk_id, array_position($2, fact_id) + ON CONFLICT DO NOTHING + RETURNING chunk_id", + ) + .bind(ids[0]) + .bind(&ids[1..]) + .fetch_all(&mut *tx) + .await?; + copied_to.extend(std::iter::repeat_n(ids[0], copied.len())); + copied_chunks.extend(copied); dup_ids.extend_from_slice(&ids[1..]); } if !dup_ids.is_empty() { @@ -2252,6 +2272,8 @@ pub async fn merge_entities( .await?; invalidated.extend(dup_ids); } + // 抄过去的若是身份证据，存活的名字就不该再留向量（0071 决定 5） + crate::name_vectors::drop_with_identity_evidence(&mut tx, &copied_to).await?; // 画像加权合并 let (profile, profile_n) = match (&target.profile_embedding, &source.profile_embedding) { @@ -2356,8 +2378,8 @@ pub async fn merge_entities( "INSERT INTO entity_merges (id, kb_id, source_id, target_id, moved_subject_facts, moved_object_facts, invalidated_facts, target_profile_before, target_profile_n_before, target_type_before, - merged_by, reason) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)", + merged_by, reason, copied_evidence_facts, copied_evidence_chunks) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)", ) .bind(merge_id) .bind(kb_id) @@ -2371,6 +2393,8 @@ pub async fn merge_entities( .bind(target.type_id) .bind(merged_by) .bind(reason) + .bind(&copied_to) + .bind(&copied_chunks) .execute(&mut *tx) .await?; @@ -2430,6 +2454,8 @@ struct MergeRow { moved_subject_facts: Vec, moved_object_facts: Vec, invalidated_facts: Vec, + copied_evidence_facts: Vec, + copied_evidence_chunks: Vec, target_profile_before: Option, target_profile_n_before: i32, target_type_before: Option, @@ -2470,8 +2496,8 @@ async fn with_rewrites( pub async fn revert_merge(pool: &PgPool, kb_id: Uuid, merge_id: Uuid) -> AppResult<()> { let m: MergeRow = sqlx::query_as( "SELECT source_id, target_id, moved_subject_facts, moved_object_facts, - invalidated_facts, target_profile_before, - target_profile_n_before, target_type_before, reverted_at + invalidated_facts, copied_evidence_facts, copied_evidence_chunks, + target_profile_before, target_profile_n_before, target_type_before, reverted_at FROM entity_merges WHERE id = $1 AND kb_id = $2", ) .bind(merge_id) @@ -2546,6 +2572,23 @@ pub async fn revert_merge(pool: &PgPool, kb_id: Uuid, merge_id: Uuid) -> AppResu crate::temporal::rehome_tx(&mut tx, id, subject, object).await?; } } + // 合并时抄到存活行上的证据删掉：重复的那条复活了，证据还在它身上（#1426）。 + // 存活行之后可能被改写过（人改区间、时间线闭合、上面的搬回），证据随行复制到了新行上， + // 所以顺着 supersedes 删到底，跟 `with_rewrites` 同一条链。 + // 存活的名字因此不再有身份证据的话，它重新等着补向量，跟来源关掉身份名一样（0071 决定 6） + sqlx::query( + "WITH RECURSIVE chain(fact_id, chunk_id) AS ( + SELECT * FROM unnest($1::uuid[], $2::uuid[]) + UNION + SELECT f.id, c.chunk_id FROM facts f JOIN chain c ON f.supersedes = c.fact_id + ) + DELETE FROM fact_evidence fe USING chain c + WHERE fe.fact_id = c.fact_id AND fe.chunk_id = c.chunk_id", + ) + .bind(&m.copied_evidence_facts) + .bind(&m.copied_evidence_chunks) + .execute(&mut *tx) + .await?; sqlx::query("UPDATE facts SET invalidated_at = NULL WHERE id = ANY($1)") .bind(&m.invalidated_facts) .execute(&mut *tx) diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rs index 2536754f9..d82aa2f80 100644 --- a/crates/utopia-store/src/sources.rs +++ b/crates/utopia-store/src/sources.rs @@ -94,6 +94,7 @@ pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult> { listed_source.sync_interval_minutes, listed_source.sync_cron, listed_source.decay, listed_source.last_sync_at, listed_source.last_sync_status, listed_source.last_sync_error, listed_source.last_sync_added, + listed_source.sync_checkpoint, (SELECT count(*) FROM documents d WHERE d.source_id = listed_source.id AND d.deleted_at IS NULL) AS doc_count, (SELECT count(*) FROM documents d @@ -475,6 +476,76 @@ pub async fn touch_sync_time(pool: &PgPool, id: Uuid, at: DateTime) -> AppR // 同步运行记录（渠道审计历史） // --------------------------------------------------------------------------- +/// 工单来源的进度独立于只保留 50 条的审计历史；NULL 也不能回退到旧的 ok 行。 +pub async fn sync_checkpoint(pool: &PgPool, source_id: Uuid) -> AppResult>> { + sqlx::query_scalar("SELECT sync_checkpoint FROM sources WHERE id = $1") + .bind(source_id) + .fetch_optional(pool) + .await? + .ok_or(AppError::NotFound) +} + +/// 完整时间戳组的进度与成功记录一起提交，任一写入失败都保留旧游标。 +pub async fn finish_timestamp_sync( + pool: &PgPool, + run_id: Uuid, + source_id: Uuid, + previous_checkpoint: Option>, + checkpoint: Option>, + created_docs: i32, + updated_docs: i32, +) -> AppResult<()> { + if checkpoint < previous_checkpoint { + return Err(AppError::Validation( + "Source checkpoint cannot move backwards".into(), + )); + } + let mut tx = pool.begin().await?; + let source = sqlx::query( + "UPDATE sources SET sync_checkpoint = $3, last_sync_status = 'ok', + last_sync_error = NULL, last_sync_added = $4, last_sync_at = now() + WHERE id = $1 AND kind IN ('github_issues', 'jira_issues') + AND sync_checkpoint IS NOT DISTINCT FROM $2::timestamptz", + ) + .bind(source_id) + .bind(previous_checkpoint) + .bind(checkpoint) + .bind(created_docs + updated_docs) + .execute(&mut *tx) + .await?; + if source.rows_affected() != 1 { + return Err(AppError::Conflict( + "Source checkpoint changed during sync".into(), + )); + } + let run = sqlx::query( + "UPDATE source_sync_runs SET finished_at = now(), status = 'ok', error = NULL, + created_docs = $3, updated_docs = $4 + WHERE id = $1 AND source_id = $2 AND status = 'running'", + ) + .bind(run_id) + .bind(source_id) + .bind(created_docs) + .bind(updated_docs) + .execute(&mut *tx) + .await?; + if run.rows_affected() != 1 { + return Err(AppError::Conflict( + "Source sync run is no longer running".into(), + )); + } + sqlx::query( + "DELETE FROM source_sync_runs WHERE source_id = $1 AND id NOT IN + (SELECT id FROM source_sync_runs WHERE source_id = $1 + ORDER BY started_at DESC LIMIT 50)", + ) + .bind(source_id) + .execute(&mut *tx) + .await?; + tx.commit().await?; + Ok(()) +} + /// 调度时钟包含失败尝试，不能作增量游标。回到上次成功运行的开始， /// 让那次拉取期间发生的更新也能在下一轮读到；没有成功记录就重新全量读取。 pub async fn last_successful_sync_start( diff --git a/crates/utopia-store/src/temporal.rs b/crates/utopia-store/src/temporal.rs index a653fe2af..e5a71b17d 100644 --- a/crates/utopia-store/src/temporal.rs +++ b/crates/utopia-store/src/temporal.rs @@ -851,7 +851,11 @@ pub async fn reconcile_predicate( /// 撤掉一条事实（人判它是抽取错误）：它从来不在，时间线按剩下的行重算——关在它开始时的 /// 前任重新接上。先锁时间线再作废（见模块头）。返回有没有撤掉一行；已经作废的不算 -pub async fn retract(pool: &PgPool, kb_id: Uuid, fact_id: Uuid) -> AppResult { +pub async fn retract<'a>( + pool: impl sqlx::Acquire<'a, Database = Postgres>, + kb_id: Uuid, + fact_id: Uuid, +) -> AppResult { let mut tx = pool.begin().await?; let timelines = timelines_of(&mut *tx, kb_id, &[fact_id], None).await?; lock_timelines(&mut tx, kb_id, &timelines).await?; @@ -1087,19 +1091,16 @@ async fn rewrite_end_tx( }; let corrected = Uuid::now_v7(); sqlx::query( - "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, object_value, + "INSERT INTO facts (id, kb_id, subject_id, predicate_id, layer, phrase, object_id, object_value, valid_from, valid_from_precision, valid_from_grade, valid_to, valid_to_precision, confidence, supersedes, attested_from, attested_by, attested_to, end_derived, - from_statement_id, implied, layer, phrase, - corrected_ends, end_after_period) - SELECT $1, kb_id, subject_id, predicate_id, object_id, object_value, + from_statement_id, implied, corrected_ends, end_after_period) + SELECT $1, kb_id, subject_id, predicate_id, layer, phrase, object_id, object_value, valid_from, valid_from_precision, valid_from_grade, $3, $4, confidence, id, attested_from, attested_by, CASE WHEN $4::text = 'unknown' THEN COALESCE($5, now()) END, $6, - -- 开放陈述关上后还是陈述（RC-004）：少了这两列，修正行成了没有谓词的类型化行， - -- 物化再也读不到它 - from_statement_id, implied, layer, phrase, + from_statement_id, implied, -- 终点在这里被重新写下（文档说了终点，或时间线推了一个），人改过的终点就不在了； -- 起点那一半照旧 CASE corrected_ends WHEN 'both' THEN 'start' WHEN 'end' THEN NULL @@ -1176,17 +1177,15 @@ pub async fn rehome_tx( } let moved = Uuid::now_v7(); sqlx::query( - "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, object_value, + "INSERT INTO facts (id, kb_id, subject_id, predicate_id, layer, phrase, object_id, object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, derived_by_rule, supersedes, - attested_from, attested_to, end_derived, - from_statement_id, implied, layer, phrase, - corrected_ends, end_after_period) - SELECT $1, kb_id, COALESCE($3, subject_id), predicate_id, COALESCE($4, object_id), - object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, - confidence, derived_by_rule, id, attested_from, attested_to, end_derived, - -- 搬回源实体的陈述还是陈述（RC-004 后记），与 `rewrite_end_tx` 同一条 - from_statement_id, implied, layer, phrase, corrected_ends, end_after_period + attested_from, attested_by, attested_to, end_derived, + from_statement_id, implied, corrected_ends, end_after_period) + SELECT $1, kb_id, COALESCE($3, subject_id), predicate_id, layer, phrase, + COALESCE($4, object_id), object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, + confidence, derived_by_rule, id, attested_from, attested_by, + attested_to, end_derived, from_statement_id, implied, corrected_ends, end_after_period FROM facts WHERE id = $2", ) .bind(moved) @@ -1259,6 +1258,8 @@ async fn copy_qualifiers( } /// 证据引用随修正行复制。表层谓词一起搬：纠正的是时间区间，不是原文说了什么。 +/// 章节日期也一起搬（0104）：开放陈述改挂、改区间之后删掉一篇文档，见证照还在的证据 +/// 重算（`documents::reattest_tx`），新行的证据没带章节日期就退回文档自己的日期 async fn copy_evidence( tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, from: Uuid, @@ -1266,9 +1267,9 @@ async fn copy_evidence( ) -> AppResult<()> { sqlx::query( "INSERT INTO fact_evidence (fact_id, chunk_id, quote, proposed_predicate, document_id, doc_version, - quote_start, quote_end) + quote_start, quote_end, attested_at, attested_by) SELECT $1, chunk_id, quote, proposed_predicate, document_id, doc_version, - quote_start, quote_end + quote_start, quote_end, attested_at, attested_by FROM fact_evidence WHERE fact_id = $2 ON CONFLICT DO NOTHING", ) @@ -1327,18 +1328,16 @@ pub async fn correct_interval( let corrected = Uuid::now_v7(); // 标记变了也算终点变了（#994）：人把显示的 2024 改成 2025，存的时刻没动，说的却是另一年 let inserted: Option<(Uuid,)> = sqlx::query_as( - "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, object_value, + "INSERT INTO facts (id, kb_id, subject_id, predicate_id, layer, phrase, object_id, object_value, valid_from, valid_from_precision, valid_to, valid_to_precision, confidence, supersedes, - attested_from, attested_to, - from_statement_id, implied, layer, phrase, - corrected_ends, end_after_period) - SELECT $1, kb_id, subject_id, predicate_id, object_id, object_value, + attested_from, attested_by, attested_to, + from_statement_id, implied, corrected_ends, end_after_period) + SELECT $1, kb_id, subject_id, predicate_id, layer, phrase, object_id, object_value, $3, $4, $5, $6, confidence, id, - attested_from, + attested_from, attested_by, CASE WHEN $6::text = 'unknown' THEN COALESCE(attested_to, now()) END, - -- 人改了区间的陈述还是陈述（RC-004 后记），与 `rewrite_end_tx` 同一条 - from_statement_id, implied, layer, phrase, + from_statement_id, implied, CASE WHEN start_marked AND end_marked THEN 'both' WHEN start_marked THEN 'start' WHEN end_marked THEN 'end' END, @@ -1444,9 +1443,9 @@ pub async fn list_conflicts( let rows: Vec = sqlx::query_as( "SELECT c.id, c.reason, c.created_at, r.label AS predicate_label, c.old_fact_id, os.canonical_name AS old_subject, - oo.canonical_name AS old_object, fo.valid_from AS old_valid_from, + oo.canonical_name AS old_object, fo.valid_from AS old_valid_from, fo.valid_from_precision AS old_valid_from_precision, c.new_fact_id, ns.canonical_name AS new_subject, - no_.canonical_name AS new_object, fn_.valid_from AS new_valid_from, + no_.canonical_name AS new_object, fn_.valid_from AS new_valid_from, fn_.valid_from_precision AS new_valid_from_precision, fn_.confidence AS new_confidence FROM fact_conflicts c JOIN facts fo ON fo.id = c.old_fact_id @@ -1468,84 +1467,8 @@ pub async fn list_conflicts( Ok(rows) } -/// 人工裁决：close（旧事实闭合于 close_at 或新事实起点）/ keep（并存不矛盾）/ -/// reject_new（新事实是抽取错误，作废）。 -/// 一条待裁决的冲突：旧事实、新事实、新事实的起点及其精度 -type ConflictRow = (Uuid, Uuid, Option>, Option); - -pub async fn resolve_conflict( - pool: &PgPool, - kb_id: Uuid, - conflict_id: Uuid, - resolution: &str, - close_at: Option>, - close_at_precision: &str, -) -> AppResult<()> { - let row: Option = sqlx::query_as( - "SELECT c.old_fact_id, c.new_fact_id, fn_.valid_from, fn_.valid_from_precision - FROM fact_conflicts c JOIN facts fn_ ON fn_.id = c.new_fact_id - WHERE c.id = $1 AND c.kb_id = $2 AND c.status = 'open'", - ) - .bind(conflict_id) - .bind(kb_id) - .fetch_optional(pool) - .await?; - let Some((old_fact_id, new_fact_id, new_from, new_from_precision)) = row else { - return Err(utopia_core::AppError::NotFound); - }; - - let stored = match resolution { - "close" => { - // 闭合点带着它的精度走：人给了日期就用人给的精度，没给就闭合在新事实的 - // 起点——那个起点是几月还是几号，闭合点就是几月还是几号。从前一律写 day， - // 「2023 年 6 月接任」把前任闭合成了 6 月 1 日 - let (at, precision) = match close_at { - Some(at) => (at, close_at_precision), - None => ( - new_from.ok_or_else(|| { - utopia_core::AppError::invalid( - "close_at_required", - "close_at is required when the new fact has no start time", - ) - })?, - new_from_precision.as_deref().unwrap_or("day"), - ), - }; - close_superseded(pool, old_fact_id, at, precision).await?; - "closed" - } - "keep" => "kept_both", - "reject_new" => { - retract(pool, kb_id, new_fact_id).await?; - // 波及：同一新事实撞出的其他 open 冲突一并出队（新事实已死，无从裁起） - sqlx::query( - "UPDATE fact_conflicts - SET status = 'resolved', resolution = 'rejected_new', resolved_at = now() - WHERE new_fact_id = $1 AND status = 'open' AND id <> $2", - ) - .bind(new_fact_id) - .bind(conflict_id) - .execute(pool) - .await?; - "rejected_new" - } - other => { - return Err(utopia_core::AppError::Validation(format!( - "Unknown resolution: {other}" - ))) - } - }; - sqlx::query( - "UPDATE fact_conflicts SET status = 'resolved', resolution = $3, resolved_at = now() - WHERE id = $1 AND kb_id = $2", - ) - .bind(conflict_id) - .bind(kb_id) - .bind(stored) - .execute(pool) - .await?; - Ok(()) -} +mod conflicts; +pub use conflicts::{resolve_conflict, resolve_review_conflict, ConflictDecision}; /// 一条谓词的一端挂着**两个以上开放值**的持有者——唯一性没声明（或声明来晚了） /// 时账本的样子（#341）。这是给人看的提议依据，不是判决：`declared` 为假时它说 diff --git a/crates/utopia-store/src/temporal/conflicts.rs b/crates/utopia-store/src/temporal/conflicts.rs new file mode 100644 index 000000000..4cd8e4da0 --- /dev/null +++ b/crates/utopia-store/src/temporal/conflicts.rs @@ -0,0 +1,263 @@ +use super::*; +use serde_json::json; +use utopia_core::AppError; + +pub struct ConflictDecision<'a> { + pub action: &'a str, + pub close_at: Option>, + pub close_precision: &'a str, + pub new_from: Option>, + pub new_precision: &'a str, +} + +/// The ordinary card and subsequent proposal answers share this human writer. +pub async fn resolve_review_conflict( + pool: &PgPool, + kb_id: Uuid, + conflict_id: Uuid, + decision: ConflictDecision<'_>, + actor_id: Uuid, +) -> AppResult<()> { + resolve(pool, kb_id, conflict_id, decision, Some(actor_id)).await +} + +/// 不记操作者的入口：只有 store 的测试在用，人工审核走 [`resolve_review_conflict`]。 +pub async fn resolve_conflict( + pool: &PgPool, + kb_id: Uuid, + conflict_id: Uuid, + action: &str, + close_at: Option>, + close_precision: &str, +) -> AppResult<()> { + resolve( + pool, + kb_id, + conflict_id, + ConflictDecision { + action, + close_at, + close_precision, + new_from: None, + new_precision: "day", + }, + None, + ) + .await +} + +async fn resolve( + pool: &PgPool, + kb_id: Uuid, + conflict_id: Uuid, + decision: ConflictDecision<'_>, + actor_id: Option, +) -> AppResult<()> { + let (action, event) = match decision.action { + "close" => ("close_old", "conflict.close_old"), + "retime_new" => ("retime_new", "conflict.retime_new"), + "keep" => ("keep", "conflict.keep_both"), + "reject_new" => ("reject_new", "conflict.reject_new"), + other => return Err(AppError::Validation(format!("Unknown resolution: {other}"))), + }; + for precision in [decision.close_precision, decision.new_precision] { + if !crate::graph::WORLD_PRECISIONS.contains(&precision) { + return Err(AppError::invalid( + "bad_precision", + "Unknown date precision.", + )); + } + } + let mut tx = pool.begin().await?; + let pair: Option<(Uuid, Uuid)> = sqlx::query_as( + "SELECT old_fact_id, new_fact_id FROM fact_conflicts + WHERE kb_id = $1 AND id = $2 AND status = 'open'", + ) + .bind(kb_id) + .bind(conflict_id) + .fetch_optional(&mut *tx) + .await?; + let (old, new) = pair.ok_or(AppError::NotFound)?; + // Reconciliation takes timeline locks before fact/conflict rows. Re-read the + // pair afterwards: a rewrite may have carried this conflict to different rows. + let timelines = timelines_of(&mut *tx, kb_id, &[old, new], None).await?; + lock_timelines(&mut tx, kb_id, &timelines).await?; + sqlx::query("SELECT id FROM facts WHERE kb_id = $1 AND id = ANY($2) ORDER BY id FOR UPDATE") + .bind(kb_id) + .bind([old, new]) + .fetch_all(&mut *tx) + .await?; + let current: Option<(Uuid, Uuid)> = sqlx::query_as( + "SELECT c.old_fact_id, c.new_fact_id FROM fact_conflicts c + JOIN facts a ON a.id = c.old_fact_id JOIN facts b ON b.id = c.new_fact_id + WHERE c.kb_id = $1 AND c.id = $2 AND c.status = 'open' + AND a.invalidated_at IS NULL AND b.invalidated_at IS NULL FOR UPDATE OF c", + ) + .bind(kb_id) + .bind(conflict_id) + .fetch_optional(&mut *tx) + .await?; + if current != pair { + return Err(AppError::Conflict( + "the conflict changed before it could be decided".into(), + )); + } + let (new_from, new_precision): (Option>, Option) = + sqlx::query_as("SELECT valid_from, valid_from_precision FROM facts WHERE id = $1") + .bind(new) + .fetch_one(&mut *tx) + .await?; + let mut detail: serde_json::Value = sqlx::query_scalar( + "SELECT jsonb_build_object('predicate', r.label, + 'old_subject', os.canonical_name, 'old_object', COALESCE(to_jsonb(oo.canonical_name), fo.object_value), + 'new_subject', ns.canonical_name, 'new_object', COALESCE(to_jsonb(no_.canonical_name), fn_.object_value), + 'old_fact_id', fo.id, 'new_fact_id', fn_.id) + FROM facts fo JOIN facts fn_ ON fn_.id = $2 + JOIN entities os ON os.id = fo.subject_id JOIN entities ns ON ns.id = fn_.subject_id + JOIN relation_types r ON r.id = fo.predicate_id + LEFT JOIN entities oo ON oo.id = fo.object_id LEFT JOIN entities no_ ON no_.id = fn_.object_id + WHERE fo.id = $1", + ).bind(old).bind(new).fetch_one(&mut *tx).await?; + let stored = match action { + "close_old" => { + // 闭合点带着它的精度走：人给了日期就用人给的精度，没给就闭合在新事实的 + // 起点——那个起点是几月还是几号，闭合点就是几月还是几号。从前一律写 day， + // 「2023 年 6 月接任」把前任闭合成了 6 月 1 日 + let (at, precision) = match decision.close_at { + Some(at) => (at, decision.close_precision), + None => ( + new_from.ok_or_else(|| { + AppError::invalid( + "close_at_required", + "close_at is required when the new fact has no start time", + ) + })?, + new_precision.as_deref().unwrap_or("day"), + ), + }; + let corrected = close_superseded(&mut *tx, old, at, precision) + .await? + .ok_or(AppError::NotFound)?; + detail["close_at"] = json!(crate::graph::truncate_to(at, Some(precision))); + detail["close_at_precision"] = json!(precision); + detail["corrected_fact_id"] = json!(corrected); + "closed" + } + "retime_new" => { + let at = decision.new_from.ok_or_else(|| { + AppError::invalid( + "new_from_required", + "A new start is required when retiming a fact.", + ) + })?; + let corrected = rewrite_start(&mut tx, new, at, decision.new_precision).await?; + detail["new_from"] = json!(crate::graph::truncate_to(at, Some(decision.new_precision))); + detail["new_from_precision"] = json!(decision.new_precision); + detail["corrected_fact_id"] = json!(corrected); + "retimed_new" + } + "keep" => "kept_both", + _ => { + // Invalidation's trigger withdraws these rows. Settle the siblings + // first so they retain the human rejection rather than a withdrawal. + sqlx::query("UPDATE fact_conflicts SET status = 'resolved', resolution = 'rejected_new', resolved_at = now() + WHERE kb_id = $1 AND new_fact_id = $2 AND status = 'open'") + .bind(kb_id).bind(new).execute(&mut *tx).await?; + retract(&mut *tx, kb_id, new).await?; + "rejected_new" + } + }; + sqlx::query( + "UPDATE fact_conflicts SET status = 'resolved', resolution = $3, resolved_at = now() + WHERE kb_id = $1 AND id = $2", + ) + .bind(kb_id) + .bind(conflict_id) + .bind(stored) + .execute(&mut *tx) + .await?; + // The settlement is visible to reconciliation in this same transaction. + // Explicit ends stay fixed; only the engine's ends can be redrawn. + if matches!(action, "close_old" | "retime_new") { + tidy_timelines_tx(&mut tx, kb_id, &timelines).await?; + } + crate::audit::record_on( + &mut tx, + Some(kb_id), + actor_id, + event, + "conflict", + Some(conflict_id), + detail, + ) + .await?; + tx.commit().await?; + Ok(()) +} + +/// A start edit must not turn an engine-derived end into a person's fixed end. +/// Preserve the full row and its provenance, unlike an edit of the entire span. +async fn rewrite_start( + tx: &mut Transaction<'_, Postgres>, + fact_id: Uuid, + at: DateTime, + precision: &str, +) -> AppResult { + let (predicate, to, to_precision, derived): ( + Option, + Option>, + Option, + bool, + ) = sqlx::query_as( + "SELECT predicate_id, valid_to, valid_to_precision, end_derived FROM facts WHERE id = $1", + ) + .bind(fact_id) + .fetch_one(&mut **tx) + .await?; + let from = crate::graph::truncate_to(at, Some(precision)); + let temporal = crate::graph::predicate_temporal(&mut **tx, predicate).await?; + if temporal != crate::graph::Temporal::State { + return Err(AppError::invalid( + "not_a_state", + "Only a state has a start to retime.", + )); + } + if !derived { + if to.is_some_and(|end| from > crate::graph::truncate_to(end, to_precision.as_deref())) { + return Err(AppError::invalid( + "interval_inverted", + "The start of the interval is after its end.", + )); + } + crate::graph::Validity { + from: Some(from), + from_precision: Some(precision), + to, + to_precision: to_precision.as_deref(), + ..Default::default() + } + .under(temporal)?; + } + let corrected = Uuid::now_v7(); + sqlx::query( + "INSERT INTO facts (id, kb_id, subject_id, predicate_id, object_id, object_value, + valid_from, valid_from_precision, valid_from_grade, valid_to, valid_to_precision, + layer, confidence, derived_by_rule, supersedes, attested_from, attested_by, + attested_to, end_derived, from_statement_id, implied, corrected_ends, end_after_period) + SELECT $1, kb_id, subject_id, predicate_id, object_id, object_value, + $3, $4, NULL, valid_to, valid_to_precision, + layer, confidence, derived_by_rule, id, attested_from, attested_by, + attested_to, end_derived, from_statement_id, implied, + CASE WHEN corrected_ends IN ('end', 'both') THEN 'both' ELSE 'start' END, end_after_period + FROM facts WHERE id = $2", + ).bind(corrected).bind(fact_id).bind(from).bind(precision).execute(&mut **tx).await?; + carry_open_conflicts(tx, fact_id, corrected).await?; + sqlx::query("UPDATE facts SET invalidated_at = now() WHERE id = $1") + .bind(fact_id) + .execute(&mut **tx) + .await?; + copy_evidence(tx, fact_id, corrected).await?; + copy_qualifiers(tx, fact_id, corrected).await?; + copy_materialization_links(tx, fact_id, corrected).await?; + Ok(corrected) +} diff --git a/crates/utopia-store/tests/a_sync_runs_while_model_work_holds_every_slot.rs b/crates/utopia-store/tests/intake_runs_while_model_work_holds_every_slot.rs similarity index 75% rename from crates/utopia-store/tests/a_sync_runs_while_model_work_holds_every_slot.rs rename to crates/utopia-store/tests/intake_runs_while_model_work_holds_every_slot.rs index 51df79ec3..f236278ac 100644 --- a/crates/utopia-store/tests/a_sync_runs_while_model_work_holds_every_slot.rs +++ b/crates/utopia-store/tests/intake_runs_while_model_work_holds_every_slot.rs @@ -1,7 +1,7 @@ -//! #1300：常规名额全被调模型的任务占着时，后来的同步照样有名额跑（`jobs::NO_CHAT_LANE`）。 +//! #1300、#1351：常规名额全被调模型的任务占着时，同步和人判定后的物化照样有名额跑。 //! //! 起真实的 `run_worker`，它会认领库里任何排队任务，所以自成一个二进制（见 -//! `tests/store/main.rs`）。处理器只认带本测试标记的两条，别人的任务一概报错，不替人干活。 +//! `tests/store/main.rs`）。处理器只认带本测试标记的任务，别人的任务一概报错，不替人干活。 use std::sync::atomic::AtomicUsize; use std::sync::Arc; @@ -9,7 +9,7 @@ use std::time::Duration; use utopia_store::jobs; #[tokio::test] -async fn a_sync_runs_while_model_work_holds_every_slot() -> anyhow::Result<()> { +async fn intake_runs_while_model_work_holds_every_slot() -> anyhow::Result<()> { let Some(url) = utopia_store::test_db::url() else { return Ok(()); }; @@ -48,12 +48,17 @@ async fn a_sync_runs_while_model_work_holds_every_slot() -> anyhow::Result<()> { let result = async { queue("extract_document").await?; tokio::time::timeout(Duration::from_secs(10), extracting.recv()).await?; - queue("sync_source").await?; - let synced = tokio::time::timeout(Duration::from_secs(10), ran.recv()).await; - anyhow::ensure!( - matches!(&synced, Ok(Some(kind)) if kind == "sync_source"), - "the sync waited behind the extraction: {synced:?}" - ); + for kind in [ + "sync_source", + utopia_store::phrase_bindings::MATERIALIZE_KIND, + ] { + queue(kind).await?; + let claimed = tokio::time::timeout(Duration::from_secs(10), ran.recv()).await; + anyhow::ensure!( + matches!(&claimed, Ok(Some(claimed_kind)) if claimed_kind == kind), + "{kind} waited behind the extraction: {claimed:?}" + ); + } anyhow::Ok(()) } .await; diff --git a/crates/utopia-store/tests/store/a_rewritten_statement_keeps_its_section_dates.rs b/crates/utopia-store/tests/store/a_rewritten_statement_keeps_its_section_dates.rs new file mode 100644 index 000000000..db6ad279e --- /dev/null +++ b/crates/utopia-store/tests/store/a_rewritten_statement_keeps_its_section_dates.rs @@ -0,0 +1,180 @@ +//! 开放陈述改挂到另一个实体、人改了它的区间，新行还是那条开放陈述：层和短语不变，带着 +//! 见证日期的名字，它的证据也照样记着各自那一节的日期（0104）。 +//! +//! 之后删掉最早那篇文档，见证挪到还在的证据里最早的那一节，连同名字；不退回那篇文档 +//! 自己的日期。 +//! +//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆，绝不碰已有的库。 + +use chrono::{DateTime, TimeZone, Utc}; +use sqlx::PgPool; +use utopia_store::graph::{self, FactObject, Validity}; +use uuid::Uuid; + +const ORG: &str = "rewritten-statement-keeps-its-section-dates-test"; + +fn day(y: i32, m: u32, d: u32) -> DateTime { + Utc.with_ymd_and_hms(y, m, d, 0, 0, 0).unwrap() +} + +type Anchor = (Option>, Option); + +/// 改写出来的新行还是开放陈述，说的还是原文那个短语 +async fn shape(pool: &PgPool, fact: Uuid) -> anyhow::Result<(String, Option)> { + Ok( + sqlx::query_as("SELECT layer, phrase FROM facts WHERE id = $1") + .bind(fact) + .fetch_one(pool) + .await?, + ) +} + +async fn anchor(pool: &PgPool, fact: Uuid) -> anyhow::Result { + Ok( + sqlx::query_as("SELECT attested_from, attested_by FROM facts WHERE id = $1") + .bind(fact) + .fetch_one(pool) + .await?, + ) +} + +/// 一篇自己带日期的文档和它的一块正文 +async fn document(pool: &PgPool, kb: Uuid, own: DateTime) -> anyhow::Result<(Uuid, Uuid)> { + let (doc, chunk) = (Uuid::now_v7(), Uuid::now_v7()); + sqlx::query( + "INSERT INTO documents (id, kb_id, filename, sha256, doc_time, doc_time_source) + VALUES ($1, $2, $3, $3, $4, 'content')", + ) + .bind(doc) + .bind(kb) + .bind(format!("doc-{doc}.md")) + .bind(own) + .execute(pool) + .await?; + sqlx::query( + "INSERT INTO chunks (id, kb_id, document_id, seq, text) VALUES ($1, $2, $3, 0, $4)", + ) + .bind(chunk) + .bind(kb) + .bind(doc) + .bind("# 年报\n码表所属市场为全球。") + .execute(pool) + .await?; + Ok((doc, chunk)) +} + +/// 一条开放陈述，两篇文档各有一条证据，各记着自己那一节的日期 +async fn statement( + pool: &PgPool, + kb: Uuid, + subject: Uuid, + object: Uuid, + evidence: &[(Uuid, DateTime, &str)], +) -> anyhow::Result { + let (id, _) = graph::insert_open_statement( + pool, + kb, + subject, + "所属市场为", + FactObject::Entity(object), + None, + 0.9, + ) + .await?; + for (chunk, at, by) in evidence { + graph::add_evidence(pool, id, *chunk, Some("码表所属市场为全球"), None).await?; + graph::witness_evidence(pool, id, *chunk, Some((*at, *by))).await?; + graph::attest_statement(pool, id, *at, by).await?; + } + Ok(id) +} + +#[tokio::test] +async fn a_rewritten_statement_keeps_the_dates_its_sections_speak_from() -> anyhow::Result<()> { + let Some(url) = utopia_store::test_db::url() else { + return Ok(()); + }; + let pool = PgPool::connect(&url).await?; + sqlx::query("DELETE FROM organizations WHERE name = $1") + .bind(ORG) + .execute(&pool) + .await?; + let (org, ws, kb, watch, meter, market) = ( + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + Uuid::now_v7(), + ); + sqlx::raw_sql(&format!( + "INSERT INTO organizations(id,name) VALUES ('{org}','{ORG}'); + INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','{ORG}'); + INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','{ORG}'); + INSERT INTO entities(id,kb_id,canonical_name) VALUES + ('{watch}','{kb}','码表'), ('{meter}','{kb}','里程表'), ('{market}','{kb}','全球');" + )) + .execute(&pool) + .await?; + + let run = async { + let (chapter_2015, chapter_2020) = (day(2015, 6, 1), day(2020, 6, 1)); + let (by_2015, by_2020) = ("2015 年年报", "2020 年年报"); + // 两篇文档自己的日期都在 2026 年，章节说的是 2015 和 2020 + let (early, early_chunk) = document(&pool, kb, day(2026, 1, 10)).await?; + let (_late, late_chunk) = document(&pool, kb, day(2026, 3, 1)).await?; + let evidence = [(early_chunk, chapter_2015, by_2015), (late_chunk, chapter_2020, by_2020)]; + + // 一条改挂到另一个实体上（合并时走的那条路） + let to_move = statement(&pool, kb, watch, market, &evidence).await?; + let mut tx = pool.begin().await?; + let moved = utopia_store::temporal::rehome_tx(&mut tx, to_move, Some(meter), None) + .await? + .expect("a live row is rehomed"); + tx.commit().await?; + // 一条由人改了区间 + let to_correct = statement(&pool, kb, watch, market, &evidence).await?; + let corrected = utopia_store::temporal::correct_interval( + &pool, + to_correct, + Validity { + from: Some(day(2014, 1, 1)), + from_precision: Some("year"), + ..Default::default() + }, + ) + .await? + .expect("a live row is corrected"); + + let earliest: Anchor = (Some(chapter_2015), Some(by_2015.to_string())); + for fact in [moved, corrected] { + assert_eq!( + shape(&pool, fact).await?, + ("open".to_string(), Some("所属市场为".to_string())), + "a rewritten statement stays an open statement with its phrase" + ); + assert_eq!(anchor(&pool, fact).await?, earliest, "before the deletion"); + } + + utopia_store::documents::delete(&pool, kb, early, None).await?; + + let remaining: Anchor = (Some(chapter_2020), Some(by_2020.to_string())); + assert_eq!( + anchor(&pool, moved).await?, + remaining, + "a rehomed statement reads the chapter date of the evidence that remains, not its document's own date" + ); + assert_eq!( + anchor(&pool, corrected).await?, + remaining, + "a corrected statement reads the chapter date of the evidence that remains, not its document's own date" + ); + anyhow::Ok(()) + } + .await; + sqlx::query("DELETE FROM organizations WHERE id = $1") + .bind(org) + .execute(&pool) + .await?; + run +} diff --git a/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs b/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs index 19609d976..1eb2a2b2b 100644 --- a/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs +++ b/crates/utopia-store/tests/store/an_identity_name_attaches_by_its_exact_name.rs @@ -272,3 +272,140 @@ async fn turning_the_flag_on_drops_the_sources_name_vectors_and_off_makes_them_p }) .await } + +/// `entity` 现行的名字事实。 +async fn live_name(pool: &PgPool, entity: Uuid) -> anyhow::Result { + Ok(sqlx::query_scalar( + "SELECT f.id FROM facts f JOIN relation_types r ON r.id = f.predicate_id + WHERE f.subject_id = $1 AND r.key = $2 AND f.invalidated_at IS NULL", + ) + .bind(entity) + .bind(utopia_store::names::KNOWN_AS) + .fetch_one(pool) + .await?) +} + +/// #1426：人把身份来源的实体并进一个早就从散文里记下同名的实体。存活的名字留住身份证据， +/// 它仍是身份实体；撤回合并只收回抄过去的那些。 +#[tokio::test] +async fn a_merge_keeps_the_identity_evidence_of_the_name_it_dedupes() -> anyhow::Result<()> { + use utopia_store::execution_gate::{hold_for_merge, Hold}; + use utopia_store::resolution::{merge_entities, revert_merge}; + in_kb(|pool, kb, project, _| async move { + let (_, identity) = chunk(&pool, kb, Some(json!({"names": "identity"}))).await?; + let (_, prose) = chunk(&pool, kb, None).await?; + let keeper = holder(&pool, kb, Some(project), "acme/a.ts").await?; + let kept_name = named(&pool, kb, keeper, "acme/a.ts", &[prose]).await?; + name_vectors::set(&pool, kb, &[(kept_name, keeper, vec![1.0, 0.0, 0.0])]).await?; + let pushed = holder(&pool, kb, Some(project), "acme/a.ts").await?; + named(&pool, kb, pushed, "acme/a.ts", &[identity]).await?; + let bystander = holder(&pool, kb, Some(project), "acme/z.ts").await?; + let evidence = |fact: Uuid| { + let pool = pool.clone(); + async move { + let mut chunks: Vec = + sqlx::query_scalar("SELECT chunk_id FROM fact_evidence WHERE fact_id = $1") + .bind(fact) + .fetch_all(&pool) + .await?; + chunks.sort(); + anyhow::Ok(chunks) + } + }; + let mut both = vec![identity, prose]; + both.sort(); + + let merge = merge_entities(&pool, kb, pushed, keeper, None, "test").await?; + assert_eq!(live_name(&pool, keeper).await?, kept_name); + assert_eq!(evidence(kept_name).await?, both); + assert_eq!( + hold_for_merge(&pool, kb, keeper, bystander).await?, + Some(Hold::Identity) + ); + assert_eq!(vectors(&pool, kb).await?, Vec::::new()); + + revert_merge(&pool, kb, merge).await?; + assert_eq!(evidence(kept_name).await?, vec![prose]); + assert_eq!( + evidence(live_name(&pool, pushed).await?).await?, + vec![identity] + ); + assert_ne!( + hold_for_merge(&pool, kb, keeper, bystander).await?, + Some(Hold::Identity) + ); + assert!(pending_entities(&pool, kb).await?.contains(&keeper)); + Ok(()) + }) + .await +} + +/// 存活行原本就有的同一分块证据是它自己的，撤回合并不动。 +#[tokio::test] +async fn undoing_a_merge_leaves_the_survivors_own_evidence() -> anyhow::Result<()> { + use utopia_store::resolution::{merge_entities, revert_merge}; + in_kb(|pool, kb, project, _| async move { + let (_, prose) = chunk(&pool, kb, None).await?; + let keeper = holder(&pool, kb, Some(project), "acme/a.ts").await?; + let kept_name = named(&pool, kb, keeper, "acme/a.ts", &[prose]).await?; + let other = holder(&pool, kb, Some(project), "acme/a.ts").await?; + named(&pool, kb, other, "acme/a.ts", &[prose]).await?; + let merge = merge_entities(&pool, kb, other, keeper, None, "test").await?; + revert_merge(&pool, kb, merge).await?; + let kept: Vec = + sqlx::query_scalar("SELECT chunk_id FROM fact_evidence WHERE fact_id = $1") + .bind(kept_name) + .fetch_all(&pool) + .await?; + assert_eq!(kept, vec![prose]); + Ok(()) + }) + .await +} + +/// 合并之后存活的名字可能被改写：人改了区间，证据随行到新行上。撤回合并要从现在持有它的 +/// 那一行收回，不管存活行原本属于哪一边。 +#[tokio::test] +async fn undoing_a_merge_takes_the_copied_evidence_off_a_rewritten_name() -> anyhow::Result<()> { + use utopia_store::graph::Validity; + use utopia_store::resolution::{merge_entities, revert_merge}; + in_kb(|pool, kb, project, _| async move { + let (_, identity) = chunk(&pool, kb, Some(json!({"names": "identity"}))).await?; + let (_, prose) = chunk(&pool, kb, None).await?; + // 散文的名字先记，无论往哪边并都是它存活 + for prose_moves in [false, true] { + let name = format!("acme/{prose_moves}.ts"); + let prose_holder = holder(&pool, kb, Some(project), &name).await?; + let kept_name = named(&pool, kb, prose_holder, &name, &[prose]).await?; + let identity_holder = holder(&pool, kb, Some(project), &name).await?; + named(&pool, kb, identity_holder, &name, &[identity]).await?; + let (source, target) = if prose_moves { + (prose_holder, identity_holder) + } else { + (identity_holder, prose_holder) + }; + let merge = merge_entities(&pool, kb, source, target, None, "test").await?; + utopia_store::temporal::correct_interval( + &pool, + kept_name, + Validity { + from: Some(chrono::Utc::now() - chrono::Duration::days(30)), + from_precision: Some("day"), + ..Default::default() + }, + ) + .await? + .expect("the surviving name is live"); + revert_merge(&pool, kb, merge).await?; + let live = live_name(&pool, prose_holder).await?; + let chunks: Vec = + sqlx::query_scalar("SELECT chunk_id FROM fact_evidence WHERE fact_id = $1") + .bind(live) + .fetch_all(&pool) + .await?; + assert_eq!(chunks, vec![prose], "prose_moves = {prose_moves}"); + } + Ok(()) + }) + .await +} diff --git a/crates/utopia-store/tests/store/main.rs b/crates/utopia-store/tests/store/main.rs index 3ef2bf29f..2a1e58c9f 100644 --- a/crates/utopia-store/tests/store/main.rs +++ b/crates/utopia-store/tests/store/main.rs @@ -25,7 +25,7 @@ //! - `a_due_base_is_queued_once`：`queue_due_inferences` 扫全库到期的库。0050 起新建的库开着 //! 推理又从没推过，别的测试刚建的库全算到期，会被它排上推理任务——数自己库任务的测试 //! （比如 lib 单元测试里的 `phrase_bindings_delivery_tests`）就会多数出一个。 -//! - `a_sync_runs_while_model_work_holds_every_slot`：同样起真实的 `run_worker`，同样会认领 +//! - `intake_runs_while_model_work_holds_every_slot`：同样起真实的 `run_worker`，同样会认领 //! 库里任何排队任务，不是自己的就报错。 //! - `a_name_embedding_runs_while_model_work_holds_every_slot`：同上，换成 `embed_names`。 //! @@ -86,6 +86,7 @@ mod a_retraction_leaves_the_graph; mod a_retrieval_is_recorded_only_for_a_decaying_source; mod a_review_has_a_summary; mod a_review_page_is_assembled_in_batches; +mod a_rewritten_statement_keeps_its_section_dates; mod a_rewritten_statement_stays_a_statement; mod a_rule_computes_what_it_concludes; mod a_rule_concludes_a_type; diff --git a/docs/decisions/0004-language-and-localization.md b/docs/decisions/0004-language-and-localization.md index 23762b9d7..49aa85b87 100644 --- a/docs/decisions/0004-language-and-localization.md +++ b/docs/decisions/0004-language-and-localization.md @@ -1,6 +1,6 @@ # 0004 · Language follows the reader of each text -- **Status**: Implemented · open: the UI does not guess the browser language +- **Status**: Implemented · revised 2026-11-03 (#1468) · open: the UI does not guess the browser language - **Written**: 2026-08-29 · condensed into English 2026-09-03 - **Related**: [0001](0001-ontology-import-and-governance.md) made `description` load-bearing; [0003](0003-ontology-growth-loop.md) separated `reason` from @@ -47,6 +47,12 @@ The last row is not copy. It is a quotation. someone clicks "Suggest with AI", so the caller states the language; `label`/`description` in the same call follow `kb.ontology_lang`. `lang_name()` sends `Chinese` rather than `zh`; the cold-start path passes `en`. + > **Revised 2026-11-03 (#1468).** `lang_name()` and the cold-start path that passed `en` + > went with Suggest in #1129. The [0061](0061-the-ontology-is-proposed-from-the-open-graph-and-judged-by-its-questions.md) + > agent then asked for "the language of the ontology", which an empty base does not have, + > so the model picked one per batch (#1467). The agent now takes `kb.ontology_lang` from + > its caller and names it in the prompt, `Chinese` rather than `zh`, for labels and + > definitions; keys stay ASCII. 6. **Extracted data stays verbatim.** An entity is `张三` because paragraph 3 says so; translating it breaks the evidence match, splits one person across Chinese and English documents, and empties `surface_predicate`. Provenance has no switch. diff --git a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md index 83bf54601..263c73ddf 100644 --- a/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md +++ b/docs/decisions/0014-identity-from-the-person-scope-from-the-token.md @@ -129,6 +129,11 @@ push documents in. Chunk search (`POST /kbs/{id}/search`, added 2026-10-24, #1271) takes a token on the same terms, so the service can ask the base about what it mirrored. Each hit carries its score, its BM25 and vector ranks and its source; `offset` pages through one fused list, at most 100 deep. + `source_id` narrows the search to one source of the base, and a source outside the base answers + 404 like a missing one. `external_id_prefix` narrows it further to a push source's documents + whose `external_id` starts with the prefix, taken literally; without the `source_id` of an + `api` or `statements` source it answers 422 `prefix_needs_push_source`. Both filters apply + inside each channel, before fusion, so other documents cannot fill the candidates first. ## Dead ends @@ -161,7 +166,8 @@ push documents in. and document content" now also counts them. 2026-10-19: the graph overview and neighbourhood join them, and so does reading a pushed document back by its `external_id` (#1276). The ontology and a class's instances follow (#1265), then a document's extractions and the - base's export (#1266). 2026-10-24: chunk search joins them (#1271). + base's export (#1266). 2026-10-24: chunk search joins them (#1271). 2026-10-25: it can be + narrowed to one source and, for a push source, to an `external_id` prefix (#1271). - 2026-11-02: a service that sets up a push source also has to keep it tidy, and that still took a session. A write token may now rename, delete, re-extract and clean up the missing documents of a `statements` or `api` source, with the same guards as creating one. Other diff --git a/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md b/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md index 59e240e7d..5dccaebf5 100644 --- a/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md +++ b/docs/decisions/0018-the-lakehouse-is-one-protocol-away.md @@ -37,7 +37,7 @@ Left out on purpose: Snowflake key-pair JWT (local RSA signing, a dependency for 0011 set up defense in depth: parse and admit only SELECT, wrap a LIMIT, a read-only session with a timeout, JSON Lines out. The HTTP family **has no session**, so the third gate is a timeout alone (Trino's `query_max_execution_time` session property, Databricks' `wait_timeout`, Snowflake's `timeout`), and read-only rests entirely on the first gate. The first gate therefore parses with each engine's dialect: `DatabricksDialect`, `SnowflakeDialect`, and `GenericDialect` for Trino (sqlparser has no Trino dialect; Generic is a superset). One test runs the same three checks under all four dialects: SELECT passes, DELETE fails, two statements fail. -The fourth gate is assembled here for the HTTP family. Databricks' `JSON_ARRAY` and Snowflake's `data` return every value as a string; `coerce` restores numbers and booleans from the manifest / rowType column types, otherwise a model handed `"42"` stops doing arithmetic. Key order is assembled by hand instead of through `serde_json::Map`, which sorts keys unless `preserve_order` is on, and column order is the order the query wrote. +The fourth gate is assembled here for the HTTP family. Databricks' `JSON_ARRAY` and Snowflake's `data` return every value as a string; `coerce` restores numbers and booleans from the manifest / rowType column types, otherwise a model handed `"42"` stops doing arithmetic. Snowflake dates, times and timestamps are also restored from their documented epoch encodings. Their protocol-replay tests validate that conversion, not acceptance against a live Snowflake cluster; the live-cluster gate remains open (#241, #242). Trino writes decimal cells as strings too, and they go through the same `coerce`: short decimals reach JSON Lines as numbers, and decimals beyond f64 precision stay strings rather than rounded values. Key order is assembled by hand instead of through `serde_json::Map`, which sorts keys unless `preserve_order` is on, and column order is the order the query wrote. ## Loopback goes direct @@ -54,7 +54,7 @@ It is the one name of the four that is not "JSON in, JSON out": requests are sig ## Open questions - **How much schema to fetch.** All three expose `information_schema.columns`, and a lakehouse catalog can hold thousands of tables; `sync_schema_doc` caps at 200. With a schema in the connection string only that schema is read, otherwise the whole catalog. Whether that is enough waits for a real cluster. -- **The type-restoration table** in `coerce` is hand-written from the three vendors' docs. Snowflake's `fixed` with a scale returns `"42.10"`, which becomes 42.1 and loses the trailing zero; harmless for a model, possibly not for an "exact definition". Revisit when the semantic layer keeps evidence (0016 D1) and decide whether to keep the raw string alongside. +- **The type-restoration table** in `coerce` is hand-written from the three vendors' docs. Long exact decimal values with more than 15 significant digits stay strings rather than being rounded through an f64; approximate floating-point types remain numbers. Snowflake's `fixed` with a scale returns `"42.10"`, which becomes 42.1 and loses the trailing zero; harmless for a model, possibly not for an "exact definition". Revisit when the semantic layer keeps evidence (0016 D1) and decide whether to keep the raw string alongside. - **Trino's `ssl` inference**: a password, `ssl=true`, or port 443 / 8443 means https, anything else is plaintext. That is trino-python's rule, and someone who gets it wrong sees a TLS error instead of a hint. ## Status history diff --git a/docs/decisions/0043-every-review-queue-is-governed.md b/docs/decisions/0043-every-review-queue-is-governed.md index e61d22870..657c649da 100644 --- a/docs/decisions/0043-every-review-queue-is-governed.md +++ b/docs/decisions/0043-every-review-queue-is-governed.md @@ -74,3 +74,20 @@ Agent rows for facts and conflicts carry their summary in place of two names. A The status line as it stood on 2026-09-27, before status lines were cut to one line: > Decided 2026-09-14 · **no code on `dev`** · cut 1 was built in PR #699 — migration 0058 widening `agent_decisions` to `fact` and `conflict` with `summary` and `detail`, `queue_agent` walking low-confidence and stale facts then temporal conflicts after the duplicate rounds of the same `govern` job, every applied action revertible from the Agent queue — and that PR was **closed on 2026-09-17**, to re-land on the open graph once [0044](0044-the-ontology-is-a-view-over-what-documents-say.md) cut 2 has settled alignment · violations, ontology defects and concept mappings are cut 2 · **the decisions below stand; only the code was withdrawn** + +### 2026-10-20: the human conflict path lands first (#1131) + +The review split begins with people: a conflict card can close the old fact at a +chosen date, retime the new fact's start, keep both, or reject the new fact. Dates +carry the precision the person entered. Retiming changes only the start: an +explicit end and its period mark remain, and an engine-derived end stays derived. + +The writer locks the affected timelines before fact and conflict rows, then +checks that the conflict is still open on the live pair it locked. Fact rewrites, +timeline reconciliation, the conflict's resolution and the actor's audit event +commit together. A failed audit or stale pair writes nothing. `retimed_new` joins +the existing conflict resolution vocabulary; existing rows need no backfill. + +This is the human slice of the accepted actions above. The governor still handles +only duplicates. Proposals follow in a separate PR; automatic conflict application +and undo require the real-model lease measurement requested in #1131's review. diff --git a/docs/decisions/0071-a-source-may-declare-its-names-identities.md b/docs/decisions/0071-a-source-may-declare-its-names-identities.md index 8b9f5f9df..3ef35319d 100644 --- a/docs/decisions/0071-a-source-may-declare-its-names-identities.md +++ b/docs/decisions/0071-a-source-may-declare-its-names-identities.md @@ -1,6 +1,6 @@ # 0071 · A source may declare its names identities -- **Status**: In progress · 2026-10-19 · [#1293](https://github.com/deeplethe/utopia/issues/1293) · open: decision 7 +- **Status**: Implemented · 2026-10-25 · #1333, #1390, #1425, migration 0125 · open: none - **Written**: 2026-10-19 (conventions in the [README](README.md)) - **Related**: [0054](0054-a-source-may-push-statements-in-the-open-contract.md) decision 3 sends pushed statements down the same resolution path as prose, and this record narrows that for one kind of source; [0041](0041-a-name-is-a-claim-about-an-entity.md) owns name facts and name-vector recall; [#1248](https://github.com/deeplethe/utopia/issues/1248) meets the same problem from the merge side and is decided here as decision 7; [#1296](https://github.com/deeplethe/utopia/issues/1296) measured what the recall costs; [#1308](https://github.com/deeplethe/utopia/pull/1308) is the disjointness fix that stands on its own @@ -41,12 +41,14 @@ One name fact can carry evidence from both an identity source and a prose docume - **On:** the update that sets the flag also deletes the stored vectors of name facts that now have identity evidence, in the same transaction. Leaving them in place and skipping them would put an evidence join into every `nearest` query, which is the per-mention cost #1296 measured. - **Off:** nothing is deleted. Those name facts become pending again, and the next extraction in the base backfills their vectors through `pending`, as it does for any name whose vector is missing. Recall for them returns from then on. Pairs that were never proposed while the flag was on are not proposed in retrospect. -**7. A `same` verdict that touches an identity entity goes to a person.** +**7. A `same` verdict that touches an identity entity goes to a person.** Implemented 2026-10-25. *Revised 2026-10-22 (#1248):* decision 1 said #1248 decides the merge side with the same flag; this is that decision. An identity entity is one with a name fact that has evidence from an identity source, the same join decision 4 uses. When the adjudicator returns `same` at or above `AUTO_CONF` for a pair where either side is an identity entity, the pair is escalated to human review with the reason `escalate_identity` instead of being merged. This holds on both paths that apply a verdict: `apply_verdict` in `adjudication.rs` with governance off, the second look included (#889), and `apply` in `governance.rs` with governance on. It reuses the execution gate's escalation (0027), so a person sees why the pair is waiting, not "the adjudicator was unsure". A `kept` verdict still closes the pair automatically, which keeps the queue short. A merge a person makes is unchanged. Why a person and not a stricter threshold: an identity source has already said what its names are. Decisions 2 to 4 remove the pairs that name vectors and the single-candidate split would raise, so what is left are containment and type-drift pairs and the cross-source duplicate (one thing pushed under two names by two producers). Those are exactly the cases where the model's `same` is a guess about a claim the producer made, and a wrong merge of identities is what #1248 measured: on a small local model, 40 automatic merges with governance off, nearly all wrong (a method into its file, two unrelated types). A person sees few of them because the vectors are off. Raising `AUTO_CONF` for these pairs would still let a confident wrong verdict through, and a per-base switch is a dead end (below). +**A merge keeps the identity evidence of a name it dedupes.** *Revised 2026-10-30 (#1426):* `merge_entities` invalidates a duplicate name fact, and its evidence stayed on the invalidated row. An entity whose name came from an identity source, merged into one that already held that name from prose with an earlier `recorded_at`, stopped being an identity entity, so decisions 2 and 7 no longer reached it. The merge now copies the duplicate's evidence onto the surviving fact, one row per chunk, and deletes the surviving name's vector if the copy makes it an identity name (decision 5). `entity_merges` records the copied pairs. Undoing the merge deletes them, following `supersedes` because a rewrite of the surviving fact carries its evidence forward, and the revived duplicate still has its own. The name is then pending again, as in decision 6. In a chain S→T→C, undoing only S→T leaves the copy on C's name until T→C is undone too, which errs toward a person. + **A NULL-typed holder takes the declared class.** *Revised 2026-10-23 (#1390):* this note said a holder with `type_id IS NULL` keeps its class and left the choice to the next PR. A typed identity mention that attaches to such a holder now gives it the declared class, as the type-drift path does and with the same helper, so a class a person removed stays removed. ## Not doing @@ -67,3 +69,4 @@ Why a person and not a stricter threshold: an identity source has already said w - `a_name_with_identity_evidence_gets_no_vector_even_when_prose_names_it_too` (decision 5). - Turning the flag on deletes the identity names' vectors; turning it off makes them pending again (decision 6). - `a_same_verdict_on_an_identity_entity_waits_for_a_person`: with governance off and with it on, a `same` verdict at or above `AUTO_CONF` on a pair with one identity side leaves both entities in place and an open review with `escalate_identity`; a `kept` verdict on the same pair still closes it (decision 7). +- `a_merge_keeps_the_identity_evidence_of_the_name_it_dedupes`, `undoing_a_merge_leaves_the_survivors_own_evidence` and `undoing_a_merge_takes_the_copied_evidence_off_a_rewritten_name` (#1426). diff --git a/docs/decisions/0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md b/docs/decisions/0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md index d0f0bbbc9..72e55351a 100644 --- a/docs/decisions/0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md +++ b/docs/decisions/0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md @@ -1,6 +1,6 @@ # A source checkpoint ends at a timestamp boundary -Proposed 2026-10-19 (#1212, #1213); open: implementation and regression coverage. +In progress 2026-10-25 (#1359, migration 0119); open: custom batch progression and recovery (#1397). ## Context @@ -27,8 +27,14 @@ This follows the [maintainer's decision](https://github.com/deeplethe/utopia/iss ### GitHub and Jira -Read items in ascending update-time order. Skip items at or before the -checkpoint before counting either budget. When the 200-item processing budget +GitHub/Jira store their watermark in `sources.sync_checkpoint`, initially NULL even +when successful audit rows exist. Progress and the successful run commit together. +The 2026-10-23 revision below records the listing verification, complete GitHub +discussion reads and bounded waits used by these readers. + +Read items in ascending update-time order. Jira breaks ties by key so an unchanged +result keeps the same page boundaries. Skip items at or before the checkpoint +before counting either budget. When the 200-item processing budget or ten-page listing budget is exhausted inside a group, continue reading and processing until that group is complete. Stop before consuming the next timestamp group, then save the completed group's time as the checkpoint. @@ -58,6 +64,48 @@ group cap or a group-size alert; use the existing rate-limit backoff. The exception applies to source listing and processing, not permission to replace a GitHub document with a truncated comments or events history. +### Revision 2026-10-23: verification, complete discussions and bounded waits + +Keep the second listing pass before ingestion. Re-read through the selected +timestamp boundary and compare the listed identities and update times with the +first pass, including skipped old rows and excluded GitHub PRs. Those rows can +still move offsets. For example, after reading issues 1–100, updating issue 1 +can move it to the tail; the next page then starts at issue 102 and silently +misses issue 101. Advancing the checkpoint without detecting that movement can +make issue 101 unreachable on later incremental runs. Jira has the same risk. + +The second pass roughly doubles requests for the selected listing prefix, not +GitHub's per-ticket comments and events requests. It does not extend the chosen +window, and changes only beyond that boundary do not invalidate the prefix. +Comparison ignores ordering within one timestamp group. A changed identity or +update time fails the run before ingestion, preserves the previous checkpoint, +sets `last_sync_status` to `failed` and raises the existing `source.sync_failed` +alert. A later run starts at the same checkpoint. Keep this failure behavior; +do not add repeated scans inside the same run. This detects observed movement, +not an atomic snapshot. A continuously changing prefix can repeatedly fail; +progress under sustained concurrent updates is not guaranteed. + +GitHub comments and events must be read to the end, even beyond ten pages. +The listing budget is not a limit on a selected ticket's discussion history. +Do not skip a large discussion, return truncated content or advance progress +past a ticket whose details could not be read completely. A detail-page failure +keeps the old document and checkpoint. Large discussions therefore increase +requests, memory use and run duration. On Jira Server and Data Center the search +`total` remains required: without it, a premature empty page could be mistaken +for a complete group. Jira Cloud's `search/jql` has no total (Atlassian +CHANGE-2046), so there an empty page, or a page without `nextPageToken`, before +`isLast` fails the run instead (#1453). + +For explicit rate limits, keep at most three retries of the current request, +preserving its in-memory position. Honor a valid wait hint only when it is at +most 30 seconds per retry, retaining the existing minimum backoff. An invalid +or longer hint fails the run immediately without sleeping or shortening the +provider's requested wait. This prevents one long wait from occupying a sync +slot for hours; it is not a total run-duration limit. The existing job retry +and source schedule govern later attempts and do not persist `Retry-After`, so +they do not guarantee that a later attempt waits until the provider's deadline. +Strict cross-job rate-limit scheduling remains outside this revision. + ### Custom The endpoint already returns the whole requested response. Walk its items in @@ -73,7 +121,7 @@ incomplete runs. No source-update field, pagination token or saved response is added to the custom protocol; `doc_time` remains document time. The custom fix is being reviewed in -[PR #1246](https://github.com/deeplethe/utopia/pull/1246). It does not resolve +[PR #1397](https://github.com/deeplethe/utopia/pull/1397). It does not resolve the GitHub and Jira parts of #1212. ## Recovery and tradeoffs @@ -120,6 +168,12 @@ must demonstrate: this progress without omissions or duplicate versions and processing jobs. - A failed page fetch, detail fetch or write cannot advance the checkpoint; retry recovers the whole group. Updates during pagination remain reachable. +- A page-moving update rejects the window; once the listing stabilizes, a + retry recovers the skipped issue and a later run reads the moved issue. +- GitHub comments and events beyond ten pages are fully read; failure on a + later detail page preserves the old document and checkpoint. +- A 30-second rate-limit hint is accepted; a 31-second hint returns an error + without sleeping or retrying that request. Cover seconds and HTTP-date hints. - Custom's 201 changed items finish on the second run; 450 finish over three capped runs. Cover an unchanged prefix and no custom update field. - The one-time recovery scan preserves existing data and returns to incremental diff --git a/docs/decisions/0075-a-ticket-source-says-where-its-reading-stopped.md b/docs/decisions/0075-a-ticket-source-says-where-its-reading-stopped.md new file mode 100644 index 000000000..6366c1ce7 --- /dev/null +++ b/docs/decisions/0075-a-ticket-source-says-where-its-reading-stopped.md @@ -0,0 +1,79 @@ +# 0075 · A ticket source says in the API where its reading stopped + +- **Status**: Implemented · 2026-10-28 · no migration (the column came with 0119) · open: the interface does not read it yet +- **Written**: 2026-10-28 +- **Related**: [0072](0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md) (what the checkpoint is and how it moves); [#1212](https://github.com/deeplethe/utopia/issues/1212) (the tickets a round used to skip); [0033](0033-rss-source-summaries-are-source-scoped.md) (the RSS summary — a different shape for a different question) + +## Problem + +A ticket source stops at its per-round budget all the time: 200 tickets, or ten +listing pages, whichever comes first. A round that stops there is a **success** — +it consumed a prefix, and the next round continues from where it stopped (0072). +The source list then says "synced" and says how many documents the round added. +It does not say *where* the reading stopped. + +The one thing that says where is the checkpoint, and it lived on the source row +with nothing carrying it out of the API. So an operator asking "did this source +get everything?" had nothing to read: the run history says how many documents +each round added, not how far into the provider's timeline the reading got. The +answer was in a log line. + +## Decisions + +1. **The source API carries the checkpoint.** `Source.sync_checkpoint` and + `SourceView.sync_checkpoint` hold the update time a ticket source has consumed + through — the time whose whole group was read (0072). The list response carries + it, and so do the create/update responses, so the two shapes of a source never + disagree about the same row. + +2. **It is the raw timestamp, not a state the server computes.** The RSS summary + (0033) is a computed block: the server works out which of the five counts + applies, so the client does not re-derive it from `kind` and `content_mode`. + The checkpoint is not like that. Whether the source has more to give is a + question only the provider can answer, and the server does not ask it between + rounds — a state word would have to say "caught up" or "behind", and the server + cannot know which. The timestamp is the fact; what it means is the reader's. + +3. **NULL means there is no checkpoint.** Not zero, not the epoch, not "now". + NULL says the source has not finished a timestamp group yet (first round), or + that this kind does not read by timestamp at all. It doubles as the + applicability signal: a kind that does not read by timestamp has NULL there, so + the client does not need to know the kinds to know whether the field means + anything. + +4. **It says where the reading stopped, not when the round ran.** `last_sync_at` is + the round's time and exists for every kind. The checkpoint is a position in the + provider's timeline and exists only for the kinds that have one. The two answer + different questions, and both are carried. + +5. **The interface does not read it yet.** Where to put it and what to call it is a + change to the source row, and that row is already dense — status, time, added, + error, endpoint, schedule. A diagnostic that only appears in a tooltip is not + obviously better than one that is in the API. This record settles the shape; the + interface follows when someone wants it. + +## Dead ends + +- **A `state` word beside the timestamp** ("caught up", "behind", "partly read"). + The server cannot know it: the provider is not asked between rounds, so + "caught up" would be a guess. A guess in a field that looks authoritative is how + a wrong answer gets believed — the same reason the RSS summary computes what it + can actually compute instead of guessing the rest. +- **A count of tickets still to read.** The server does not know the provider's + total for a ticket source, and counting one would need a listing the round never + made. +- **Putting the checkpoint in the run history instead.** The runs are a tail, not a + ledger: they are capped and pruned. The position a source is at would disappear + with them. +- **Only the list, and not the source responses.** Then the two shapes of a source + disagree about the same row, and a client that read one and wrote the other + would see the field vanish. +- **A separate endpoint for it.** One more route for one nullable column, and the + list already carries the rest of the source's sync state. + +## Open questions + +- **Where the interface shows it.** The source row is dense; a second timestamp + there needs a place and a wording, and neither is settled. +- **Whether a ticket source should say how much of the provider's window it has + read.** That is a count the server does not have. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 64b16381b..564febbf8 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -56,7 +56,7 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | | Record | Status | Overtaken | |---|---|---|---| | 0041 | [A name is a claim about an entity](0041-a-name-is-a-claim-about-an-entity.md) | In progress | | -| 0071 | [A source may declare its names identities](0071-a-source-may-declare-its-names-identities.md) | In progress | | +| 0071 | [A source may declare its names identities](0071-a-source-may-declare-its-names-identities.md) | Implemented | | ### [time](../design/time.md) @@ -115,8 +115,9 @@ By domain; the domains are the files of [../design/](../design/README.md). **Sta | 0052 | [Document content is a read contract over the retained ledger](0052-document-content-is-a-read-contract.md) | Proposed | | | 0054 | [A source may push statements in the open contract](0054-a-source-may-push-statements-in-the-open-contract.md) | Implemented | | | 0065 | [The OCR reader chooses its provider](0065-readers-choose-their-provider.md) | Implemented | | -| 0072 | [A source checkpoint ends at a timestamp boundary](0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md) | Proposed | | +| 0072 | [A source checkpoint ends at a timestamp boundary](0072-a-source-checkpoint-ends-at-a-timestamp-boundary.md) | In progress | | | 0074 | [A Google Drive source is connected by a person](0074-a-google-drive-source-is-connected-by-a-person.md) | Implemented | | +| 0075 | [A ticket source says in the API where its reading stopped](0075-a-ticket-source-says-where-its-reading-stopped.md) | Implemented | | ### [lakehouse-and-actions](../design/lakehouse-and-actions.md) diff --git a/docs/design/chat-and-mcp.md b/docs/design/chat-and-mcp.md index 9c63cbe07..45ff3cd95 100644 --- a/docs/design/chat-and-mcp.md +++ b/docs/design/chat-and-mcp.md @@ -79,11 +79,10 @@ RRF score ranks hits within one query only; it is not comparable across queries page fuses 100 candidates per channel where chat fuses 24, so the search page can rank the same query differently from chat and from before #1271. Pages join up only while the base is unchanged between requests, and each channel counts its ranks before the final `as_of` filter, so a -historical page can skip rank numbers. The body may name a `source_id` of the base (another base's -source is a 404) and, with it, up to 8 `external_id_prefixes` of at most 256 bytes; a prefix without a -`source_id` is refused. Both channels then recall only from that source's documents, live at `as_of`, -whose `external_id` starts with one of the prefixes, so the base's other documents cannot take the -recall a filter after fusion would lose (2026-10-19). +historical page can skip rank numbers. A `source_id` of the base (another base's source is a 404) +and, for a push source, an `external_id_prefix` limit both channels to that source's documents +before fusion, so the base's other hits cannot take the candidates a filter after fusion would +lose; the prefix compares against the key `pushed_key` builds (#1271). A search that returns a document of a source that decays counts as a use of it, in the background and never failing the read ([RC-003](../decisions/re-cinq/RC-003-a-source-may-decay.md)): `search_chunks` diff --git a/docs/design/identity.md b/docs/design/identity.md index 36d8b87c7..a2f2d1869 100644 --- a/docs/design/identity.md +++ b/docs/design/identity.md @@ -42,8 +42,9 @@ is the fallback when nothing is declared [0009, 0016 B3]. Untyped same-name enti [0009]. **Merges are reversible on both clocks.** `entity_merges` records what moved with `created_at` and -`reverted_at`; `revert_merge` moves facts, name facts and qualifiers back; a read `as_of` a moment -before a merge shows two nodes [0019, 0041]. A merge is a person's action, the adjudicator's at 0.8, +`reverted_at`; `revert_merge` moves facts, name facts and qualifiers back and takes back the evidence +the merge copied from a duplicate onto its survivor; a read `as_of` a moment before a merge shows two +nodes [0019, 0041, 0071]. A merge is a person's action, the adjudicator's at 0.8, or the governor's through its gate, and an automatic merge is held for a person when it would leave the graph (a functional contradiction at one moment, a derivation, a cited answer) [0027]. Retypes and renames are audited with before and after; a name collision on rename is a prompt to merge, not diff --git a/docs/design/sources.md b/docs/design/sources.md index d042991ef..c459b1772 100644 --- a/docs/design/sources.md +++ b/docs/design/sources.md @@ -28,6 +28,29 @@ from a `HEAD` on the file before anything is downloaded [#1327]. **Google Drive.** A person connects it with OAuth and it reads Drive as them; the refresh token is a sealed source secret and the browser never holds a Google token; the flow is bound to the session that started it; only the person who connected picks folders, subfolders come along, and a sync stops once that person is no longer an editor; a run downloads at most 2000 files and says so when it stops there; Docs, Sheets and Slides are exported to DOCX, XLSX and PPTX; change is Drive's marker (`md5Checksum`, or `modifiedTime` for Google formats) because an export's bytes differ every time; "not in source" only after a complete walk [0074]. +**Ticket checkpoints.** GitHub/Jira read by ascending update time and stop only after +a whole timestamp group, even when that exceeds 200 items or ten listing pages. +Already consumed timestamps use neither budget. Their private source checkpoint is +committed with the successful run, independent of retained audit history; empty or +failed runs do not advance it. Upgrades start these two readers from NULL and reuse +document/version/job deduplication. A second listing pass checks the selected prefix +for page movement before ingestion, roughly doubling listing requests. A changed +prefix fails with the existing sync alert and keeps the old checkpoint; neither +API provides a snapshot, and a constantly changing prefix may repeatedly fail. +GitHub reads selected tickets' comments and events to the end, including beyond +ten pages, before replacing a document. Explicit rate limits retry the current +request at most three times with waits of at most 30 seconds each; longer hints +fail immediately. Later ordinary job retries and source schedules do not preserve +the provider's `Retry-After` deadline. Jira Server and Data Center require the +search `total` to detect premature pagination endings; Jira Cloud, which removed +that search and gives no total, pages `search/jql` by token and requires a token +on every page before `isLast` [#1453]. Custom batch progression is separate work in #1397 +[0072, revision 2026-10-23]. The source API carries the checkpoint on the +list and on the create/update responses, so an operator can see where a round +stopped rather than only that it succeeded; a kind that does not read by +timestamp has NULL there, which says "no checkpoint" rather than a time of zero +[0075]. + **Statements.** A `statements` source takes the open extraction contract itself (`e`, `s`, `n`) on `POST /sources/{id}/statements`, with the `api` push's identity, versions, tombstones and run history; the payload is the document and its single chunk, and extraction parses the chunk instead of prompting a model, then runs the unchanged path, so a pushed statement is an open statement with the item as its evidence (null offsets) and reaches the typed graph only through alignment. The contract has no slot for a property or a class and unknown keys are refused; a second push under the same identity marks the earlier statements stale rather than closing them; tables belong on a mount, not here [0054]. **RSS.** Observations are not documents: one table with baseline, candidate and no_source rows; diff --git a/migrations/0117_a_person_can_retime_a_conflict.sql b/migrations/0117_a_person_can_retime_a_conflict.sql new file mode 100644 index 000000000..ffbe271e2 --- /dev/null +++ b/migrations/0117_a_person_can_retime_a_conflict.sql @@ -0,0 +1,6 @@ +-- ADR 0043's human slice: changing the successor's start is a decision of its own. +ALTER TABLE fact_conflicts DROP CONSTRAINT fact_conflicts_resolution_check; +ALTER TABLE fact_conflicts ADD CONSTRAINT fact_conflicts_resolution_check + CHECK (resolution IS NULL OR resolution IN ('closed', 'retimed_new', 'kept_both', 'rejected_new')); +COMMENT ON COLUMN fact_conflicts.resolution IS + 'Human decisions: closed | retimed_new | kept_both | rejected_new; withdrawn has NULL (0043, 0051)'; diff --git a/migrations/0119_a_source_checkpoint_ends_at_a_timestamp_boundary.sql b/migrations/0119_a_source_checkpoint_ends_at_a_timestamp_boundary.sql new file mode 100644 index 000000000..5659156b0 --- /dev/null +++ b/migrations/0119_a_source_checkpoint_ends_at_a_timestamp_boundary.sql @@ -0,0 +1,4 @@ +-- 不从旧的 ok 审计记录回填：旧版本可能只读完前 200 条就记成功。 +-- GitHub/Jira 从 NULL 开始恢复扫描，保留全部文档、版本、任务与审计历史。 +-- Custom 的批次推进由 #1397 单独实现，这里不改变它的游标。 +ALTER TABLE sources ADD COLUMN sync_checkpoint timestamptz; diff --git a/migrations/0122_fact_evidence_is_found_by_document.sql b/migrations/0122_fact_evidence_is_found_by_document.sql new file mode 100644 index 000000000..8f4c28dd2 --- /dev/null +++ b/migrations/0122_fact_evidence_is_found_by_document.sql @@ -0,0 +1,6 @@ +-- 身份文档每次抽取都按 document_id 清理名字向量，来源开启身份名时也按文档找证据。 +-- 原主键 (fact_id, chunk_id) 服务的是从事实溯源，不能定位一篇文档的证据；没有这个 +-- 反向入口，一批逐篇推送会反复扫描整张 fact_evidence（#1408）。 +-- 有下游分支早已建过同名同形的索引，所以 IF NOT EXISTS，那里是空操作； +-- 不用 CONCURRENTLY：理由同 0094；而且并发建失败会留下无效索引，下次 IF NOT EXISTS 会把它跳过。 +CREATE INDEX IF NOT EXISTS fact_evidence_document_idx ON fact_evidence (document_id); diff --git a/migrations/0123_human_phrase_materialization_is_intake.sql b/migrations/0123_human_phrase_materialization_is_intake.sql new file mode 100644 index 000000000..f5c2e239b --- /dev/null +++ b/migrations/0123_human_phrase_materialization_is_intake.sql @@ -0,0 +1,12 @@ +-- 人判定短语后的物化只读写库，不等模型，不能排在抽取积压后面（#1351）。 +-- 优先级 0 也让它能用 #1349 留给非对话任务的两个名额。 +CREATE OR REPLACE FUNCTION job_priority(kind text) RETURNS smallint +LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $$ + SELECT CASE WHEN kind IN ('sync_source', 'process_document', 'hydrate_rss_entry', + 'embed_ontology', 'memory_ingest', 'build_vector_index', + 'materialize_inferences', 'materialize_typed', 'noop') + THEN 0::smallint ELSE 1::smallint END +$$; + +-- IMMUTABLE 函数改了不会更新已经存下的索引键；旧的物化任务也必须按新优先级被找到。 +REINDEX INDEX jobs_priority_claim_idx; diff --git a/migrations/0125_a_merge_keeps_the_evidence_of_the_fact_it_dedupes.sql b/migrations/0125_a_merge_keeps_the_evidence_of_the_fact_it_dedupes.sql new file mode 100644 index 000000000..296e52bce --- /dev/null +++ b/migrations/0125_a_merge_keeps_the_evidence_of_the_fact_it_dedupes.sql @@ -0,0 +1,6 @@ +-- 合并作废重复事实之前，把它的证据抄到存活的那条上（#1426）：身份来源的名字证据 +-- 否则留在作废行上，存活方不再是身份实体。抄过去的 (fact_id, chunk_id) 按位成对记在这里， +-- 撤回合并只删这些，存活行原有的证据不动 +ALTER TABLE entity_merges + ADD COLUMN copied_evidence_facts UUID[] NOT NULL DEFAULT '{}', + ADD COLUMN copied_evidence_chunks UUID[] NOT NULL DEFAULT '{}'; diff --git a/migrations/9008_embed_names_and_typed_materialization_share_the_fast_lane.sql b/migrations/9008_embed_names_and_typed_materialization_share_the_fast_lane.sql new file mode 100644 index 000000000..7e78d2dfc --- /dev/null +++ b/migrations/9008_embed_names_and_typed_materialization_share_the_fast_lane.sql @@ -0,0 +1,27 @@ +-- re:cinq fork only (RE-CINQ.md: fork migrations are numbered from 9000 and idempotent). +-- +-- 上游 0123 把 materialize_typed 列进 job_priority 的快道，9003 把 embed_names 列进去；两份 +-- 函数体各自只认自己那个。新库按 0123、9003 的顺序跑，留下的是 9003 的函数（没有 +-- materialize_typed）；已经跑过 9003 的库再跑 0123，留下的是上游的函数（没有 embed_names）。 +-- 这里把两个都列上，两种顺序到头都一样。 +-- +-- 函数体已经两个都有时什么都不动。jobs_priority_claim_idx 是 IMMUTABLE 函数的表达式索引， +-- 函数一变就要重建（同 9003、0123）。 +DO $$ +DECLARE + body text := (SELECT prosrc FROM pg_proc WHERE proname = 'job_priority'); +BEGIN + IF body LIKE '%embed_names%' AND body LIKE '%materialize_typed%' THEN + RETURN; + END IF; + CREATE OR REPLACE FUNCTION job_priority(kind text) RETURNS smallint + LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $f$ + SELECT CASE WHEN kind IN ('sync_source', 'process_document', 'hydrate_rss_entry', + 'embed_ontology', 'memory_ingest', 'build_vector_index', + 'materialize_inferences', 'materialize_typed', 'noop', + 'embed_names') + THEN 0::smallint ELSE 1::smallint END + $f$; + REINDEX INDEX jobs_priority_claim_idx; +END +$$; diff --git a/web/src/api.ts b/web/src/api.ts index 1b91f3e99..a947713fd 100644 --- a/web/src/api.ts +++ b/web/src/api.ts @@ -267,6 +267,10 @@ export interface SourceView { last_sync_status: "never" | "queued" | "running" | "ok" | "failed"; last_sync_error: string | null; last_sync_added: number; + /** 工单来源读到了哪里（ADR 0072）：消费到的更新时刻，且那一刻的整组都读完了。 + * null = 没有游标（首轮同步，或这一类来源不用游标）。说的是「读到哪」， + * 不是 `last_sync_at` 的「这一轮什么时候跑」 */ + sync_checkpoint: string | null; doc_count: number; missing_count: number; /** 全文补全那一块；**不是 RSS 全文来源的就是 null**，不用再拿 kind 与 @@ -972,10 +976,12 @@ export interface ConflictItem { old_subject: string; old_object: string | null; old_valid_from: string | null; + old_valid_from_precision: string | null; new_fact_id: string; new_subject: string; new_object: string | null; new_valid_from: string | null; + new_valid_from_precision: string | null; new_confidence: number; } @@ -2605,9 +2611,11 @@ export const api = { kbId: string, conflictId: string, body: { - action: "close" | "keep" | "reject_new"; + action: "close" | "retime_new" | "keep" | "reject_new"; close_at?: string; close_at_precision?: string; + new_from?: string; + new_from_precision?: string; }, ) => request<{ ok: boolean }>(`/api/v1/kbs/${kbId}/conflicts/${conflictId}`, { @@ -2818,19 +2826,20 @@ export const api = { }, ), - testSettings: (workspaceId: string) => + /** 只测 `card` 那一张卡，回来的也只有那一项（#1424） */ + testSettings: (workspaceId: string, card: "chat" | "embed" | "ocr" | "transcribe") => request<{ /** `ok` only when all three request shapes the jobs send pass; `checks` has each one (#1354) */ - chat: { + chat?: { ok: boolean; reply?: string; error?: string; checks?: Record<"plain" | "streaming" | "tools", { ok: boolean; timed_out?: boolean; error?: string }>; }; - embed: { ok: boolean; dim?: number; error?: string }; + embed?: { ok: boolean; dim?: number; error?: string }; ocr?: { ok: boolean; version?: string | null; error?: string }; transcribe?: { ok: boolean; error?: string }; - }>(`/api/v1/workspaces/${workspaceId}/settings/test`, { method: "POST" }), + }>(`/api/v1/workspaces/${workspaceId}/settings/test?card=${card}`, { method: "POST" }), /** 读扫描件的服务、转写模型各自一个保存：存它们不碰对话与嵌入那几列。 * `requeued`：因为缺它而等着的文件，这一存重新排进了处理队列几份 */ saveOcrSettings: ( diff --git a/web/src/docs/ingest.md b/web/src/docs/ingest.md index a4db0cd30..5b99955a7 100644 --- a/web/src/docs/ingest.md +++ b/web/src/docs/ingest.md @@ -225,6 +225,8 @@ Content-Type: application/json `q` is required. `top_k` defaults to 10 and is capped at 50; `offset` defaults to 0. `as_of` (an RFC 3339 instant) reads the base as it was then; ranks are counted before `as_of` filters the hits. Any other field is a `422`. +To search one source, add its `source_id`; a source of another base is a `404`. To search part of what you pushed, add `external_id_prefix` as well, for example `"docs/"`: only documents whose `external_id` starts with it are searched, and `_` or `%` in it are plain characters. The prefix needs the `source_id` of an API or Statements source, otherwise it is a `422` (`prefix_needs_push_source`). For one exact `external_id`, read the document back as above. Both filters apply inside each channel before the hits are fused, so paging works the same with them as without. + ```json { "results": [ diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index ddadca68f..c9ad5b4dd 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -118,6 +118,7 @@ export const en = { unknown_entity_type: "That class is not in this ontology.", nothing_to_update: "Nothing to change.", self_merge: "An entity cannot be merged into itself.", + new_from_required: "Enter the new start date for this fact.", close_at_required: "Pick the date this fact ended — the new one does not say when it started.", empty_query: "Type something to search for.", @@ -2181,6 +2182,7 @@ export const en = { "fact.nod_rejected": "Rejected a remembered fact", "fact.close": "Closed", "conflict.close_old": "Closed old", + "conflict.retime_new": "Retimed new start", "conflict.keep_both": "Kept both", "conflict.reject_new": "Rejected new", "merge.revert": "Reverted merge", @@ -2197,6 +2199,8 @@ export const en = { escalate_sample: "Sampled for a person; the adjudicator was confident", /* 执行闸门（0027）：合并会立刻送出图外的东西，把握再高也留给人 */ escalate_impact: "Held for a person; the merge would not stay in the graph", + escalate_identity: + "Held for a person; a source treats one side's names as identifiers", proposed: "The agent looked and left a proposal", governed: "Decided by the agent from precedent", namesake: "Two entities with this name in one document", @@ -2442,6 +2446,8 @@ export const en = { conflictSince: (d: string) => `since ${d}`, closeOld: "Close old", closeOldAt: (d: string) => `Close old at ${d}`, + retimeNew: "Retime new start", + conflictDate: "Date to close the old fact or retime the new start", keepBoth: "Keep both", rejectNew: "Reject new", closeAtPlaceholder: "2024-06-15 · 2024-06 · 2024 · 2024-06-15T14:32Z", diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts index d7d531dbb..b8f99faad 100644 --- a/web/src/i18n/zh.ts +++ b/web/src/i18n/zh.ts @@ -109,6 +109,7 @@ export const zh: Strings = { unknown_entity_type: "这个类不在当前本体里。", nothing_to_update: "没有要改的内容。", self_merge: "实体不能合并到自己身上。", + new_from_required: "填写这条事实的新起点日期。", close_at_required: "选一个这条事实结束的日期——新的那条没说自己何时开始。", empty_query: "输入点什么再检索。", no_data_sources: "这个知识库没有挂载任何数据库。", @@ -1908,6 +1909,7 @@ export const zh: Strings = { "fact.nod_rejected": "已驳回一条记忆事实", "fact.close": "已闭合", "conflict.close_old": "闭合旧的", + "conflict.retime_new": "重定了新事实起点", "conflict.keep_both": "两条都留", "conflict.reject_new": "驳回新的", "merge.revert": "撤销合并", @@ -1921,6 +1923,7 @@ export const zh: Strings = { escalate_unsure: "裁决的把握不够", escalate_sample: "抽给人看的一份；裁决器本有把握", escalate_impact: "留给人；这次合并不会只留在图里", + escalate_identity: "留给人；来源把一侧的名字声明为身份标识", proposed: "agent 看过了，留了一条建议", governed: "agent 按先例裁的", namesake: "同一篇文档里有两个同名实体", @@ -2136,6 +2139,8 @@ export const zh: Strings = { conflictSince: (d: string) => `自 ${d} 起`, closeOld: "闭合旧的", closeOldAt: (d: string) => `在 ${d} 闭合旧的`, + retimeNew: "重定新事实起点", + conflictDate: "闭合旧事实或重定新事实起点的日期", keepBoth: "两条都留", rejectNew: "驳回新的", closeAtPlaceholder: "2024-06-15 · 2024-06 · 2024 · 2024-06-15T14:32Z", diff --git a/web/src/objectValue.ts b/web/src/objectValue.ts index f650295c3..359524fed 100644 --- a/web/src/objectValue.ts +++ b/web/src/objectValue.ts @@ -15,9 +15,13 @@ export function fmtObjectValue(v: Record | null): string | null 因为值要能比大小才有资格不当节点；可原样念出来是「5000000000 $」， 比原文那句「$5 billion」难读得多。符号在前、数收成紧凑写法，两头都要 */ if (typeof v.value === "number" && unit && unit !== "%") { + /* 两位小数只给紧凑写法：值是按声明单位读出来的，「350万元」进单位亿元的属性是 + 0.035，舍到两位就显示成 0.04，「3万元」成了 0。普通写法留十五位有效数， + 也不把规则算术的 0.1 + 0.2 显示成 0.30000000000000004 */ + const compact = Math.abs(v.value) >= 10000; const n = new Intl.NumberFormat(undefined, { - notation: Math.abs(v.value) >= 10000 ? "compact" : "standard", - maximumFractionDigits: 2, + notation: compact ? "compact" : "standard", + ...(compact ? { maximumFractionDigits: 2 } : { maximumSignificantDigits: 15 }), }).format(v.value); // Currency symbols precede an amount; physical units follow a quantity. return /\p{Sc}/u.test(unit) ? `${unit}${n}` : `${n} ${unit}`; diff --git a/web/src/objectValue.units.test.ts b/web/src/objectValue.units.test.ts index 351a93477..0d5424ded 100644 --- a/web/src/objectValue.units.test.ts +++ b/web/src/objectValue.units.test.ts @@ -5,6 +5,14 @@ describe("adopted physical quantities", () => { it.each(["kg", "MW", "km", "m²"])("keeps %s after the numeric quantity", (unit) => { expect(fmtObjectValue({ value: 42, unit })).toBe(`42 ${unit}`); }); + it("keeps the digits of a small quantity read into a larger declared unit", () => { + // 「350万元」读进单位亿元的属性是 0.035：只给两位小数时显示成 0.04，「3万元」成了 0 + expect(fmtObjectValue({ value: 0.035, unit: "亿元" })).toBe("0.035 亿元"); + expect(fmtObjectValue({ value: 0.0003, unit: "亿元" })).toBe("0.0003 亿元"); + }); + it("does not expose floating-point noise from rule arithmetic", () => { + expect(fmtObjectValue({ value: 0.1 + 0.2, unit: "kg" })).toBe("0.3 kg"); + }); it("still places currency symbols before adopted amounts", () => { expect(fmtObjectValue({ value: 42, unit: "$" })).toBe("$42"); }); diff --git a/web/src/pages/KbSettings.test.ts b/web/src/pages/KbSettings.test.ts new file mode 100644 index 000000000..14acde6a1 --- /dev/null +++ b/web/src/pages/KbSettings.test.ts @@ -0,0 +1,220 @@ +import { + Children, isValidElement, useState, type ChangeEvent, type ComponentProps, + type JSXElementConstructor, type ReactNode, +} from "react"; +import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { api, type Kb } from "../api"; +import { S } from "../i18n"; +import { Button, Checkbox, Input, Segmented, SettingsCard } from "../ui"; +import { KbSettings } from "./KbSettings"; + +// Node 中执行真实组件与保存回调；查询快照模拟刷新边界，不覆盖 TanStack 缓存集成。 +const hooks = vi.hoisted(() => ({ + kbId: "kb-a", + data: undefined as Kb | undefined, + query: undefined as { queryFn: () => Promise } | undefined, + invalidate: vi.fn(), + states: [] as unknown[], + dependencies: [] as Array, + effects: [] as Array<() => void>, + stateIndex: 0, + effectIndex: 0, + changed: false, + saving: undefined as Promise | undefined, +})); +vi.mock("react", async (original) => ({ + ...await original(), + useState: (initial: unknown) => { + const index = hooks.stateIndex++; + if (!(index in hooks.states)) hooks.states[index] = initial; + return [hooks.states[index], (value: unknown) => { + hooks.changed ||= !Object.is(hooks.states[index], value); + hooks.states[index] = value; + }]; + }, + useEffect: (effect: () => void, dependencies: readonly unknown[]) => { + const previous = hooks.dependencies[hooks.effectIndex]; + hooks.dependencies[hooks.effectIndex++] = dependencies; + if (!previous || dependencies.some((value, index) => !Object.is(value, previous[index]))) { + hooks.effects.push(effect); + } + }, +})); +vi.mock("@tanstack/react-router", () => ({ + useParams: () => ({ kbId: hooks.kbId }), + useNavigate: () => vi.fn(), +})); +vi.mock("@tanstack/react-query", () => ({ + useQueryClient: () => ({ invalidateQueries: hooks.invalidate }), + useQuery: (query: { queryKey: string[]; queryFn: () => Promise }) => { + if (query.queryKey[0] !== "kbOne") return { data: { failed: 0 } }; + hooks.query = query; + return { data: hooks.data, isPending: !hooks.data }; + }, + useMutation: (options: { + mutationFn: (patch: Record) => Promise; + onSuccess: (result: unknown) => void; + }) => { + const [status, setStatus] = useState({ isPending: false, isSuccess: false }); + return { + ...status, + mutate: (patch: Record) => { + setStatus({ isPending: true, isSuccess: false }); + hooks.saving = options.mutationFn(patch).then((result) => { + options.onSuccess(result); + setStatus({ isPending: false, isSuccess: true }); + }); + }, + }; + }, +})); + +const initial: Kb = { + id: "kb-a", workspace_id: "workspace", name: "Research", kind: "general", + description: "Original description", visibility: "open", is_default: false, + auto_extend_ontology: false, materialize_inferences: false, + auto_type_resolution: false, governance: false, governance_since: null, + inference_interval_minutes: 60, last_inference_at: null, ontology_lang: "en", +}; +let server: Kb; + +beforeEach(() => { + server = { ...initial }; + Object.assign(hooks, { + kbId: initial.id, data: undefined, query: undefined, + states: [], dependencies: [], effects: [], saving: undefined, + }); + hooks.invalidate.mockReset(); + vi.spyOn(api, "kbDetail").mockImplementation(async () => ({ ...server })); + vi.spyOn(api, "updateKb").mockImplementation(async (_id, patch) => { + server = { ...server, ...patch }; + return server; + }); +}); +afterEach(() => vi.restoreAllMocks()); + +async function refresh() { + render(); + expect(hooks.query).toBeDefined(); + hooks.data = await hooks.query!.queryFn(); +} + +function render(): ReactNode { + hooks.stateIndex = hooks.effectIndex = 0; + hooks.changed = false; + const tree = KbSettings(); + for (const effect of hooks.effects.splice(0)) effect(); + return hooks.changed ? render() : tree; +} + +function find

(node: ReactNode, type: JSXElementConstructor

| string): P[] { + if (!isValidElement<{ children?: ReactNode; action?: ReactNode }>(node)) return []; + // JSX 的 type 与组件一致时，props 就是该组件的公开属性。 + const matched = node.type === type ? [node.props as P] : []; + return [...matched, ...Children.toArray([node.props.children, node.props.action]) + .flatMap((child) => find(child, type))]; +} + +function card(title: string) { + const found = find(render(), SettingsCard).find((item) => item.title === title); + expect(found, title).toBeDefined(); + return found!; +} + +function editIdentity(name: string, description: string) { + const inputs = find(card(S.kbset.cardIdentity).children, Input); + for (const [index, value] of [name, description].entries()) { + inputs[index].onChange!({ target: { value } } as ChangeEvent); + } +} + +async function save(title: string) { + const button = find(card(title).action, Button)[0]; + expect(button.disabled).toBe(false); + button.onClick!({} as Parameters["onClick"]>>[0]); + await hooks.saving; + expect(hooks.invalidate.mock.calls).toEqual([ + [{ queryKey: ["kbOne", hooks.kbId] }], [{ queryKey: ["kbs"] }], + ]); + await refresh(); +} + +function expectSaved(title: string) { + const action = card(title).action; + expect(find(action, Button)[0].disabled).toBe(true); + expect(find<{ children?: ReactNode }>(action, "span") + .some((span) => span.children === S.kbset.saved)).toBe(true); +} + +it.each(["Original description", "", null])( + "keeps identity drafts when only visibility is saved (description: %s)", + async (description) => { + server.description = description; + await refresh(); + editIdentity("Draft name", "Draft description"); + find(card(S.settings.kbs.visibility).children, Segmented)[0].onChange("restricted"); + await save(S.settings.kbs.visibility); + + expect(api.updateKb).toHaveBeenCalledExactlyOnceWith("kb-a", { visibility: "restricted" }); + expect(api.kbDetail).toHaveBeenCalledTimes(2); + expect(find(card(S.kbset.cardIdentity).children, Input).map((input) => input.value)) + .toEqual(["Draft name", "Draft description"]); + expect(find(card(S.settings.kbs.visibility).children, Segmented)[0].value).toBe("restricted"); + expectSaved(S.settings.kbs.visibility); + }, +); + +it("synchronizes saved identity values and status while preserving other cards' drafts", async () => { + await refresh(); + editIdentity(" New name ", " New description "); + find(card(S.settings.kbs.visibility).children, Segmented)[0].onChange("restricted"); + find(card(S.kbset.cardAutomation).children, Checkbox)[0].onChange!(true); + find(card(S.kbset.ontologyLang).children, Segmented)[0].onChange("zh"); + await save(S.kbset.cardIdentity); + + expect(api.updateKb).toHaveBeenLastCalledWith("kb-a", { name: "New name", description: "New description" }); + expect(find(card(S.kbset.cardIdentity).children, Input).map((input) => input.value)) + .toEqual(["New name", "New description"]); + expectSaved(S.kbset.cardIdentity); + expect(find(card(S.settings.kbs.visibility).children, Segmented)[0].value).toBe("restricted"); + expect(find(card(S.kbset.cardAutomation).children, Checkbox)[0].checked).toBe(true); + expect(find(card(S.kbset.ontologyLang).children, Segmented)[0].value).toBe("zh"); +}); + +it("accepts server updates on unedited cards", async () => { + await refresh(); + render(); + server = { + ...server, name: "Renamed remotely", description: "Updated remotely", visibility: "restricted", + materialize_inferences: true, auto_type_resolution: true, governance: true, + inference_interval_minutes: 120, ontology_lang: "zh", + }; + await refresh(); + expect(find(card(S.kbset.cardIdentity).children, Input).map((input) => input.value)) + .toEqual([server.name, server.description]); + expect(find(card(S.settings.kbs.visibility).children, Segmented)[0].value).toBe("restricted"); + expect(find(card(S.kbset.cardAutomation).children, Checkbox).map((input) => input.checked)) + .toEqual([true, true, true]); + expect(find(card(S.kbset.cardAutomation).children, Input)[0].value).toBe(120); + expect(find(card(S.kbset.ontologyLang).children, Segmented)[0].value).toBe("zh"); +}); + +it.each([false, true])("resets drafts when switching KB with identical server values (preloaded: %s)", async (preloaded) => { + await refresh(); + editIdentity("Draft name", "Draft description"); + find(card(S.settings.kbs.visibility).children, Segmented)[0].onChange("restricted"); + find(card(S.kbset.cardAutomation).children, Checkbox)[0].onChange!(true); + find(card(S.kbset.ontologyLang).children, Segmented)[0].onChange("zh"); + render(); + hooks.kbId = "kb-b"; + server = { ...initial, id: "kb-b" }; + hooks.data = preloaded ? { ...server } : undefined; + render(); + await refresh(); + expect(find(card(S.kbset.cardIdentity).children, Input).map((input) => input.value)) + .toEqual([initial.name, initial.description]); + expect(find(card(S.settings.kbs.visibility).children, Segmented)[0].value).toBe("open"); + expect(find(card(S.kbset.cardAutomation).children, Checkbox).map((input) => input.checked)) + .toEqual([false, false, false]); + expect(find(card(S.kbset.ontologyLang).children, Segmented)[0].value).toBe("en"); +}); diff --git a/web/src/pages/KbSettings.tsx b/web/src/pages/KbSettings.tsx index b8478e20c..8a30f5aad 100644 --- a/web/src/pages/KbSettings.tsx +++ b/web/src/pages/KbSettings.tsx @@ -125,18 +125,40 @@ export function KbSettings() { const [confirmingDelete, setConfirmingDelete] = useState(false); const [error, setError] = useState(null); + // 每张卡只跟随自己的服务端字段；保存别的卡触发重取时，保留这张卡的草稿。 useEffect(() => { if (kb.data) { setName(kb.data.name); setDesc(kb.data.description ?? ""); + } + }, [kbId, kb.data?.name, kb.data?.description]); + + useEffect(() => { + if (kb.data) { setVisibility(kb.data.visibility); + } + }, [kbId, kb.data?.visibility]); + + useEffect(() => { + if (kb.data) { setMaterialize(kb.data.materialize_inferences); setAutoResolve(kb.data.auto_type_resolution); setGovernance(kb.data.governance); setInferMins(kb.data.inference_interval_minutes); + } + }, [ + kbId, + kb.data?.materialize_inferences, + kb.data?.auto_type_resolution, + kb.data?.governance, + kb.data?.inference_interval_minutes, + ]); + + useEffect(() => { + if (kb.data) { setOntoLang(kb.data.ontology_lang); } - }, [kb.data]); + }, [kbId, kb.data?.ontology_lang]); // 四张卡，四个保存：改名字那一下不该把下面的开关一起送上去 const saveIdentity = useKbPatch(kbId!, setError); @@ -509,6 +531,19 @@ function auditDetailName(e: AuditEvent): string { const AUDIT_PAGE = 50; +/** 两个日期框说的是读者日历上的几天：起始那天零点起、截止那天过完为止。服务端的区间 + * 含起点不含终点、按时刻比，所以截止日要送下一天的零点，零点也按本地时区换成时刻 */ +export function auditBounds(since: string, until: string) { + const midnight = (day: string, later: number) => { + const [y, m, d] = day.split("-").map(Number); + return new Date(y, m - 1, d + later).toISOString(); + }; + return { + since: since ? midnight(since, 0) : undefined, + until: until ? midnight(until, 1) : undefined, + }; +} + function KbActivity({ kbId }: { kbId: string }) { // 筛选按真实查法来：查一类动作、查一个人、查一段时间。 // 动作前缀匹配——`entity.` 就能把 retyped / renamed 一族一起捞出来 @@ -521,8 +556,7 @@ function KbActivity({ kbId }: { kbId: string }) { queryFn: () => api.kbAudit(kbId, { action: action || undefined, - since: since || undefined, - until: until || undefined, + ...auditBounds(since, until), limit: AUDIT_PAGE, offset: page * AUDIT_PAGE, }), diff --git a/web/src/pages/Review.tsx b/web/src/pages/Review.tsx index ac6c213a6..ef6c00c6f 100644 --- a/web/src/pages/Review.tsx +++ b/web/src/pages/Review.tsx @@ -26,7 +26,7 @@ import { type ReviewSide, type ViolationResolution, } from "../api"; -import { parseDateInput } from "../time"; +import { fmtTime, parseDateInput } from "../time"; import { PendingFactRow, useCanDecide } from "./PendingFacts"; import { ReviewOverview } from "./ReviewOverview"; import { S } from "../i18n"; @@ -305,7 +305,7 @@ function ConflictRow({ conflict: ConflictItem; busy: boolean; onResolve: ( - action: "close" | "keep" | "reject_new", + action: "close" | "retime_new" | "keep" | "reject_new", closeAt?: string, closeAtPrecision?: string, ) => void; @@ -316,6 +316,7 @@ function ConflictRow({ // 写多少位就是多少精度（time.ts）：「2023-06」闭合在那个月，不编一个 1 日 const closeParsed = parseDateInput(closeAt); const closeAtIso = closeParsed?.iso; + const invalidDate = closeAt.trim() !== "" && !closeParsed; return (

@@ -329,7 +330,7 @@ function ConflictRow({ {c.old_valid_from && ( - ({S.review.conflictSince(c.old_valid_from.slice(0, 10))}) + ({S.review.conflictSince(fmtTime(c.old_valid_from, c.old_valid_from_precision)!)}) )} {S.review.conflictVs} @@ -342,7 +343,7 @@ function ConflictRow({ {c.new_valid_from && ( - ({S.review.conflictSince(c.new_valid_from.slice(0, 10))}) + ({S.review.conflictSince(fmtTime(c.new_valid_from, c.new_valid_from_precision)!)}) )} @@ -356,20 +357,27 @@ function ConflictRow({ > {S.review.keepBoth} - {needsDate && ( - setCloseAt(e.target.value)} - /> - )} + setCloseAt(e.target.value)} + /> +