From 8ec78631fd31741987fc0e8fc10f818f1cedc735 Mon Sep 17 00:00:00 2001 From: liuedcson <332840681+liuedcson@users.noreply.github.com> Date: Sat, 17 Oct 2026 12:16:02 +0800 Subject: [PATCH 1/2] feat(coding-agent): add SQLite session search index Add a derived, rebuildable SQLite index over session JSONL files for session lists and full-text search, for Web and Desktop hosts. JSONL stays the only authoritative data: the index only reads session files and can be deleted at any time. Index: - node:sqlite with FTS5 trigram at /index/session-search.v1.db; application_id STCX, schema major in the file name, minor in user_version. - One worker thread owns the connection. Opening never rejects: a foreign file, missing SQLite/FTS5, or a dead worker falls back to a JSONL scan with session-level hits and status "degraded". - Incremental sync verifies header and tail hashes before appending; rewritten, shrunk, v1, or older-projector files are rebuilt in one BEGIN IMMEDIATE transaction with a snapshot check. - Queries of 3+ characters use FTS with bm25; shorter ones use LIKE over the 200 most recent sessions, at most 50 hits. - Before each search or list, a 30 ms freshness pass re-syncs the 20 most recent files and lists directories whose mtime changed, so sessions written by other processes appear without notifications. Integration: - SessionManager.setFilePersistedListener() reports create/append/rewrite. - attachSessionIndex() and withSessionIndex() wire managers and factories. - createSessionSearchService() exposes search.query, sessions.list, and index.status without choosing a transport, validating untrusted params. - `step session-index status | reindex [--full] | clear`. - The CLI does not open the index during interactive or print sessions. - The worker ships beside client.js in dist and the Node bundle, and as an extra entry point in compiled Bun binaries. docs/session-index.md describes storage, upgrades, and the steps for a host to adopt the index. --- apps/cli/src/main.ts | 13 +- packages/coding-agent/docs/docs.json | 4 + packages/coding-agent/docs/index.md | 1 + packages/coding-agent/docs/session-format.md | 1 + packages/coding-agent/docs/session-index.md | 137 +++ packages/coding-agent/package.json | 2 +- .../src/core/session-index/attach.ts | 30 + .../src/core/session-index/client.ts | 370 ++++++++ .../src/core/session-index/command.ts | 92 ++ .../src/core/session-index/fallback.ts | 114 +++ .../src/core/session-index/index.ts | 24 + .../src/core/session-index/projector.ts | 114 +++ .../src/core/session-index/reader.ts | 206 ++++ .../src/core/session-index/schema.ts | 331 +++++++ .../session-index/session-index-worker.ts | 230 +++++ .../src/core/session-index/store.ts | 890 ++++++++++++++++++ .../src/core/session-index/types.ts | 102 ++ .../coding-agent/src/core/session-manager.ts | 112 +-- .../coding-agent/src/core/session-summary.ts | 108 +++ packages/coding-agent/src/index.ts | 18 +- .../src/server/session-search-service.ts | 181 ++++ .../test/session-index/client.test.ts | 189 ++++ .../test/session-index/service.test.ts | 121 +++ .../test/session-index/store.test.ts | 395 ++++++++ scripts/build-binaries.sh | 4 +- scripts/build-coding-agent-bundle.mjs | 16 +- 26 files changed, 3730 insertions(+), 75 deletions(-) create mode 100644 packages/coding-agent/docs/session-index.md create mode 100644 packages/coding-agent/src/core/session-index/attach.ts create mode 100644 packages/coding-agent/src/core/session-index/client.ts create mode 100644 packages/coding-agent/src/core/session-index/command.ts create mode 100644 packages/coding-agent/src/core/session-index/fallback.ts create mode 100644 packages/coding-agent/src/core/session-index/index.ts create mode 100644 packages/coding-agent/src/core/session-index/projector.ts create mode 100644 packages/coding-agent/src/core/session-index/reader.ts create mode 100644 packages/coding-agent/src/core/session-index/schema.ts create mode 100644 packages/coding-agent/src/core/session-index/session-index-worker.ts create mode 100644 packages/coding-agent/src/core/session-index/store.ts create mode 100644 packages/coding-agent/src/core/session-index/types.ts create mode 100644 packages/coding-agent/src/core/session-summary.ts create mode 100644 packages/coding-agent/src/server/session-search-service.ts create mode 100644 packages/coding-agent/test/session-index/client.test.ts create mode 100644 packages/coding-agent/test/session-index/service.test.ts create mode 100644 packages/coding-agent/test/session-index/store.test.ts diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts index afef34e7..d3dc737b 100644 --- a/apps/cli/src/main.ts +++ b/apps/cli/src/main.ts @@ -7,7 +7,7 @@ // very first side effect (before the coding-agent barrel is evaluated) so // config.ts resolves the Step storage namespace regardless of launcher name. import "#bootstrap/environment"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { ambiguousPluginServerNames, @@ -30,6 +30,7 @@ import { getStepAuthPath, getStepDefaultTheme, getStepLoginStatus, + getStepSessionDirOverride, hasConfiguredStepCodeCredential, installProcessStderrDevLogCapture, isStepConfigCommand, @@ -60,6 +61,7 @@ import { resolveStepStorageRoot, restoreStdout, runFeedbackCommand, + runSessionIndexCommand, runStepConfigCommand, runStepLogin, runStepMcpImportPrompt, @@ -353,6 +355,7 @@ const isTopLevelLogin = topLevelStepCommand === "login"; const isTopLevelLoginStatus = isTopLevelLogin && process.argv[3] === "status"; const isTopLevelLogout = topLevelStepCommand === "logout"; const isTopLevelFeedback = topLevelStepCommand === "feedback"; +const isTopLevelSessionIndex = topLevelStepCommand === "session-index"; const topLevelAuthHelp = process.argv.slice(3).some((arg) => arg === "--help" || arg === "-h"); let telemetryExitReason: "normal" | "error" = "normal"; @@ -659,6 +662,14 @@ try { } } } + } else if (isTopLevelSessionIndex) { + const sessionDirOverride = getStepSessionDirOverride(process.env); + const exitCode = await runSessionIndexCommand(process.argv.slice(3), { + agentDir: resolveStepAgentDir(), + ...(sessionDirOverride ? { extraRoots: [resolve(sessionDirOverride)] } : {}), + }); + if (exitCode !== 0) process.exitCode = exitCode; + forceOneShotExit = true; } else if (isTopLevelFeedback) { const feedbackSettingsManager = currentStepSettingsManager ?? createStepSettingsManager(process.cwd(), resolveStepAgentDir()); diff --git a/packages/coding-agent/docs/docs.json b/packages/coding-agent/docs/docs.json index bbc9e74b..d4bbab02 100644 --- a/packages/coding-agent/docs/docs.json +++ b/packages/coding-agent/docs/docs.json @@ -84,6 +84,10 @@ { "title": "Session Format", "path": "session-format.md" + }, + { + "title": "Session Index", + "path": "session-index.md" } ] }, diff --git a/packages/coding-agent/docs/index.md b/packages/coding-agent/docs/index.md index 1ba79715..8b1c444e 100644 --- a/packages/coding-agent/docs/index.md +++ b/packages/coding-agent/docs/index.md @@ -66,6 +66,7 @@ For the full first-run flow, see [Quickstart](quickstart.md). - [Environment variables](environment-variables.md) - Step process configuration and session metadata available to bash tools. - [Session format](session-format.md) - JSONL session file format, entry types, and SessionManager API. +- [Session index](session-index.md) - derived SQLite index for session lists and full-text search. ## Platform setup diff --git a/packages/coding-agent/docs/session-format.md b/packages/coding-agent/docs/session-format.md index 8a97a1d9..2622b877 100644 --- a/packages/coding-agent/docs/session-format.md +++ b/packages/coding-agent/docs/session-format.md @@ -438,3 +438,4 @@ Key methods for working with sessions programmatically. - `getSessionId()` - Session UUID - `getSessionFile()` - Session file path (undefined for in-memory) - `isPersisted()` - Whether session is saved to disk +- `setFilePersistedListener(listener)` - Observe `create` / `append` / `rewrite` writes, e.g. for the [session index](session-index.md) diff --git a/packages/coding-agent/docs/session-index.md b/packages/coding-agent/docs/session-index.md new file mode 100644 index 00000000..d08c72ca --- /dev/null +++ b/packages/coding-agent/docs/session-index.md @@ -0,0 +1,137 @@ +# Session index + +The session index is a derived SQLite database for session lists and full-text search. Session JSONL files stay the only authoritative data: the index only reads them, and deleting it loses nothing but the time to rebuild it. + +## Storage and lifecycle + +| Item | Value | +| --- | --- | +| Location | `/index/session-search.v1.db` (plus `-wal` / `-shm`); the schema major is part of the name | +| Engine | Built-in `node:sqlite` with FTS5 `trigram`; no extra dependency | +| Owner | One worker thread per process holds the only connection. The CLI does not open the index during interactive or print sessions; only `step session-index` and hosts that call `openSessionIndex()` do | +| Roots | `/sessions` and any directory added with `addRoot()` or found holding a persisted file | +| Ownership check | `PRAGMA application_id` (`STCX`); a file without our id is never modified and the index runs degraded | +| Damaged file | Unreadable, missing tables, or `schema_major` disagreeing with the name: moved aside as `session-search.v1.db.corrupt-