feat(coding-agent): add SQLite session search index

apps/cli/src/main.ts

@@ -7,7 +7,7 @@
 // very first side effect (before the coding-agent barrel is evaluated) so
 // config.ts resolves the Step storage namespace regardless of launcher name.
 import "#bootstrap/environment";
-import { join } from "node:path";
+import { join, resolve } from "node:path";
 
 import {
 	ambiguousPluginServerNames,
@@ -30,6 +30,7 @@ import {
 	getStepAuthPath,
 	getStepDefaultTheme,
 	getStepLoginStatus,
+	getStepSessionDirOverride,
 	hasConfiguredStepCodeCredential,
 	installProcessStderrDevLogCapture,
 	isStepConfigCommand,
@@ -60,6 +61,7 @@ import {
 	resolveStepStorageRoot,
 	restoreStdout,
 	runFeedbackCommand,
+	runSessionIndexCommand,
 	runStepConfigCommand,
 	runStepLogin,
 	runStepMcpImportPrompt,
@@ -353,6 +355,7 @@ const isTopLevelLogin = topLevelStepCommand === "login";
 const isTopLevelLoginStatus = isTopLevelLogin && process.argv[3] === "status";
 const isTopLevelLogout = topLevelStepCommand === "logout";
 const isTopLevelFeedback = topLevelStepCommand === "feedback";
+const isTopLevelSessionIndex = topLevelStepCommand === "session-index";
 const topLevelAuthHelp = process.argv.slice(3).some((arg) => arg === "--help" || arg === "-h");
 
 let telemetryExitReason: "normal" | "error" = "normal";
@@ -659,6 +662,14 @@ try {
 				}
 			}
 		}
+	} else if (isTopLevelSessionIndex) {
+		const sessionDirOverride = getStepSessionDirOverride(process.env);
+		const exitCode = await runSessionIndexCommand(process.argv.slice(3), {
+			agentDir: resolveStepAgentDir(),
+			...(sessionDirOverride ? { extraRoots: [resolve(sessionDirOverride)] } : {}),
+		});
+		if (exitCode !== 0) process.exitCode = exitCode;
+		forceOneShotExit = true;
 	} else if (isTopLevelFeedback) {
 		const feedbackSettingsManager =
 			currentStepSettingsManager ?? createStepSettingsManager(process.cwd(), resolveStepAgentDir());

apps/cli/src/ui/view/dialogs/session-selector.ts

@@ -6,7 +6,7 @@ import type { SessionInfo, SessionListProgress } from "@step-harness/coding-agen
 import {
 	canonicalizePath as _canonicalizePath,
 	DynamicBorder,
-	isChildAgentSessionId,
+	isChildAgentSession,
 	KeybindingsManager,
 	keyHint,
 	keyText,
@@ -954,7 +954,7 @@ export class SessionSelectorComponent extends Container implements Focusable {
 			// a respawned child still reattaches to its own transcript.
 			const sessions = (
 				await (scope === "current" ? this.currentSessionsLoader(onProgress) : this.allSessionsLoader(onProgress))
-			).filter((session) => !isChildAgentSessionId(session.id));
+			).filter((session) => !isChildAgentSession(session));
 
 			if (scope === "current") {
 				this.currentSessions = sessions;

packages/coding-agent/docs/docs.json

@@ -84,6 +84,10 @@
         {
           "title": "Session Format",
           "path": "session-format.md"
+        },
+        {
+          "title": "Session Index",
+          "path": "session-index.md"
         }
       ]
     },

packages/coding-agent/docs/index.md

@@ -66,6 +66,7 @@ For the full first-run flow, see [Quickstart](quickstart.md).
 
 - [Environment variables](environment-variables.md) - Step process configuration and session metadata available to bash tools.
 - [Session format](session-format.md) - JSONL session file format, entry types, and SessionManager API.
+- [Session index](session-index.md) - derived SQLite index for session lists and full-text search.
 
 ## Platform setup
 

packages/coding-agent/docs/session-format.md

@@ -202,6 +202,12 @@ For sessions with a parent (created via `/fork`, `/clone`, or `newSession({ pare
 {"type":"session","version":3,"id":"uuid","timestamp":"2024-12-03T14:00:00.000Z","cwd":"/path/to/project","parentSession":"/path/to/original/session.jsonl"}
 ```
 
+A session created by a child agent (a subagent lane or workflow agent, spawned with `STEPCODE_SUBAGENT_CHILD=1`) carries `origin: "subagent"` and, when known, the id of the session that spawned it. Session lists, the resume picker, and session search skip these sessions (`isChildAgentSession`, which also recognizes the `subagent-` / `workflow-` id prefixes of older child sessions):
+
+```json
+{"type":"session","version":3,"id":"subagent-uuid","timestamp":"2024-12-03T14:00:00.000Z","cwd":"/path/to/project","origin":"subagent","parentSessionId":"uuid"}
+```
+
 ### SessionMessageEntry
 
 A message in the conversation. The `message` field contains an `AgentMessage`.
@@ -401,7 +407,7 @@ Key methods for working with sessions programmatically.
 - `SessionManager.listAll(onProgress?)` - List all sessions across all projects
 
 ### Instance Methods - Session Management
-- `newSession(options?)` - Start a new session (options: `{ parentSession?: string }`)
+- `newSession(options?)` - Start a new session (options: `{ id?, parentSession?, origin?, parentSessionId? }`)
 - `setSessionFile(path)` - Switch to a different session file
 - `createBranchedSession(leafId)` - Extract branch to new session file
 
@@ -438,3 +444,4 @@ Key methods for working with sessions programmatically.
 - `getSessionId()` - Session UUID
 - `getSessionFile()` - Session file path (undefined for in-memory)
 - `isPersisted()` - Whether session is saved to disk
+- `setFilePersistedListener(listener)` - Observe `create` / `append` / `rewrite` writes, e.g. for the [session index](session-index.md)

packages/coding-agent/docs/session-index.md

@@ -0,0 +1,139 @@
+# Session index
+
+The session index is a derived SQLite database for session lists and full-text search. Session JSONL files stay the only authoritative data: the index only reads them, and deleting it loses nothing but the time to rebuild it.
+
+## Storage and lifecycle
+
+| Item | Value |
+| --- | --- |
+| Location | `<agentDir>/index/session-search.v1.db` (plus `-wal` / `-shm`); the schema major is part of the name |
+| Engine | Built-in `node:sqlite` with FTS5 `trigram`; no extra dependency |
+| Owner | One worker thread per process holds the only connection. The CLI does not open the index during interactive or print sessions; only `step session-index` and hosts that call `openSessionIndex()` do |
+| Roots | `<agentDir>/sessions` and any directory added with `addRoot()` or found holding a persisted file |
+| Ownership check | `PRAGMA application_id` (`STCX`); a file without our id is never modified and the index runs degraded |
+| Damaged file | Unreadable, missing tables, or `schema_major` disagreeing with the name: moved aside as `session-search.v1.db.corrupt-<time>` (one kept) and rebuilt |
+| Schema upgrade | Incompatible layout: new file name, the old file is left alone. Older minor (`user_version`): migrate in place. Newer minor: use known columns, no migration. Tokenizer change: rebuild FTS tables from `entries` without rereading JSONL |
+| Projection change | `projector_version` per source; older sources are reprojected on the next reconcile. Rows from a newer projector are left as is, and an older binary appending to one marks it for reprojection |
+| Metadata | `index_meta` holds `schema_major`, `fts_definition`, `projector_version`, `backfill_state`, `last_reconcile_at` |
+
+## What is indexed
+
+| Source | Indexed text |
+| --- | --- |
+| `message` with role `user` / `assistant` | Text blocks, capped at 64 KiB per entry |
+| `compaction`, `branch_summary` | `summary` |
+| `custom_message` with `display: true` | Text content |
+| `label` | Label text |
+| `session_info` and the first user message | Session title search (`entryId` is `null`) |
+| Tool results, bash output, hidden custom messages, extension entries | Not indexed |
+
+v1 files without entry ids are indexed with ids `L<physical line>`. Files are migrated to v3 in memory only.
+
+Child-agent sessions (subagent lanes, workflow agents) are indexed with `sessions.origin = 'subagent'` but are excluded from every search and list, including the short-query window and degraded mode. A session is a child when its header has `origin: "subagent"` or its id starts with `subagent-` / `workflow-` (`isChildAgentSession`); sessions written before the header marker carry only the prefix. Schema minor 2 added the column and marked existing rows by prefix.
+
+## Updates
+
+| Trigger | Behavior |
+| --- | --- |
+| `notifyPersisted({ path, kind })` | Debounced per path (200 ms), then the file is synced |
+| Append | Reads from the stored offset after checking the header and last-line hashes |
+| Header or tail changed, file shrank, v1 file | Session rows are deleted and rebuilt in one transaction |
+| Empty file or incomplete header | Marked `pending`; existing rows stay searchable |
+| Unfinished final line | Left for the next sync |
+| Unparseable complete line | Skipped like `SessionManager` does; the count is kept in `sources.error` and the source stays `ok` |
+| Reconcile (open, `reindex`) | Lists every root, syncs new or changed files newest first, and deletes rows for files that are gone from a root that was listed successfully |
+| Before each search or list | Freshness pass with a 30 ms budget: re-syncs the 20 most recently modified known files, then lists only directories whose mtime moved since the last reconcile to pick up files written by other processes. A pass cut short by the budget schedules a background reconcile. Deletions wait for the next reconcile |
+
+Writes check the source row they started from inside `BEGIN IMMEDIATE`; when another process changed it meanwhile the sync is retried once.
+
+## Search
+
+| Query | Path |
+| --- | --- |
+| Every token has 3+ characters | FTS5 `MATCH` of quoted phrases, ordered by bm25 |
+| Any token is shorter | `LIKE` on `entries.text` within the 200 most recently modified matching sessions, newest first, at most 50 hits. Without `cwd` or `since`, only the last 30 days are searched |
+
+Snippets wrap matches in `SNIPPET_MATCH_START` / `SNIPPET_MATCH_END`. Results carry `coverage` so a caller can tell a partial backfill from a complete answer.
+
+## Degraded mode
+
+When `node:sqlite`, FTS5 trigram, the database, or the worker is unavailable, `openSessionIndex()` still resolves. Searches and lists scan JSONL on each call, hits are session-level only, and `status()` returns `state: "degraded"` with the reason. A worker that fails after opening switches the same object to this mode.
+
+## API
+
+```ts
+import {
+  attachSessionIndex,
+  createStepSessionManagerFactory,
+  openSessionIndex,
+  withSessionIndex,
+} from "@step-harness/coding-agent";
+
+const index = await openSessionIndex({ agentDir });
+attachSessionIndex(sessionManager, index); // one manager
+const factory = withSessionIndex(createStepSessionManagerFactory(agentDir), index); // every manager it builds
+
+for await (const batch of index.search({ text: "migration", cwd, limit: 20 })) {
+  console.log(batch.coverage, batch.hits);
+}
+const page = await index.listSessions({ cwd, limit: 50 });
+await index.close();
+```
+
+`SessionManager.setFilePersistedListener()` reports `create`, `append`, and `rewrite` after each successful write. Listener errors are ignored; the next reconcile repairs any missed update. `attachSessionIndex()` sets that listener; `withSessionIndex()` wraps a `SessionManagerFactory` so created, opened, forked, and resumed managers are attached (in-memory managers have no file and are left alone).
+
+## Host methods
+
+`createSessionSearchService(index)` exposes the index to a Web or Desktop host without choosing a transport. `call(method, params, signal)` takes untrusted JSON params, validates them, and yields JSON-safe results; the host owns framing and the index lifetime.
+
+| Method | Params | Result |
+| --- | --- | --- |
+| `search.query` | `q` (1–500 chars), `cwd?` (absolute), `entryTypes?`, `limit?` (1–100, default 20), `since?` (epoch ms) | One or more `{ hits, coverage }` batches |
+| `sessions.list` | `cwd?`, `limit?` (1–200, default 50), `cursor?` | `{ sessions, nextCursor? }`; `created` / `modified` are epoch ms |
+| `index.status` | none | `SessionIndexStatus` |
+
+Invalid requests throw `SessionSearchRequestError` with `code` `unknown_method` or `invalid_params` before the index is queried.
+
+## Integrating a host
+
+A Web or Desktop host that wants session lists and search follows these steps. Nothing here needs changes to `SessionManager`, the JSONL format, or the CLI.
+
+| Step | What to do | Notes |
+| --- | --- | --- |
+| 1. Open once | At host startup: `openSessionIndex({ agentDir, roots })`. Roots are `defaultSessionIndexRoots(agentDir)` plus any configured session directory (`STEP_CODING_AGENT_SESSION_DIR`, `--session-dir`) as `kind: "custom"` | Never rejects. Backfill runs in the worker; the host stays responsive. Keep one index per host process and reuse it |
+| 2. Wire writes | Pass `withSessionIndex(factory, index)` as the runtime's `sessionManagerFactory`; call `attachSessionIndex(manager, index)` for a manager built outside the factory (the initial session) | Writes from this process are indexed within the 200 ms debounce. Writes from CLI processes are found by the freshness pass before each query |
+| 3. Expose methods | Route the transport's requests to `createSessionSearchService(index).call(method, params, signal)`. Map `SessionSearchRequestError.code` to the transport's error codes; abort the signal when a client cancels or retypes | Results are JSON-safe; stream each `search.query` batch as it arrives |
+| 4. Show progress | Poll `index.status`: `backfilling` with `indexed / total` drives a progress indicator; `degraded` with `degradedReason` means results come from a JSONL scan (session-level hits only). Each search batch also carries `coverage` | `reindex()` yields progress for an explicit rebuild |
+| 5. Shut down | `await index.close()` | Flushes pending notifications and stops the worker |
+
+Constraints for hosts:
+
+- Treat the database as private to the index. Do not open it, write it, or copy rows elsewhere; it may be deleted and rebuilt at any time.
+- Session data changes only through `SessionManager`; the index never writes JSONL.
+- Several processes may open the same index safely (each sync runs in `BEGIN IMMEDIATE` and rechecks its source row), but each costs a worker and a backfill pass. Prefer one long-lived host per agent directory.
+- Packaged builds must ship the worker: `session-index-worker.js` beside `client.js` for `dist/` and the Node bundle, and `src/core/session-index/session-index-worker.ts` as an extra entry point for `bun build --compile`. Without it, the index runs degraded.
+- Requires `node:sqlite` with FTS5 `trigram` (Node.js `>=22.19.0`, or the compiled Bun binary).
+
+Source map:
+
+| Path | Role |
+| --- | --- |
+| `src/core/session-index/client.ts` | `openSessionIndex`, worker RPC, degraded fallback switch |
+| `src/core/session-index/session-index-worker.ts` | Worker that owns the connection; request handling, backfill, freshness pass |
+| `src/core/session-index/store.ts` | Sync, reconcile, freshness pass, search, list |
+| `src/core/session-index/schema.ts` | DDL, open / inspect / upgrade, versions, `getSessionIndexPath` |
+| `src/core/session-index/reader.ts`, `projector.ts` | Read-only JSONL access and entry projection |
+| `src/core/session-index/fallback.ts` | JSONL scan used when the index is degraded |
+| `src/core/session-index/attach.ts` | `attachSessionIndex`, `withSessionIndex` |
+| `src/server/session-search-service.ts` | Transport-neutral host methods and parameter validation |
+| `test/session-index/` | Store, client / worker, and service tests |
+
+## CLI
+
+```bash
+step session-index status [--json]
+step session-index reindex [--full] [--json]
+step session-index clear
+```
+
+`STEP_CODING_AGENT_SESSION_DIR` is included as an extra root when set.

packages/coding-agent/package.json

@@ -28,7 +28,7 @@
 		"clean": "shx rm -rf dist",
 		"build": "npm run build:unbundled && node ../../scripts/build-coding-agent-bundle.mjs",
 		"build:unbundled": "tsgo -p tsconfig.build.json && npm run copy-assets",
-		"build:binary": "npm --prefix ../tui run build && npm --prefix ../telemetry run build && npm --prefix ../providers run build && npm --prefix ../agent-core run build && npm run build && bun build --compile --no-compile-autoload-bunfig --env=STEPCODE_BUILD_* ../../apps/cli/src/bun/stepcode.ts ./src/utils/image-resize-worker.ts ./src/step/usage/activity-worker.ts --outfile dist/step-bin && npm run copy-binary-assets",
+		"build:binary": "npm --prefix ../tui run build && npm --prefix ../telemetry run build && npm --prefix ../providers run build && npm --prefix ../agent-core run build && npm run build && bun build --compile --no-compile-autoload-bunfig --env=STEPCODE_BUILD_* ../../apps/cli/src/bun/stepcode.ts ./src/utils/image-resize-worker.ts ./src/step/usage/activity-worker.ts ./src/core/session-index/session-index-worker.ts --outfile dist/step-bin && npm run copy-binary-assets",
 		"copy-assets": "shx mkdir -p dist/theme && shx cp src/theme/*.json dist/theme/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/",
 		"copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx mkdir -p dist/theme && shx cp src/theme/*.json dist/theme/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && shx cp -r docs dist/ && shx cp -r examples dist/ && node ../../scripts/copy-photon-wasm.mjs dist",
 		"test": "vitest --run",

packages/coding-agent/src/core/child-session.ts

@@ -0,0 +1,30 @@
+/**
+ * Sessions written by child agents (subagent lanes and workflow agents) rather
+ * than by a user. A child runs in the parent's cwd, so its transcript lands in
+ * the same per-cwd session directory as the parent's; session lists and search
+ * use this to keep to sessions a user started.
+ *
+ * Two markers identify a child, and either is enough:
+ * - `origin: "subagent"` in the session header, written by the child process
+ *   when it creates the session (see `childSessionHeader`).
+ * - The session id prefix. Every child is spawned through
+ *   `runStepSubagentProcess` with an id the parent generates
+ *   (`subagent-<uuid>`, `workflow-<runId>-<agentId>`). Sessions written before
+ *   the header marker existed carry only this.
+ */
+
+export const SUBAGENT_SESSION_ID_PREFIX = "subagent-";
+export const WORKFLOW_SESSION_ID_PREFIX = "workflow-";
+
+/** Header `origin` of a session a child agent created. */
+export const CHILD_SESSION_ORIGIN = "subagent";
+
+/** True for an id the parent generated for a spawned child agent. */
+export function isChildAgentSessionId(sessionId: string): boolean {
+	return sessionId.startsWith(SUBAGENT_SESSION_ID_PREFIX) || sessionId.startsWith(WORKFLOW_SESSION_ID_PREFIX);
+}
+
+/** True for a session written by a spawned child agent, not by a user. */
+export function isChildAgentSession(session: { readonly id: string; readonly origin?: unknown }): boolean {
+	return session.origin === CHILD_SESSION_ORIGIN || isChildAgentSessionId(session.id);
+}

packages/coding-agent/src/core/session-index/attach.ts

@@ -0,0 +1,30 @@
+import type { SessionManager } from "../session-manager.ts";
+import type { SessionManagerFactory } from "../session-manager-factory.ts";
+import type { SessionIndex } from "./types.ts";
+
+type PersistedSink = Pick<SessionIndex, "notifyPersisted">;
+
+/**
+ * Forward a manager's file writes to the index. The listener only enqueues a
+ * debounced sync, so a slow or degraded index never delays persistence.
+ * Replaces any listener already set on the manager.
+ */
+export function attachSessionIndex(manager: SessionManager, index: PersistedSink): SessionManager {
+	manager.setFilePersistedListener((event) => index.notifyPersisted(event));
+	return manager;
+}
+
+/** Wrap a factory so every manager it creates, opens, or forks reports writes to the index. */
+export function withSessionIndex(factory: SessionManagerFactory, index: PersistedSink): SessionManagerFactory {
+	const attach = (manager: SessionManager): SessionManager => attachSessionIndex(manager, index);
+	return {
+		create: (cwd, sessionDir, options) => attach(factory.create(cwd, sessionDir, options)),
+		open: (path, sessionDir, cwdOverride) => attach(factory.open(path, sessionDir, cwdOverride)),
+		inMemory: (cwd, options) => factory.inMemory(cwd, options),
+		forkFrom: (sourcePath, targetCwd, sessionDir, options) =>
+			attach(factory.forkFrom(sourcePath, targetCwd, sessionDir, options)),
+		continueRecent: (cwd, sessionDir) => attach(factory.continueRecent(cwd, sessionDir)),
+		list: factory.list.bind(factory),
+		listAll: factory.listAll.bind(factory),
+	};
+}