From 15383206ca4eb4cc502c15f429dccdec76bf509a Mon Sep 17 00:00:00 2001 From: Jeremy lewi Date: Tue, 29 Sep 2026 16:38:16 -0700 Subject: [PATCH 01/11] Recover missing OPFS notebooks from Google Drive Signed-off-by: Jeremy lewi --- app/src/storage/filePayloads.ts | 10 +- app/src/storage/local.test.ts | 297 +++++++++++++++++++++++++++ app/src/storage/local.ts | 165 ++++++++++++++- app/src/storage/operationLogs.ts | 15 +- docs-dev/CUJs/drive-sync-recovery.md | 23 +++ 5 files changed, 499 insertions(+), 11 deletions(-) diff --git a/app/src/storage/filePayloads.ts b/app/src/storage/filePayloads.ts index d6d7e294..dbd4c99d 100644 --- a/app/src/storage/filePayloads.ts +++ b/app/src/storage/filePayloads.ts @@ -103,10 +103,12 @@ export class MemoryFilePayloadStorage implements FilePayloadStorage { } async read(ref: FilePayloadRef): Promise { const content = this.values.get(ref.path) - if ( - content === undefined || - (await filePayloadChecksum(content)) !== ref.checksum - ) + if (content === undefined) + throw new DOMException( + `Notebook payload missing or corrupt: ${ref.path}`, + 'NotFoundError' + ) + if ((await filePayloadChecksum(content)) !== ref.checksum) throw new Error(`Notebook payload missing or corrupt: ${ref.path}`) return content } diff --git a/app/src/storage/local.test.ts b/app/src/storage/local.test.ts index 1d72987c..f9015dc8 100644 --- a/app/src/storage/local.test.ts +++ b/app/src/storage/local.test.ts @@ -60,6 +60,7 @@ import LocalNotebooks, { } from './local' import { NotebookStoreItemType } from './notebook' import { MemoryOperationLogStorage } from './operationLogs' +import { OwnedOperationLogs } from './ownedOperationLogs' import { MemoryRevisionDocStorage } from './revisionDocs' /** Collect small fixture pages for assertions; production APIs always expose a cursor. */ @@ -9955,3 +9956,299 @@ describe('OPFS notebook payload migration', () => { expect((await store.files.get(uri))?.name).toBe('renamed.json') }) }) + +describe('LocalNotebooks missing OPFS recovery', () => { + const uri = 'local://file/lost-opfs' + const remoteId = 'https://drive.google.com/file/d/recoverable/view' + + /** Keep IndexedDB locators while removing the durable bytes, as after OPFS loss. */ + async function fixture(format = 'runme') { + const notebook = create(parser_pb.NotebookSchema, { + cells: [ + create(parser_pb.CellSchema, { + refId: 'recover-cell', + kind: parser_pb.CellKind.CODE, + languageId: 'python', + value: 'print("from Drive")', + }), + ], + }) + const content = + format === 'runme' + ? ( + await convertLegacyNotebookFileToRunme( + encodeRunmeNotebook(notebook), + 'source.json' + ) + ).content + : format === 'ipynb' + ? encodeIpynbNotebook(notebook).text + : encodeRunmeNotebook(notebook) + const drive = { + loadContent: vi.fn(async () => content), + getVersionMetadata: vi.fn(async () => ({ + md5Checksum: md5(content), + version: '1', + })), + saveContent: vi.fn(), + save: vi.fn(), + } + const logs = new MemoryOperationLogStorage() + const store = createTestStore(drive, { operationLogStorage: logs }) + const payloads = (store as any).payloadStorage as MemoryFilePayloadStorage + const operationLogRef = + format === 'runme' ? (await logs.initialize(uri, content)).ref : undefined + const contentRef = + format !== 'runme' + ? await payloads.write(encodeRunmeNotebook(notebook)) + : undefined + await store.files.put({ + id: uri, + name: `lost.${format}`, + remoteId, + doc: '', + operationLogRef, + contentRef, + md5Checksum: 'lost-local-version', + lastRemoteChecksum: 'old-upstream', + lastSynced: new Date().toISOString(), + }) + if (operationLogRef) await logs.delete(operationLogRef) + payloads.values.clear() + return { store, drive, logs, payloads, content, operationLogRef } + } + + it.each([ + 'load', + 'loadContent', + 'loadOperationLogSnapshot', + 'createOperationLogSaveStore', + ] as const)( + 'restores a missing journal through %s, preserving history and subsequent edits', + async (method) => { + const f = await fixture() + try { + if (method === 'createOperationLogSaveStore') + await f.store.createOperationLogSaveStore(uri, { + actorId: 'open-test', + }) + else await f.store[method](uri) + expect(await f.store.loadContent(uri)).toBe(f.content) + expect(f.drive.loadContent).toHaveBeenCalledTimes(1) + expect(f.drive.saveContent).not.toHaveBeenCalled() + expect((await f.store.files.get(uri))?.id).toBe(uri) + const editor = await f.store.createOperationLogSaveStore(uri, { + actorId: 'recovered-editor', + }) + editor.initialNotebook.cells[0].value = 'print("edited")' + await editor.save(uri, editor.initialNotebook) + expect((await f.store.load(uri)).cells[0].value).toBe('print("edited")') + // A new execution must persist fresh output in the recovered journal. + editor.initialNotebook.cells[0].metadata = { + ...editor.initialNotebook.cells[0].metadata, + [RunmeMetadataKey.LastRunID]: 'recovered-run', + [RunmeMetadataKey.ExecutionState]: 'completed', + } + editor.initialNotebook.cells[0].outputs = [ + create(parser_pb.CellOutputSchema, { + items: [ + create(parser_pb.CellOutputItemSchema, { + mime: 'text/plain', + data: new TextEncoder().encode('edited\n'), + }), + ], + }), + ] + await editor.save(uri, editor.initialNotebook) + const reopened = await f.store.load(uri) + expect( + new TextDecoder().decode(reopened.cells[0].outputs[0].items[0].data) + ).toBe('edited\n') + const before = parseOperationLog(f.content) + const after = parseOperationLog(await f.store.loadContent(uri)) + expect(after.header.notebook_id).toBe(before.header.notebook_id) + for (const op of before.operations) + expect(after.operations).toContainEqual(op) + expect(f.drive.loadContent).toHaveBeenCalledTimes(1) + } finally { + f.store.stopSyncQueue() + } + } + ) + + it.each(['json', 'ipynb'])( + 'restores missing %s payloads and persists edits', + async (format) => { + const f = await fixture(format) + try { + const loaded = await f.store.load(uri) + expect(loaded.cells[0].value).toBe('print("from Drive")') + loaded.cells[0].value = 'edited' + await f.store.save(uri, loaded) + expect((await f.store.load(uri)).cells[0].value).toBe('edited') + expect(f.drive.loadContent).toHaveBeenCalledTimes(1) + } finally { + f.store.stopSyncQueue() + } + } + ) + + it('restores through the storage owner without acknowledging an intervening edit', async () => { + const f = await fixture() + const generations = new Map() + Object.assign(f.store.files, { + db: { + transaction: async (...args: unknown[]) => + (args.at(-1) as () => Promise)(), + }, + }) + const owner = new OwnedOperationLogs( + f.logs, + f.store.files, + { + get: async (path: string) => generations.get(path), + put: async (value: { path: string; generation: number }) => + generations.set(value.path, value), + } as any, + (id, changes) => (f.store as any).updateFile(id, changes) + ) + Object.assign(f.store, { + ownedLogs: owner, + operationLogStorage: owner, + runtime: { owner: true }, + }) + const initialize = owner.initialize.bind(owner) + vi.spyOn(owner, 'initialize').mockImplementationOnce( + async (id, content) => { + const restored = await initialize(id, content) + const editor = await f.store.createOperationLogSaveStore(uri, { + actorId: 'concurrent-editor', + initialDocument: content, + }) + editor.initialNotebook.cells[0].value = 'new edit during recovery' + await editor.save(uri, editor.initialNotebook) + return restored + } + ) + try { + expect((await f.store.load(uri)).cells[0].value).toBe( + 'new edit during recovery' + ) + const record = await f.store.files.get(uri) + expect(record?.md5Checksum).toBe('') + expect(record?.lastRemoteChecksum).toBe(md5(f.content)) + expect(record?.pendingInitializationRef).toBeUndefined() + } finally { + f.store.stopSyncQueue() + } + }) + + it('recovers when the journal and pending initialization payload are both missing', async () => { + const f = await fixture() + const ref = await f.payloads.write(f.content) + await f.store.files.update(uri, { pendingInitializationRef: ref }) + f.payloads.values.clear() + try { + expect((await f.store.load(uri)).cells[0].value).toBe( + 'print("from Drive")' + ) + expect( + (await f.store.files.get(uri))?.pendingInitializationRef + ).toBeUndefined() + expect(await f.store.loadContent(uri)).toBe(f.content) + } finally { + f.store.stopSyncQueue() + } + }) + + it('downloads once when two opens encounter the same missing file', async () => { + const f = await fixture() + try { + const results = await Promise.all([f.store.load(uri), f.store.load(uri)]) + expect(results.map((n) => n.cells[0].value)).toEqual([ + 'print("from Drive")', + 'print("from Drive")', + ]) + expect(f.drive.loadContent).toHaveBeenCalledTimes(1) + } finally { + f.store.stopSyncQueue() + } + }) + + it.each(['offline', '404', 'malformed', 'empty', 'changed'])( + 'preserves the reference if Drive is %s', + async (failure) => { + const f = await fixture() + const original = await f.store.files.get(uri) + if (failure === 'offline' || failure === '404') + f.drive.loadContent.mockRejectedValue(new Error(failure)) + else { + const content = failure === 'empty' ? '' : 'not a notebook' + f.drive.loadContent.mockResolvedValue(content) + if (failure !== 'changed') + f.drive.getVersionMetadata.mockResolvedValue({ + md5Checksum: md5(content), + version: '1', + }) + } + try { + await expect(f.store.load(uri)).rejects.toThrow() + expect(await f.store.files.get(uri)).toEqual(original) + await expect(f.logs.read(f.operationLogRef!)).rejects.toMatchObject({ + name: 'NotFoundError', + }) + expect(f.drive.saveContent).not.toHaveBeenCalled() + } finally { + f.store.stopSyncQueue() + } + } + ) + + it.each(['local-only', 'healthy', 'corrupt', 'permission'])( + 'does not download for %s storage', + async (state) => { + const f = await fixture() + try { + if (state === 'local-only') + await f.store.files.update(uri, { remoteId: uri }) + if (state === 'healthy') await f.logs.initialize(uri, f.content) + if (state === 'corrupt') + await f.logs.initialize(uri, 'invalid history\n') + if (state === 'permission') + vi.spyOn(f.logs, 'read').mockRejectedValue( + new DOMException('permission denied', 'NotAllowedError') + ) + if (state === 'healthy') + expect((await f.store.load(uri)).cells).toHaveLength(1) + else await expect(f.store.load(uri)).rejects.toThrow() + expect(f.drive.loadContent).not.toHaveBeenCalled() + if (state === 'corrupt') + expect((await f.logs.read(f.operationLogRef!)).document).toBe( + 'invalid history\n' + ) + } finally { + f.store.stopSyncQueue() + } + } + ) + + it('keeps a journal recreated while Drive downloads', async () => { + const f = await fixture() + const localContent = ( + await convertLegacyNotebookFileToRunme( + notebookJson('new local content'), + 'local.json' + ) + ).content + f.drive.loadContent.mockImplementationOnce(async () => { + await f.logs.initialize(uri, localContent) + return f.content + }) + try { + expect((await f.store.load(uri)).cells[0].value).toBe('new local content') + expect(await f.store.loadContent(uri)).toBe(localContent) + } finally { + f.store.stopSyncQueue() + } + }) +}) diff --git a/app/src/storage/local.ts b/app/src/storage/local.ts index 14ead702..ddbdc22a 100644 --- a/app/src/storage/local.ts +++ b/app/src/storage/local.ts @@ -153,6 +153,16 @@ import { import { SyncDeferred, SyncWorkQueue } from './syncWorkQueue' import { readTablePage, scanTable, tableHasRecords } from './tableScan' +/** DOMException identity does not survive every worker boundary; use its name. */ +function isMissingOpfsFile(error: unknown): boolean { + return Boolean( + error && + typeof error === 'object' && + 'name' in error && + error.name === 'NotFoundError' + ) +} + // Local folder URI is a special folder that contains all notebooks which are local (i.e. not synced to Drive) export const LOCAL_FOLDER_URI = 'local://folder/local' @@ -929,6 +939,136 @@ export class LocalNotebooks extends Dexie { } } + /** Retry an explicit open once, and only for a filesystem missing-file error. */ + private async withMissingOpfsRecovery( + uri: string, + read: () => Promise + ): Promise { + try { + return await read() + } catch (error) { + if (!isMissingOpfsFile(error)) throw error + const recovered = await this.serializeLocalSave(uri, () => + this.driveSyncCoordinator.runExclusive(uri, () => + this.restoreMissingOpfsFile(uri) + ) + ) + if (!recovered) throw error + return read() + } + } + + /** + * Restore the existing mirror identity after OPFS loss. The caller holds the + * local-save queue and Drive lock, so recovery cannot race a save or sync. + * Present but invalid files are never replaced by this missing-file path. + */ + private async restoreMissingOpfsFile(uri: string): Promise { + const record = await this.files.get(uri) + if ( + !record || + !isDriveUri(record.remoteId) || + !isNotebookFileName(record.name) + ) + return false + const operationLog = + detectNotebookFileFormat(record.name) === 'runme-operation-log' + const readLocal = async () => { + if (operationLog && record.operationLogRef) + await this.operationLogStorage.read(record.operationLogRef) + else if (record.contentRef) await this.hydrateFile(record) + else return false + return true + } + try { + // A concurrent open may already have recovered the same file. + if (await readLocal()) return true + return false + } catch (error) { + if (!isMissingOpfsFile(error)) throw error + } + + const snapshot = await this.loadConsistentDriveOperationLogSnapshot( + record.remoteId + ) + if (!snapshot) throw new DriveSnapshotChangedError(record.remoteId) + // Decode before writing, retaining the missing reference if Drive is empty, + // malformed, unavailable, or changes during the download. + const decoded = decodeNotebookFile(snapshot.content, record.name) + const latest = await this.files.get(uri) + if ( + !latest || + latest.remoteId !== record.remoteId || + latest.name !== record.name || + latest.contentRef?.path !== record.contentRef?.path || + latest.operationLogRef?.path !== record.operationLogRef?.path || + latest.md5Checksum !== record.md5Checksum + ) + throw new Error(`Notebook changed during OPFS recovery: ${uri}`) + try { + if (await readLocal()) return true + } catch (error) { + if (!isMissingOpfsFile(error)) throw error + } + + const upstreamChecksum = md5(snapshot.content) + const baseline = { + lastRemoteChecksum: upstreamChecksum, + lastUpstreamVersion: driveMetadataToUpstreamVersion(snapshot.version), + lastSynced: nowIsoString(), + lastSyncError: undefined, + } + if (operationLog) { + // initialize refuses to overwrite a journal that appeared meanwhile. + // Keep the exact Drive history, including its notebook and operation IDs. + const stored = await this.operationLogStorage.initialize( + uri, + snapshot.content + ) + if (this.ownedLogs) { + await this.ownedLogs.acknowledge(uri, stored, record.remoteId, baseline) + } else { + await this.updateFile(uri, { + ...baseline, + operationLogRef: stored.ref, + md5Checksum: stored.checksum, + }) + } + await this.updateFile(uri, { + doc: '', + pendingOperationLogInitialization: undefined, + }) + } else { + const restored = decoded.ipynb + ? await this.decodeUpstreamNotebook({ + localUri: uri, + record, + content: snapshot.content, + upstreamFingerprint: upstreamChecksum, + }) + : { + serialized: serializeNotebook(decoded.notebook), + ipynbPreservation: undefined, + } + await this.updateFile(uri, { + ...baseline, + doc: restored.serialized, + md5Checksum: checksumForSerializedNotebook(restored.serialized), + ipynbPreservation: restored.ipynbPreservation, + }) + } + appLogger.warn('Restored missing OPFS notebook from Google Drive', { + attrs: { + scope: 'storage.drive.recovery', + code: 'NOTEBOOK_OPFS_RESTORED', + localUri: uri, + remoteUri: record.remoteId, + }, + }) + this.notifySync(uri) + return true + } + /** Initialization bytes are recovery input, not part of ordinary content reads. */ private async readPendingInitialization( record: LocalFileRecord | undefined @@ -2132,8 +2272,7 @@ export class LocalNotebooks extends Dexie { } const initialDocument = - options.initialDocument ?? - (await this.operationLogStorage.read(record.operationLogRef)).document + options.initialDocument ?? (await this.loadContent(uri)) let view: ParsedOperationLog = parseOperationLog(initialDocument) let previous = materializedLogToNotebook( materializeOperationLog(view.operations) @@ -3181,7 +3320,12 @@ export class LocalNotebooks extends Dexie { } } + /** Read local content, restoring missing OPFS bytes from Drive when necessary. */ async loadContent(uri: string): Promise { + return this.withMissingOpfsRecovery(uri, () => this.loadContentInner(uri)) + } + + private async loadContentInner(uri: string): Promise { await this.recoverLocalInitialization(uri) if (!uri.startsWith('local://file/')) { throw new Error( @@ -3283,8 +3427,16 @@ export class LocalNotebooks extends Dexie { return content } - /** Materialize the current local .runme OPFS log without upstream I/O. */ + /** Materialize the local journal, downloading it only when OPFS lost the file. */ async loadOperationLogSnapshot(uri: string): Promise { + return this.withMissingOpfsRecovery(uri, () => + this.loadOperationLogSnapshotInner(uri) + ) + } + + private async loadOperationLogSnapshotInner( + uri: string + ): Promise { if (!uri.startsWith('local://file/')) { throw new Error( 'LocalNotebooks.loadOperationLogSnapshot expects a local://file/ URI; got ' + @@ -3690,7 +3842,12 @@ export class LocalNotebooks extends Dexie { this.notifySync(uri) } + /** Keep healthy opens local; a missing OPFS file can be restored from Drive. */ async load(uri: string): Promise { + return this.withMissingOpfsRecovery(uri, () => this.loadInner(uri)) + } + + private async loadInner(uri: string): Promise { await this.recoverLocalInitialization(uri) if (!uri.startsWith('local://file/')) { throw new Error( @@ -3739,7 +3896,7 @@ export class LocalNotebooks extends Dexie { } if (detectNotebookFileFormat(record.name) === 'runme-operation-log') { - return this.loadOperationLogSnapshot(uri) + return this.loadOperationLogSnapshotInner(uri) } if (!record.doc) { diff --git a/app/src/storage/operationLogs.ts b/app/src/storage/operationLogs.ts index d18fb01e..d520b956 100644 --- a/app/src/storage/operationLogs.ts +++ b/app/src/storage/operationLogs.ts @@ -331,7 +331,10 @@ export class MemoryOperationLogStorage implements OperationLogStorage { async read(ref: OperationLogRef): Promise { const document = this.documents.get(ref.path) if (document === undefined) { - throw new Error(`Operation log not found: ${ref.path}`) + throw new DOMException( + `Operation log not found: ${ref.path}`, + 'NotFoundError' + ) } validateFraming(document, 'Stored operation log') return snapshot(ref, document) @@ -354,7 +357,10 @@ export class MemoryOperationLogStorage implements OperationLogStorage { return this.coordinator.runExclusive(ref.path, async () => { const original = this.documents.get(ref.path) if (original === undefined) { - throw new Error(`Operation log not found: ${ref.path}`) + throw new DOMException( + `Operation log not found: ${ref.path}`, + 'NotFoundError' + ) } const current = options.prepareDocument?.(original) ?? original const records = await createRecords(current) @@ -376,7 +382,10 @@ export class MemoryOperationLogStorage implements OperationLogStorage { validateFraming(document, 'Replacement operation log') return this.coordinator.runExclusive(ref.path, async () => { if (!this.documents.has(ref.path)) { - throw new Error(`Operation log not found: ${ref.path}`) + throw new DOMException( + `Operation log not found: ${ref.path}`, + 'NotFoundError' + ) } if ( options.expectedChecksum && diff --git a/docs-dev/CUJs/drive-sync-recovery.md b/docs-dev/CUJs/drive-sync-recovery.md index 302c7833..4cef120d 100644 --- a/docs-dev/CUJs/drive-sync-recovery.md +++ b/docs-dev/CUJs/drive-sync-recovery.md @@ -314,3 +314,26 @@ ready items blocked by a file claim and the oldest active attempt. through the other real Chromium tab while the first is stalled on credentials. Its output includes active count and configured capacity. It then disables both tabs' credential availability and verifies offline editing and restart persistence. + +## Missing OPFS files with a Drive copy + +If IndexedDB still identifies a Drive-backed notebook but its OPFS journal or +cached JSON/IPYNB payload is missing, opening it downloads a consistent Drive +snapshot into the same local notebook identity. Healthy cached notebooks still +open offline. Recovery never uploads an empty replacement or overwrites a +present but corrupt journal. Drive/auth failures leave the missing-file +reference intact so opening can retry after connectivity is restored. + +Automated regression coverage: the `LocalNotebooks missing OPFS recovery` suite +in `local.test.ts` removes only fixture OPFS bytes, retains metadata, and verifies +open, edit, save, reopen, and fresh execution output. It also checks concurrent +opens, a journal recreated during download, JSON/IPYNB payloads, and failed, +empty, malformed, or changing Drive downloads. These are storage-level tests; +the existing browser CUJ does not yet simulate origin storage loss. + +Manual acceptance in a disposable browser profile: mirror a Drive notebook, +close its editor, and remove only its OPFS document while keeping IndexedDB. +Reopen it and verify its cells are editable, edits and a rerun survive reopening, +and the Drive file/local URI are unchanged. Repeat while offline, then restore +connectivity and reopen. Never perform this destructive setup on a user's +unsynced notebook. From e3a73c0e30d79e888dc864effb7d16def37ce9b9 Mon Sep 17 00:00:00 2001 From: Jeremy lewi Date: Tue, 29 Sep 2026 19:12:35 -0700 Subject: [PATCH 02/11] Reject invalid JSON before restoring an OPFS notebook Signed-off-by: Jeremy lewi --- app/src/storage/local.test.ts | 25 +++++++++++++++++++++++++ app/src/storage/local.ts | 10 ++++++++++ 2 files changed, 35 insertions(+) diff --git a/app/src/storage/local.test.ts b/app/src/storage/local.test.ts index f9015dc8..06414b99 100644 --- a/app/src/storage/local.test.ts +++ b/app/src/storage/local.test.ts @@ -10093,6 +10093,31 @@ describe('LocalNotebooks missing OPFS recovery', () => { } ) + it.each([ + '', + '{"error":"upstream failure"}', + '{"cells":[],"unrecognized":"data"}', + ])( + 'does not turn invalid Drive JSON %j into an empty recovered notebook', + async (content) => { + const f = await fixture('json') + const original = await f.store.files.get(uri) + f.drive.loadContent.mockResolvedValue(content) + f.drive.getVersionMetadata.mockResolvedValue({ + md5Checksum: md5(content), + version: '1', + }) + try { + await expect(f.store.load(uri)).rejects.toThrow() + expect(await f.store.files.get(uri)).toEqual(original) + expect(f.payloads.values.size).toBe(0) + expect(f.drive.saveContent).not.toHaveBeenCalled() + } finally { + f.store.stopSyncQueue() + } + } + ) + it('restores through the storage owner without acknowledging an intervening edit', async () => { const f = await fixture() const generations = new Map() diff --git a/app/src/storage/local.ts b/app/src/storage/local.ts index ddbdc22a..5548f459 100644 --- a/app/src/storage/local.ts +++ b/app/src/storage/local.ts @@ -16,6 +16,7 @@ import { decodeNotebookFile, detectNotebookFileFormat, encodeIpynbNotebook, + inspectRunmeNotebookJsonShape, isNotebookFileName, notebookFileExtension, validateNotebookRenameFormat, @@ -994,6 +995,15 @@ export class LocalNotebooks extends Dexie { if (!snapshot) throw new DriveSnapshotChangedError(record.remoteId) // Decode before writing, retaining the missing reference if Drive is empty, // malformed, unavailable, or changes during the download. + if ( + !snapshot.content.trim() || + (detectNotebookFileFormat(record.name) === 'runme-json' && + !inspectRunmeNotebookJsonShape(snapshot.content)) + ) { + throw new Error( + `Drive recovery source is not a notebook: ${record.remoteId}` + ) + } const decoded = decodeNotebookFile(snapshot.content, record.name) const latest = await this.files.get(uri) if ( From eaf8ec112fe383d1e6ddda1e4bf73e2e5396ca46 Mon Sep 17 00:00:00 2001 From: Jeremy lewi Date: Sat, 3 Oct 2026 13:19:02 -0700 Subject: [PATCH 03/11] Add Agents API monitor widgets to JavaScript notebook cells --- .../components/Actions/ActionOutputItems.tsx | 6 +- app/src/components/Actions/Actions.test.tsx | 25 ++ app/src/components/Actions/Actions.tsx | 45 +- .../components/Agents/AgentMonitor.test.tsx | 90 ++++ app/src/components/Agents/AgentMonitor.tsx | 240 +++++++++++ app/src/generated/documentationManifest.ts | 8 + app/src/lib/agents/bridge.ts | 30 ++ app/src/lib/agents/monitor.test.ts | 261 ++++++++++++ app/src/lib/agents/monitor.ts | 391 ++++++++++++++++++ app/src/lib/agents/runtime.test.ts | 65 +++ app/src/lib/agents/runtime.ts | 125 ++++++ app/src/lib/agents/transport.test.ts | 51 +++ app/src/lib/agents/transport.ts | 142 +++++++ app/src/lib/agents/types.ts | 67 +++ app/src/lib/notebookData.test.ts | 41 ++ app/src/lib/notebookData.ts | 22 +- app/src/lib/runtime/appJsGlobals.ts | 5 + app/src/lib/runtime/codeModeExecutor.ts | 3 + app/src/lib/runtime/sandboxJsKernel.test.ts | 6 +- app/src/lib/runtime/sandboxJsKernel.ts | 22 +- docs-dev/CUJs/agent-monitor.md | 16 + docs-dev/cujs/README.md | 2 + docs-dev/design/20261003_agents_api_ui.md | 110 +++++ docs/20-agent-monitors.md | 36 ++ docs/agent-monitors.json | 28 ++ testing/fake-agents-server.go | 89 ++++ 26 files changed, 1919 insertions(+), 7 deletions(-) create mode 100644 app/src/components/Agents/AgentMonitor.test.tsx create mode 100644 app/src/components/Agents/AgentMonitor.tsx create mode 100644 app/src/lib/agents/bridge.ts create mode 100644 app/src/lib/agents/monitor.test.ts create mode 100644 app/src/lib/agents/monitor.ts create mode 100644 app/src/lib/agents/runtime.test.ts create mode 100644 app/src/lib/agents/runtime.ts create mode 100644 app/src/lib/agents/transport.test.ts create mode 100644 app/src/lib/agents/transport.ts create mode 100644 app/src/lib/agents/types.ts create mode 100644 docs-dev/CUJs/agent-monitor.md create mode 100644 docs-dev/design/20261003_agents_api_ui.md create mode 100644 docs/20-agent-monitors.md create mode 100644 docs/agent-monitors.json create mode 100644 testing/fake-agents-server.go diff --git a/app/src/components/Actions/ActionOutputItems.tsx b/app/src/components/Actions/ActionOutputItems.tsx index ea8637a9..a365f4e1 100644 --- a/app/src/components/Actions/ActionOutputItems.tsx +++ b/app/src/components/Actions/ActionOutputItems.tsx @@ -1,3 +1,5 @@ +import { AgentMonitorOutput } from '../Agents/AgentMonitor' +import { AGENT_MONITOR_MIME } from '../../lib/agents/types' import { HtmlOutput } from './HtmlOutput' import React from 'react' import { LinkIcon } from '@heroicons/react/20/solid' @@ -86,7 +88,9 @@ export function ActionOutputItemView({ let content: React.ReactNode = null - if (mime === 'text/html') { + if (mime === AGENT_MONITOR_MIME) { + content = + } else if (mime === 'text/html') { content = ( { }) describe('Actions tabs', () => { + it('toggles agent widget source and render modes without executing or unmounting output', () => { + const cell = create(parser_pb.CellSchema, { + refId: 'widget-cell', kind: parser_pb.CellKind.CODE, languageId: 'javascript', value: 'agents.monitor("sess_test")', + outputs: [create(parser_pb.CellOutputSchema, { items: [create(parser_pb.CellOutputItemSchema, { + mime: 'application/vnd.runme.agent-monitor+json', + data: new TextEncoder().encode(JSON.stringify({version: 1, id: 'saved-widget', sessionId: 'sess_test', pageSize: 50})), + })] })], + }) + const stub = new StubCellData(cell) + stub.getRunnerName = () => APPKERNEL_RUNNER_NAME + const view = render() + const editor = view.container.querySelector('[data-cell-focus-role="editor"]') as HTMLElement + const output = view.container.querySelector('#widget-output-widget-cell') as HTMLElement + expect(editor.hidden).toBe(true) + const mounted = screen.getByTestId('agent-monitor') + fireEvent.click(screen.getByRole('button', {name: /^Edit$/})) + expect(editor.hidden).toBe(false) + expect(output.hidden).toBe(true) + expect(screen.getByTestId('agent-monitor')).toBe(mounted) + fireEvent.click(screen.getByRole('button', {name: /^Render$/})) + expect(output.hidden).toBe(false) + expect(stub.run).not.toHaveBeenCalled() + view.unmount() + }) + it.each([ ['url', true, true, true], diff --git a/app/src/components/Actions/Actions.tsx b/app/src/components/Actions/Actions.tsx index d01f4981..dbcd85bc 100644 --- a/app/src/components/Actions/Actions.tsx +++ b/app/src/components/Actions/Actions.tsx @@ -1,3 +1,4 @@ +import { AGENT_MONITOR_MIME } from '../../lib/agents/types' import { outputReferenceSource } from '../../lib/outputReferenceRuntime' import { OutputReferenceCell } from './OutputReferenceCell' import { @@ -264,7 +265,8 @@ function syncIndicatorPresentation(state: NotebookSyncState | null): { switch (state?.status) { case 'not-downloaded': return { - label: 'Notebook has not been downloaded to this browser. Click to download now.', + label: + 'Notebook has not been downloaded to this browser. Click to download now.', className: 'border border-nb-text-faint bg-transparent', clickable: true, } @@ -805,6 +807,12 @@ export function Action({ }) const shareTargetUri = shareTarget.docUri === docUri ? shareTarget.targetUri : null + const [widgetEditing, setWidgetEditing] = useState(false) + const hasWidgetOutput = + cell?.outputs?.some((output) => + output.items.some((item) => item.mime === AGENT_MONITOR_MIME) + ) ?? false + const [widgetSourceChanged, setWidgetSourceChanged] = useState(false) const [htmlEditRequest, setHtmlEditRequest] = useState(0) const [markdownEditRequest, setMarkdownEditRequest] = useState(0) const [pid, setPid] = useState(null) @@ -914,6 +922,8 @@ export function Action({ if (readOnly) { return } + setWidgetEditing(false) + setWidgetSourceChanged(false) cellData.run() }, [cellData, readOnly]) @@ -2133,6 +2143,7 @@ export function Action({ {/* Minimal toolbar: language + runner selectors + run/trash buttons */} + {hasWidgetOutput && ( +
+ + + {widgetSourceChanged && ( + Source changed — Run to update the monitor. + )} +
+ )}
setPassphrase(e.target.value)} + required + disabled={busy} + minLength={snapshot.status === 'new' ? 12 : undefined} + /> + + {snapshot.status === 'new' && ( + <> + +

+ Use at least 12 characters. Keep the passphrase safe: Runme + cannot recover it. Clearing this site's browser data removes + the vault. +

+ + )} + + + ) : ( + <> + +

+ Unlocked for this tab. Run only trusted browser JavaScript while + your keys are available. +

+
    + {snapshot.names.map((key) => ( +
  • +

    {key}

    +
    + + +
    + {deleting === key && ( +
    +

    + Remove this saved key? Keep a copy if you will need it + again. +

    + + +
    + )} +
  • + ))} +
+ {!snapshot.names.length && ( +

+ No keys yet. Add your first named key below. +

+ )} +
{ + event.preventDefault() + void act(async () => { + await vault.saveKey(name, value, editing) + setName('') + setEditing(undefined) + setNotice('Key saved.') + }) + }} + > +

+ {editing ? 'Edit key' : 'Add a key'} +

+ + + {editing && ( +

+ Leave the value empty to keep the existing key. +

+ )} + + {editing && ( + + )} +
+

+ Use in a browser JS cell: +

+
+              {'agents.setKey(keyvault.getKey("openai-api"))'}
+            
+ + )} + {error && ( +

+ {error} +

+ )} + {notice && ( +

+ {notice} +

+ )} +
+
+ ) +} diff --git a/app/src/components/SidePanel/SidePanel.test.tsx b/app/src/components/SidePanel/SidePanel.test.tsx index 3c6855d2..bfe417b9 100644 --- a/app/src/components/SidePanel/SidePanel.test.tsx +++ b/app/src/components/SidePanel/SidePanel.test.tsx @@ -412,6 +412,12 @@ describe('SidePanelToolbar drive status button', () => { expect(togglePanelMock).toHaveBeenCalledWith('authentication') }) + it('opens Key Vault from its dedicated navigation button', () => { + render() + fireEvent.click(screen.getByRole('button', { name: 'Toggle Key Vault panel' })) + expect(togglePanelMock).toHaveBeenCalledWith('keyvault') + }) + it('opens AI grader settings from the dedicated navigation button', () => { render() fireEvent.click(screen.getByRole('button', { name: 'Toggle AI grader settings panel' })) diff --git a/app/src/components/SidePanel/SidePanel.tsx b/app/src/components/SidePanel/SidePanel.tsx index c62ca5dd..343fd145 100644 --- a/app/src/components/SidePanel/SidePanel.tsx +++ b/app/src/components/SidePanel/SidePanel.tsx @@ -10,6 +10,7 @@ import { ServerStackIcon, UserCircleIcon, KeyIcon, + LockClosedIcon, SparklesIcon, } from '@heroicons/react/24/outline' import { XMarkIcon } from '@heroicons/react/20/solid' @@ -59,6 +60,7 @@ import { resolveDriveLoginConfiguration, } from '../../auth/appLoginConfiguration' import { getServiceAccountCredentialStatusError } from '../../auth/googleServiceAccountImpersonation' +import KeyVaultPanel from '../KeyVault/KeyVaultPanel' import AuthenticationSettingsPanel from '../AuthenticationSettings/AuthenticationSettingsPanel' import { SuggestionGraderSettings } from '../SuggestionGraderSettings/SuggestionGraderSettings' import { showToast } from '../../lib/toast' @@ -652,6 +654,13 @@ export function SidePanelToolbar() { Authentication Settings +