From 8255c69d3e09742b61df8876032568a39d898d7c Mon Sep 17 00:00:00 2001 From: Elioooon Date: Mon, 31 Aug 2026 17:54:20 +0800 Subject: [PATCH 1/3] fix: persist Daytona authentication failures --- .changeset/persist-daytona-auth-errors.md | 6 ++ .../core/sandbox/provider/DaytonaProvider.ts | 27 ++++++-- .../daytonaSnapshotRegistration.test.ts | 53 ++++++++++++---- .../trueforge/src/runtime/sessionResources.ts | 10 ++- .../trueforge/src/sandbox/providerUtils.ts | 62 ++++++++++++++++--- .../tests/unit/sandbox/providerUtils.test.ts | 60 ++++++++++++++++++ 6 files changed, 192 insertions(+), 26 deletions(-) create mode 100644 .changeset/persist-daytona-auth-errors.md create mode 100644 packages/trueforge/tests/unit/sandbox/providerUtils.test.ts diff --git a/.changeset/persist-daytona-auth-errors.md b/.changeset/persist-daytona-auth-errors.md new file mode 100644 index 000000000..d03207257 --- /dev/null +++ b/.changeset/persist-daytona-auth-errors.md @@ -0,0 +1,6 @@ +--- +'@truefoundry/trueforge': patch +'@truefoundry/trueforge-core': patch +--- + +Persist failed Daytona credentials when sandbox operations return authorization errors. diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts index 1fba52e16..176b92580 100644 --- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts +++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts @@ -99,6 +99,7 @@ export interface DaytonaSandboxProviderOptions { /** Defaults to 1 hour (same as the gateway's max agent execution time). */ previewUrlExpirySeconds?: number; logger: Logger; + onError?: ((error: unknown) => Promise) | undefined; } export class DaytonaSandboxProvider implements SandboxProvider { @@ -118,6 +119,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { private readonly apiKey: string; private readonly apiUrl: string; private readonly logger: Logger; + private readonly onError: ((error: unknown) => Promise) | undefined; private readonly daytona: Daytona; private static readonly cachedSandboxes = new Map(); // De-dupes concurrent recovery attempts on the same sandbox to a single refreshData+start round-trip. @@ -138,6 +140,15 @@ export class DaytonaSandboxProvider implements SandboxProvider { this.natsBridgePort = options.natsBridgePort ?? DEFAULT_SANDBOX_NATS_WS_PORT; this.previewUrlExpirySeconds = options.previewUrlExpirySeconds ?? DEFAULT_PREVIEW_URL_EXPIRY_SECONDS; this.logger = options.logger.child({ module: 'DaytonaProvider' }); + this.onError = options.onError; + } + + private async reportError(error: unknown): Promise { + try { + await this.onError?.(error); + } catch (reportError) { + this.logger.error('Failed to report Daytona error', extractErrorLogFields(reportError)); + } } private async getOrCreateSandbox(sandboxId?: string): Promise<{ sandbox: Sandbox; defaultTimeoutMs: number }> { @@ -246,11 +257,16 @@ export class DaytonaSandboxProvider implements SandboxProvider { } async createSandbox(): Promise<{ sandboxId: string }> { - return context.with(suppressTracing(context.active()), async () => { - const { sandbox } = await this.getOrCreateSandbox(); - this.logger.debug(`Sandbox created: name=${sandbox.name}`); - return { sandboxId: sandbox.name }; - }); + try { + return await context.with(suppressTracing(context.active()), async () => { + const { sandbox } = await this.getOrCreateSandbox(); + this.logger.debug(`Sandbox created: name=${sandbox.name}`); + return { sandboxId: sandbox.name }; + }); + } catch (error) { + await this.reportError(error); + throw error; + } } /** Resolves undefined when no snapshot carries that name; auth/other failures throw. */ @@ -403,6 +419,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { if (e instanceof SandboxNotAvailableError) { throw e; } + await this.reportError(e); this.logger.error('Sandbox execution error', extractErrorLogFields(e)); const message = e instanceof Error ? e.message : 'Unknown error'; return { success: false, error: message }; diff --git a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts index 2c94b868d..bc135a8db 100644 --- a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts +++ b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts @@ -32,6 +32,23 @@ function makeProvider(): DaytonaSandboxProvider { }); } +function makeRuntimeProvider(client: Daytona, onError?: (error: unknown) => Promise): DaytonaSandboxProvider { + return new DaytonaSandboxProvider({ + client, + apiKey: 'dtn-test', + apiUrl: API_URL, + tenantName: 'test-tenant', + sandboxImage: 'registry.example.com/sandbox:029ea5ff', + timeoutMs: 1000, + autoStopIntervalInMinutes: 5, + autoArchiveIntervalInMinutes: 60, + autoDeleteIntervalInMinutes: 7200, + fileMaxBytesForDownload: 1024, + logger: makeSilentLogger(), + onError, + }); +} + function mockFetch({ status, body }: { status: number; body: unknown }): jest.SpiedFunction { return jest .spyOn(globalThis, 'fetch') @@ -81,22 +98,32 @@ describe('DaytonaSandboxProvider register-only snapshot create', () => { }); describe('DaytonaSandboxProvider exec', () => { + it('reports sandbox creation errors before rethrowing them', async () => { + const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); + jest.spyOn(client, 'create').mockRejectedValue(new DaytonaError('unauthorized', 401)); + const onError = jest.fn().mockResolvedValue(undefined); + const provider = makeRuntimeProvider(client, onError); + + await expect(provider.createSandbox()).rejects.toMatchObject({ statusCode: 401 }); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 })); + }); + + it('reports provider errors before converting them to failed exec results', async () => { + const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); + jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('unauthorized', 401)); + const onError = jest.fn().mockResolvedValue(undefined); + const provider = makeRuntimeProvider(client, onError); + + await expect(provider.exec({ sandboxId: 'test-tenant.expired', command: 'true' })).resolves.toMatchObject({ + success: false, + }); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 })); + }); + it('rethrows SandboxNotAvailableError when the sandbox is gone', async () => { const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('not found', NOT_FOUND_STATUS)); - const provider = new DaytonaSandboxProvider({ - client, - apiKey: 'dtn-test', - apiUrl: API_URL, - tenantName: 'test-tenant', - sandboxImage: 'registry.example.com/sandbox:029ea5ff', - timeoutMs: 1000, - autoStopIntervalInMinutes: 5, - autoArchiveIntervalInMinutes: 60, - autoDeleteIntervalInMinutes: 7200, - fileMaxBytesForDownload: 1024, - logger: makeSilentLogger(), - }); + const provider = makeRuntimeProvider(client); await expect(provider.exec({ sandboxId: 'test-tenant.gone', command: 'true' })).rejects.toBeInstanceOf( SandboxNotAvailableError, diff --git a/packages/trueforge/src/runtime/sessionResources.ts b/packages/trueforge/src/runtime/sessionResources.ts index 862ea0674..e9933728e 100644 --- a/packages/trueforge/src/runtime/sessionResources.ts +++ b/packages/trueforge/src/runtime/sessionResources.ts @@ -25,7 +25,7 @@ import { isMcpAuthRequired, resolveMcpAuth } from '../mcp/auth/mcpDcr'; import type { IOAuthTokenStore } from '../mcp/auth/types'; import { LocalSandboxProvider } from '../sandbox/local/provider/LocalSandboxProvider'; import { getCachedLocalSandboxSupport, isLocalSandboxFallbackEnabled } from '../sandbox/localRuntime'; -import { toDaytonaSandboxProvider } from '../sandbox/providerUtils'; +import { recordDaytonaAccessFailure, toDaytonaSandboxProvider } from '../sandbox/providerUtils'; import type { ReasoningEffort } from '../schemas/modelProvider'; export interface McpConnection { @@ -248,6 +248,14 @@ export async function resolveSandboxProvider({ tenant_id, logger, build_metadata: record.build_metadata, + onError: async error => { + await recordDaytonaAccessFailure({ + store, + tenant_id, + error, + build_metadata: record.build_metadata, + }); + }, }); } if (!configuration.STANDALONE) { diff --git a/packages/trueforge/src/sandbox/providerUtils.ts b/packages/trueforge/src/sandbox/providerUtils.ts index 0d815dc36..d3c93fe0c 100644 --- a/packages/trueforge/src/sandbox/providerUtils.ts +++ b/packages/trueforge/src/sandbox/providerUtils.ts @@ -18,11 +18,15 @@ import { /** Daytona rejected the credentials (401 unauthorized); retrying the same key cannot succeed. */ export function isDaytonaAuthError(error: unknown): boolean { - return error instanceof DaytonaError && error.statusCode === 401; + return error instanceof DaytonaError + ? error.statusCode === 401 + : error instanceof Error && error.cause !== undefined && isDaytonaAuthError(error.cause); } export function isDaytonaPermissionError(error: unknown): boolean { - return error instanceof DaytonaError && error.statusCode === 403; + return error instanceof DaytonaError + ? error.statusCode === 403 + : error instanceof Error && error.cause !== undefined && isDaytonaPermissionError(error.cause); } /** @@ -38,11 +42,13 @@ export function toDaytonaSandboxProvider({ tenant_id, logger, build_metadata, + onError, }: { manifest: SandboxProviderManifest; tenant_id: string; logger: Logger; build_metadata?: SandboxBuildMetadata | null; + onError?: ((error: unknown) => Promise) | undefined; }): DaytonaSandboxProvider { const { apiKey, ...settings } = toDaytonaSandboxProviderInput(manifest); return new DaytonaSandboxProvider({ @@ -54,6 +60,7 @@ export function toDaytonaSandboxProvider({ buildRef: build_metadata?.['build_ref'], fileMaxBytesForDownload: configuration.SANDBOX_FILE_MAX_BYTES_FOR_DOWNLOAD, logger, + onError, }); } @@ -74,6 +81,30 @@ function sandboxStatusFromRecord(record: SandboxProviderRecord): SandboxStatus { }; } +export async function recordDaytonaAccessFailure({ + store, + tenant_id, + error, + build_metadata, +}: { + store: ISandboxProviderStore; + tenant_id: string; + error: unknown; + build_metadata?: SandboxBuildMetadata | null; +}): Promise { + const status_reason = isDaytonaAuthError(error) + ? 'Daytona rejected the API key. Check the configured credentials.' + : isDaytonaPermissionError(error) + ? 'Daytona denied access. Check the API key permissions.' + : undefined; + if (status_reason === undefined) { + return undefined; + } + const next: SandboxStatus = { status: 'failed', status_reason, build_metadata: build_metadata ?? null }; + const updated = await store.updateSandboxStatus({ tenant_id, ...next }); + return updated ? sandboxStatusFromRecord(updated) : next; +} + // Daytona deactivates idle snapshots after 14 days; revalidate at 13 to stay a day ahead. const READY_REVALIDATE_INTERVAL_MS = 13 * 24 * 60 * 60 * 1000; @@ -109,11 +140,28 @@ export async function checkSnapshotStatus({ build_metadata: record.build_metadata, }); let build: SandboxBuild; - if (record.status === 'ready') { - // this is because image may have deactivated - build = await withTimeout(provider.buildImage(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox buildImage'); - } else { - build = await withTimeout(provider.getImageBuildStatus(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox getImageBuildStatus'); + try { + if (record.status === 'ready') { + // this is because image may have deactivated + build = await withTimeout(provider.buildImage(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox buildImage'); + } else { + build = await withTimeout( + provider.getImageBuildStatus(), + STATUS_REFRESH_TIMEOUT_MS, + 'sandbox getImageBuildStatus', + ); + } + } catch (error) { + const failed = await recordDaytonaAccessFailure({ + store, + tenant_id, + error, + build_metadata: record.build_metadata, + }); + if (failed !== undefined) { + return failed; + } + throw error; } const next = toSandboxStatus(build); const updated = await store.updateSandboxStatus({ tenant_id, ...next }); diff --git a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts new file mode 100644 index 000000000..40e7a2268 --- /dev/null +++ b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts @@ -0,0 +1,60 @@ +jest.mock('@daytona/sdk', () => { + const actual = jest.requireActual('@daytona/sdk'); + return { ...actual, Daytona: jest.fn().mockImplementation(() => ({})) }; +}); + +import { DaytonaError } from '@daytona/sdk'; +import { DaytonaSandboxProvider } from '@truefoundry/trueforge-core/core'; +import { createLogger } from 'winston'; +import type { ISandboxProviderStore, SandboxProviderRecord } from '../../../src/db/sandboxProviderStore'; +import { checkSnapshotStatus } from '../../../src/sandbox/providerUtils'; + +const record: SandboxProviderRecord = { + tenant_id: 'tenant-1', + manifest: { + type: 'daytona', + auth: { api_key: 'dtn-revoked' }, + exec_timeout_ms: 60_000, + auto_stop_interval_in_minutes: 5, + auto_archive_interval_in_minutes: 60, + auto_delete_interval_in_minutes: 7200, + }, + status: 'pending', + status_reason: null, + build_metadata: null, + created_at: '2026-08-01T00:00:00.000Z', + updated_at: '2026-08-01T00:00:00.000Z', +}; + +function makeStore(): ISandboxProviderStore { + return { + getSandboxProvider: jest.fn().mockResolvedValue(record), + getSandboxProviderForUpdate: jest.fn(), + upsertSandboxProvider: jest.fn(), + updateSandboxStatus: jest.fn().mockImplementation(async input => ({ ...record, ...input })), + }; +} + +afterEach(() => { + jest.restoreAllMocks(); +}); + +it('persists Daytona authentication failures instead of surfacing a settings error', async () => { + jest + .spyOn(DaytonaSandboxProvider.prototype, 'getImageBuildStatus') + .mockRejectedValue(new DaytonaError('unauthorized', 401)); + const store = makeStore(); + + await expect( + checkSnapshotStatus({ store, tenant_id: record.tenant_id, logger: createLogger({ silent: true }) }), + ).resolves.toMatchObject({ + status: 'failed', + status_reason: 'Daytona rejected the API key. Check the configured credentials.', + }); + expect(store.updateSandboxStatus).toHaveBeenCalledWith({ + tenant_id: record.tenant_id, + status: 'failed', + status_reason: 'Daytona rejected the API key. Check the configured credentials.', + build_metadata: null, + }); +}); From c1426dfceac41f0cc1ca9e5b3ed600d27a4bf9f3 Mon Sep 17 00:00:00 2001 From: Elioooon Date: Mon, 31 Aug 2026 18:13:11 +0800 Subject: [PATCH 2/3] fix: avoid stale sandbox status writes --- .../PostgresSandboxProviderStore.ts | 11 +++++++---- .../trueforge/src/db/sandboxProviderStore.ts | 1 + .../SqliteSandboxProviderStore.ts | 11 +++++++---- .../trueforge/src/runtime/sessionResources.ts | 1 + .../trueforge/src/sandbox/providerUtils.ts | 19 +++++++++++++++---- .../db/sandboxProviderStoreContractSuite.ts | 16 ++++++++++++++++ .../tests/unit/sandbox/providerUtils.test.ts | 1 + 7 files changed, 48 insertions(+), 12 deletions(-) diff --git a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts index c0ca35dfb..30cae739b 100644 --- a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts +++ b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts @@ -88,7 +88,8 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore, ): Promise { const db = transaction ?? this.#db; - const row = await db + const expectedUpdatedAt = input.expected_updated_at; + let query = db .updateTable('sandbox_provider') .set({ status: input.status, @@ -96,9 +97,11 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore ISandboxPro }), ).toBeUndefined(); }); + + it('updateSandboxStatus ignores a stale provider version', async () => { + const store = getStore(); + await store.upsertSandboxProvider(upsertInput()); + + expect( + await store.updateSandboxStatus({ + tenant_id: TENANT, + status: 'failed', + status_reason: 'stale failure', + build_metadata: BUILD_METADATA, + expected_updated_at: '1970-01-01T00:00:00.000Z', + }), + ).toBeUndefined(); + expect((await store.getSandboxProvider(TENANT))?.status).toBe('pending'); + }); } diff --git a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts index 40e7a2268..4eb9dc892 100644 --- a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts +++ b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts @@ -56,5 +56,6 @@ it('persists Daytona authentication failures instead of surfacing a settings err status: 'failed', status_reason: 'Daytona rejected the API key. Check the configured credentials.', build_metadata: null, + expected_updated_at: record.updated_at, }); }); From f05658156971bf30f49423232226c0fa7a9daaad Mon Sep 17 00:00:00 2001 From: Elioooon Date: Mon, 31 Aug 2026 18:41:03 +0800 Subject: [PATCH 3/3] fix: compare sandbox provider manifests --- .../PostgresSandboxProviderStore.ts | 6 +++--- packages/trueforge/src/db/sandboxProviderStore.ts | 2 +- .../SqliteSandboxProviderStore.ts | 8 ++++---- packages/trueforge/src/runtime/sessionResources.ts | 2 +- packages/trueforge/src/sandbox/providerUtils.ts | 10 +++++----- .../tests/db/sandboxProviderStoreContractSuite.ts | 4 ++-- .../trueforge/tests/unit/sandbox/providerUtils.test.ts | 2 +- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts index 30cae739b..74bf9308d 100644 --- a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts +++ b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts @@ -88,7 +88,7 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore, ): Promise { const db = transaction ?? this.#db; - const expectedUpdatedAt = input.expected_updated_at; + const expectedManifest = input.expected_manifest; let query = db .updateTable('sandbox_provider') .set({ @@ -98,8 +98,8 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore, ): Promise { const db = transaction ?? this.#db; - const expectedUpdatedAt = input.expected_updated_at; + const expectedManifest = input.expected_manifest; let query = db .updateTable('sandbox_provider') .set({ @@ -101,8 +101,8 @@ export class SqliteSandboxProviderStore implements ISandboxProviderStore ISandboxPro ).toBeUndefined(); }); - it('updateSandboxStatus ignores a stale provider version', async () => { + it('updateSandboxStatus ignores a stale provider manifest', async () => { const store = getStore(); await store.upsertSandboxProvider(upsertInput()); @@ -124,7 +124,7 @@ export function runSandboxProviderStoreContractSuite(getStore: () => ISandboxPro status: 'failed', status_reason: 'stale failure', build_metadata: BUILD_METADATA, - expected_updated_at: '1970-01-01T00:00:00.000Z', + expected_manifest: manifest({ exec_timeout_ms: 120000 }), }), ).toBeUndefined(); expect((await store.getSandboxProvider(TENANT))?.status).toBe('pending'); diff --git a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts index 4eb9dc892..e69756a6f 100644 --- a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts +++ b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts @@ -56,6 +56,6 @@ it('persists Daytona authentication failures instead of surfacing a settings err status: 'failed', status_reason: 'Daytona rejected the API key. Check the configured credentials.', build_metadata: null, - expected_updated_at: record.updated_at, + expected_manifest: record.manifest, }); });