diff --git a/.changeset/persist-daytona-auth-errors.md b/.changeset/persist-daytona-auth-errors.md new file mode 100644 index 000000000..d03207257 --- /dev/null +++ b/.changeset/persist-daytona-auth-errors.md @@ -0,0 +1,6 @@ +--- +'@truefoundry/trueforge': patch +'@truefoundry/trueforge-core': patch +--- + +Persist failed Daytona credentials when sandbox operations return authorization errors. diff --git a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts index 1fba52e16..176b92580 100644 --- a/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts +++ b/packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts @@ -99,6 +99,7 @@ export interface DaytonaSandboxProviderOptions { /** Defaults to 1 hour (same as the gateway's max agent execution time). */ previewUrlExpirySeconds?: number; logger: Logger; + onError?: ((error: unknown) => Promise) | undefined; } export class DaytonaSandboxProvider implements SandboxProvider { @@ -118,6 +119,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { private readonly apiKey: string; private readonly apiUrl: string; private readonly logger: Logger; + private readonly onError: ((error: unknown) => Promise) | undefined; private readonly daytona: Daytona; private static readonly cachedSandboxes = new Map(); // De-dupes concurrent recovery attempts on the same sandbox to a single refreshData+start round-trip. @@ -138,6 +140,15 @@ export class DaytonaSandboxProvider implements SandboxProvider { this.natsBridgePort = options.natsBridgePort ?? DEFAULT_SANDBOX_NATS_WS_PORT; this.previewUrlExpirySeconds = options.previewUrlExpirySeconds ?? DEFAULT_PREVIEW_URL_EXPIRY_SECONDS; this.logger = options.logger.child({ module: 'DaytonaProvider' }); + this.onError = options.onError; + } + + private async reportError(error: unknown): Promise { + try { + await this.onError?.(error); + } catch (reportError) { + this.logger.error('Failed to report Daytona error', extractErrorLogFields(reportError)); + } } private async getOrCreateSandbox(sandboxId?: string): Promise<{ sandbox: Sandbox; defaultTimeoutMs: number }> { @@ -246,11 +257,16 @@ export class DaytonaSandboxProvider implements SandboxProvider { } async createSandbox(): Promise<{ sandboxId: string }> { - return context.with(suppressTracing(context.active()), async () => { - const { sandbox } = await this.getOrCreateSandbox(); - this.logger.debug(`Sandbox created: name=${sandbox.name}`); - return { sandboxId: sandbox.name }; - }); + try { + return await context.with(suppressTracing(context.active()), async () => { + const { sandbox } = await this.getOrCreateSandbox(); + this.logger.debug(`Sandbox created: name=${sandbox.name}`); + return { sandboxId: sandbox.name }; + }); + } catch (error) { + await this.reportError(error); + throw error; + } } /** Resolves undefined when no snapshot carries that name; auth/other failures throw. */ @@ -403,6 +419,7 @@ export class DaytonaSandboxProvider implements SandboxProvider { if (e instanceof SandboxNotAvailableError) { throw e; } + await this.reportError(e); this.logger.error('Sandbox execution error', extractErrorLogFields(e)); const message = e instanceof Error ? e.message : 'Unknown error'; return { success: false, error: message }; diff --git a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts index 2c94b868d..bc135a8db 100644 --- a/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts +++ b/packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts @@ -32,6 +32,23 @@ function makeProvider(): DaytonaSandboxProvider { }); } +function makeRuntimeProvider(client: Daytona, onError?: (error: unknown) => Promise): DaytonaSandboxProvider { + return new DaytonaSandboxProvider({ + client, + apiKey: 'dtn-test', + apiUrl: API_URL, + tenantName: 'test-tenant', + sandboxImage: 'registry.example.com/sandbox:029ea5ff', + timeoutMs: 1000, + autoStopIntervalInMinutes: 5, + autoArchiveIntervalInMinutes: 60, + autoDeleteIntervalInMinutes: 7200, + fileMaxBytesForDownload: 1024, + logger: makeSilentLogger(), + onError, + }); +} + function mockFetch({ status, body }: { status: number; body: unknown }): jest.SpiedFunction { return jest .spyOn(globalThis, 'fetch') @@ -81,22 +98,32 @@ describe('DaytonaSandboxProvider register-only snapshot create', () => { }); describe('DaytonaSandboxProvider exec', () => { + it('reports sandbox creation errors before rethrowing them', async () => { + const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); + jest.spyOn(client, 'create').mockRejectedValue(new DaytonaError('unauthorized', 401)); + const onError = jest.fn().mockResolvedValue(undefined); + const provider = makeRuntimeProvider(client, onError); + + await expect(provider.createSandbox()).rejects.toMatchObject({ statusCode: 401 }); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 })); + }); + + it('reports provider errors before converting them to failed exec results', async () => { + const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); + jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('unauthorized', 401)); + const onError = jest.fn().mockResolvedValue(undefined); + const provider = makeRuntimeProvider(client, onError); + + await expect(provider.exec({ sandboxId: 'test-tenant.expired', command: 'true' })).resolves.toMatchObject({ + success: false, + }); + expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 })); + }); + it('rethrows SandboxNotAvailableError when the sandbox is gone', async () => { const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true }); jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('not found', NOT_FOUND_STATUS)); - const provider = new DaytonaSandboxProvider({ - client, - apiKey: 'dtn-test', - apiUrl: API_URL, - tenantName: 'test-tenant', - sandboxImage: 'registry.example.com/sandbox:029ea5ff', - timeoutMs: 1000, - autoStopIntervalInMinutes: 5, - autoArchiveIntervalInMinutes: 60, - autoDeleteIntervalInMinutes: 7200, - fileMaxBytesForDownload: 1024, - logger: makeSilentLogger(), - }); + const provider = makeRuntimeProvider(client); await expect(provider.exec({ sandboxId: 'test-tenant.gone', command: 'true' })).rejects.toBeInstanceOf( SandboxNotAvailableError, diff --git a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts index c0ca35dfb..74bf9308d 100644 --- a/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts +++ b/packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts @@ -88,7 +88,8 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore, ): Promise { const db = transaction ?? this.#db; - const row = await db + const expectedManifest = input.expected_manifest; + let query = db .updateTable('sandbox_provider') .set({ status: input.status, @@ -96,9 +97,11 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore, ): Promise { const db = transaction ?? this.#db; - return await db + const expectedManifest = input.expected_manifest; + let query = db .updateTable('sandbox_provider') .set({ status: input.status, @@ -99,8 +100,10 @@ export class SqliteSandboxProviderStore implements ISandboxProviderStore { + await recordDaytonaAccessFailure({ + store, + tenant_id, + error, + build_metadata: record.build_metadata, + expected_manifest: record.manifest, + }); + }, }); } if (!configuration.STANDALONE) { diff --git a/packages/trueforge/src/sandbox/providerUtils.ts b/packages/trueforge/src/sandbox/providerUtils.ts index 0d815dc36..2504801b5 100644 --- a/packages/trueforge/src/sandbox/providerUtils.ts +++ b/packages/trueforge/src/sandbox/providerUtils.ts @@ -18,11 +18,15 @@ import { /** Daytona rejected the credentials (401 unauthorized); retrying the same key cannot succeed. */ export function isDaytonaAuthError(error: unknown): boolean { - return error instanceof DaytonaError && error.statusCode === 401; + return error instanceof DaytonaError + ? error.statusCode === 401 + : error instanceof Error && error.cause !== undefined && isDaytonaAuthError(error.cause); } export function isDaytonaPermissionError(error: unknown): boolean { - return error instanceof DaytonaError && error.statusCode === 403; + return error instanceof DaytonaError + ? error.statusCode === 403 + : error instanceof Error && error.cause !== undefined && isDaytonaPermissionError(error.cause); } /** @@ -38,11 +42,13 @@ export function toDaytonaSandboxProvider({ tenant_id, logger, build_metadata, + onError, }: { manifest: SandboxProviderManifest; tenant_id: string; logger: Logger; build_metadata?: SandboxBuildMetadata | null; + onError?: ((error: unknown) => Promise) | undefined; }): DaytonaSandboxProvider { const { apiKey, ...settings } = toDaytonaSandboxProviderInput(manifest); return new DaytonaSandboxProvider({ @@ -54,6 +60,7 @@ export function toDaytonaSandboxProvider({ buildRef: build_metadata?.['build_ref'], fileMaxBytesForDownload: configuration.SANDBOX_FILE_MAX_BYTES_FOR_DOWNLOAD, logger, + onError, }); } @@ -74,6 +81,36 @@ function sandboxStatusFromRecord(record: SandboxProviderRecord): SandboxStatus { }; } +export async function recordDaytonaAccessFailure({ + store, + tenant_id, + error, + build_metadata, + expected_manifest, +}: { + store: ISandboxProviderStore; + tenant_id: string; + error: unknown; + build_metadata?: SandboxBuildMetadata | null; + expected_manifest?: SandboxProviderManifest | undefined; +}): Promise { + const status_reason = isDaytonaAuthError(error) + ? 'Daytona rejected the API key. Check the configured credentials.' + : isDaytonaPermissionError(error) + ? 'Daytona denied access. Check the API key permissions.' + : undefined; + if (status_reason === undefined) { + return undefined; + } + const next: SandboxStatus = { status: 'failed', status_reason, build_metadata: build_metadata ?? null }; + const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_manifest }); + if (updated !== undefined) { + return sandboxStatusFromRecord(updated); + } + const current = await store.getSandboxProvider(tenant_id); + return current === undefined ? undefined : sandboxStatusFromRecord(current); +} + // Daytona deactivates idle snapshots after 14 days; revalidate at 13 to stay a day ahead. const READY_REVALIDATE_INTERVAL_MS = 13 * 24 * 60 * 60 * 1000; @@ -109,13 +146,35 @@ export async function checkSnapshotStatus({ build_metadata: record.build_metadata, }); let build: SandboxBuild; - if (record.status === 'ready') { - // this is because image may have deactivated - build = await withTimeout(provider.buildImage(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox buildImage'); - } else { - build = await withTimeout(provider.getImageBuildStatus(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox getImageBuildStatus'); + try { + if (record.status === 'ready') { + // this is because image may have deactivated + build = await withTimeout(provider.buildImage(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox buildImage'); + } else { + build = await withTimeout( + provider.getImageBuildStatus(), + STATUS_REFRESH_TIMEOUT_MS, + 'sandbox getImageBuildStatus', + ); + } + } catch (error) { + const failed = await recordDaytonaAccessFailure({ + store, + tenant_id, + error, + build_metadata: record.build_metadata, + expected_manifest: record.manifest, + }); + if (failed !== undefined) { + return failed; + } + throw error; } const next = toSandboxStatus(build); - const updated = await store.updateSandboxStatus({ tenant_id, ...next }); - return updated ? sandboxStatusFromRecord(updated) : next; + const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_manifest: record.manifest }); + if (updated !== undefined) { + return sandboxStatusFromRecord(updated); + } + const current = await store.getSandboxProvider(tenant_id); + return current === undefined ? undefined : sandboxStatusFromRecord(current); } diff --git a/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts b/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts index a337d1c0c..c16d455b2 100644 --- a/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts +++ b/packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts @@ -113,4 +113,20 @@ export function runSandboxProviderStoreContractSuite(getStore: () => ISandboxPro }), ).toBeUndefined(); }); + + it('updateSandboxStatus ignores a stale provider manifest', async () => { + const store = getStore(); + await store.upsertSandboxProvider(upsertInput()); + + expect( + await store.updateSandboxStatus({ + tenant_id: TENANT, + status: 'failed', + status_reason: 'stale failure', + build_metadata: BUILD_METADATA, + expected_manifest: manifest({ exec_timeout_ms: 120000 }), + }), + ).toBeUndefined(); + expect((await store.getSandboxProvider(TENANT))?.status).toBe('pending'); + }); } diff --git a/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts new file mode 100644 index 000000000..e69756a6f --- /dev/null +++ b/packages/trueforge/tests/unit/sandbox/providerUtils.test.ts @@ -0,0 +1,61 @@ +jest.mock('@daytona/sdk', () => { + const actual = jest.requireActual('@daytona/sdk'); + return { ...actual, Daytona: jest.fn().mockImplementation(() => ({})) }; +}); + +import { DaytonaError } from '@daytona/sdk'; +import { DaytonaSandboxProvider } from '@truefoundry/trueforge-core/core'; +import { createLogger } from 'winston'; +import type { ISandboxProviderStore, SandboxProviderRecord } from '../../../src/db/sandboxProviderStore'; +import { checkSnapshotStatus } from '../../../src/sandbox/providerUtils'; + +const record: SandboxProviderRecord = { + tenant_id: 'tenant-1', + manifest: { + type: 'daytona', + auth: { api_key: 'dtn-revoked' }, + exec_timeout_ms: 60_000, + auto_stop_interval_in_minutes: 5, + auto_archive_interval_in_minutes: 60, + auto_delete_interval_in_minutes: 7200, + }, + status: 'pending', + status_reason: null, + build_metadata: null, + created_at: '2026-08-01T00:00:00.000Z', + updated_at: '2026-08-01T00:00:00.000Z', +}; + +function makeStore(): ISandboxProviderStore { + return { + getSandboxProvider: jest.fn().mockResolvedValue(record), + getSandboxProviderForUpdate: jest.fn(), + upsertSandboxProvider: jest.fn(), + updateSandboxStatus: jest.fn().mockImplementation(async input => ({ ...record, ...input })), + }; +} + +afterEach(() => { + jest.restoreAllMocks(); +}); + +it('persists Daytona authentication failures instead of surfacing a settings error', async () => { + jest + .spyOn(DaytonaSandboxProvider.prototype, 'getImageBuildStatus') + .mockRejectedValue(new DaytonaError('unauthorized', 401)); + const store = makeStore(); + + await expect( + checkSnapshotStatus({ store, tenant_id: record.tenant_id, logger: createLogger({ silent: true }) }), + ).resolves.toMatchObject({ + status: 'failed', + status_reason: 'Daytona rejected the API key. Check the configured credentials.', + }); + expect(store.updateSandboxStatus).toHaveBeenCalledWith({ + tenant_id: record.tenant_id, + status: 'failed', + status_reason: 'Daytona rejected the API key. Check the configured credentials.', + build_metadata: null, + expected_manifest: record.manifest, + }); +});