fix: persist Daytona authentication failures

.changeset/persist-daytona-auth-errors.md

@@ -0,0 +1,6 @@
+---
+'@truefoundry/trueforge': patch
+'@truefoundry/trueforge-core': patch
+---
+
+Persist failed Daytona credentials when sandbox operations return authorization errors.

packages/trueforge-core/src/core/sandbox/provider/DaytonaProvider.ts

@@ -99,6 +99,7 @@ export interface DaytonaSandboxProviderOptions {
   /** Defaults to 1 hour (same as the gateway's max agent execution time). */
   previewUrlExpirySeconds?: number;
   logger: Logger;
+  onError?: ((error: unknown) => Promise<void>) | undefined;
 }
 
 export class DaytonaSandboxProvider implements SandboxProvider {
@@ -118,6 +119,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {
   private readonly apiKey: string;
   private readonly apiUrl: string;
   private readonly logger: Logger;
+  private readonly onError: ((error: unknown) => Promise<void>) | undefined;
   private readonly daytona: Daytona;
   private static readonly cachedSandboxes = new Map<string, { sandbox: Sandbox; defaultTimeoutMs: number }>();
   // De-dupes concurrent recovery attempts on the same sandbox to a single refreshData+start round-trip.
@@ -138,6 +140,15 @@ export class DaytonaSandboxProvider implements SandboxProvider {
     this.natsBridgePort = options.natsBridgePort ?? DEFAULT_SANDBOX_NATS_WS_PORT;
     this.previewUrlExpirySeconds = options.previewUrlExpirySeconds ?? DEFAULT_PREVIEW_URL_EXPIRY_SECONDS;
     this.logger = options.logger.child({ module: 'DaytonaProvider' });
+    this.onError = options.onError;
+  }
+
+  private async reportError(error: unknown): Promise<void> {
+    try {
+      await this.onError?.(error);
+    } catch (reportError) {
+      this.logger.error('Failed to report Daytona error', extractErrorLogFields(reportError));
+    }
   }
 
   private async getOrCreateSandbox(sandboxId?: string): Promise<{ sandbox: Sandbox; defaultTimeoutMs: number }> {
@@ -246,11 +257,16 @@ export class DaytonaSandboxProvider implements SandboxProvider {
   }
 
   async createSandbox(): Promise<{ sandboxId: string }> {
-    return context.with(suppressTracing(context.active()), async () => {
-      const { sandbox } = await this.getOrCreateSandbox();
-      this.logger.debug(`Sandbox created: name=${sandbox.name}`);
-      return { sandboxId: sandbox.name };
-    });
+    try {
+      return await context.with(suppressTracing(context.active()), async () => {
+        const { sandbox } = await this.getOrCreateSandbox();
+        this.logger.debug(`Sandbox created: name=${sandbox.name}`);
+        return { sandboxId: sandbox.name };
+      });
+    } catch (error) {
+      await this.reportError(error);
+      throw error;
+    }
   }
 
   /** Resolves undefined when no snapshot carries that name; auth/other failures throw. */
@@ -403,6 +419,7 @@ export class DaytonaSandboxProvider implements SandboxProvider {
         if (e instanceof SandboxNotAvailableError) {
           throw e;
         }
+        await this.reportError(e);
         this.logger.error('Sandbox execution error', extractErrorLogFields(e));
         const message = e instanceof Error ? e.message : 'Unknown error';
         return { success: false, error: message };

packages/trueforge-core/tests/core/sandbox/daytonaSnapshotRegistration.test.ts

@@ -32,6 +32,23 @@ function makeProvider(): DaytonaSandboxProvider {
   });
 }
 
+function makeRuntimeProvider(client: Daytona, onError?: (error: unknown) => Promise<void>): DaytonaSandboxProvider {
+  return new DaytonaSandboxProvider({
+    client,
+    apiKey: 'dtn-test',
+    apiUrl: API_URL,
+    tenantName: 'test-tenant',
+    sandboxImage: 'registry.example.com/sandbox:029ea5ff',
+    timeoutMs: 1000,
+    autoStopIntervalInMinutes: 5,
+    autoArchiveIntervalInMinutes: 60,
+    autoDeleteIntervalInMinutes: 7200,
+    fileMaxBytesForDownload: 1024,
+    logger: makeSilentLogger(),
+    onError,
+  });
+}
+
 function mockFetch({ status, body }: { status: number; body: unknown }): jest.SpiedFunction<typeof globalThis.fetch> {
   return jest
     .spyOn(globalThis, 'fetch')
@@ -81,22 +98,32 @@ describe('DaytonaSandboxProvider register-only snapshot create', () => {
 });
 
 describe('DaytonaSandboxProvider exec', () => {
+  it('reports sandbox creation errors before rethrowing them', async () => {
+    const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
+    jest.spyOn(client, 'create').mockRejectedValue(new DaytonaError('unauthorized', 401));
+    const onError = jest.fn().mockResolvedValue(undefined);
+    const provider = makeRuntimeProvider(client, onError);
+
+    await expect(provider.createSandbox()).rejects.toMatchObject({ statusCode: 401 });
+    expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 }));
+  });
+
+  it('reports provider errors before converting them to failed exec results', async () => {
+    const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
+    jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('unauthorized', 401));
+    const onError = jest.fn().mockResolvedValue(undefined);
+    const provider = makeRuntimeProvider(client, onError);
+
+    await expect(provider.exec({ sandboxId: 'test-tenant.expired', command: 'true' })).resolves.toMatchObject({
+      success: false,
+    });
+    expect(onError).toHaveBeenCalledWith(expect.objectContaining({ statusCode: 401 }));
+  });
+
   it('rethrows SandboxNotAvailableError when the sandbox is gone', async () => {
     const client = new Daytona({ apiKey: 'dtn-test', useDeprecatedPolling: true });
     jest.spyOn(client, 'get').mockRejectedValue(new DaytonaError('not found', NOT_FOUND_STATUS));
-    const provider = new DaytonaSandboxProvider({
-      client,
-      apiKey: 'dtn-test',
-      apiUrl: API_URL,
-      tenantName: 'test-tenant',
-      sandboxImage: 'registry.example.com/sandbox:029ea5ff',
-      timeoutMs: 1000,
-      autoStopIntervalInMinutes: 5,
-      autoArchiveIntervalInMinutes: 60,
-      autoDeleteIntervalInMinutes: 7200,
-      fileMaxBytesForDownload: 1024,
-      logger: makeSilentLogger(),
-    });
+    const provider = makeRuntimeProvider(client);
 
     await expect(provider.exec({ sandboxId: 'test-tenant.gone', command: 'true' })).rejects.toBeInstanceOf(
       SandboxNotAvailableError,

packages/trueforge/src/db/postgres/sandbox-provider-store/PostgresSandboxProviderStore.ts

@@ -88,17 +88,20 @@ export class PostgresSandboxProviderStore implements ISandboxProviderStore<Trans
     transaction?: Transaction<Database>,
   ): Promise<SandboxProviderRecord | undefined> {
     const db = transaction ?? this.#db;
-    const row = await db
+    const expectedManifest = input.expected_manifest;
+    let query = db
       .updateTable('sandbox_provider')
       .set({
         status: input.status,
         status_reason: input.status_reason,
         build_metadata: input.build_metadata !== null ? json(input.build_metadata) : null,
         updated_at: now(),
       })
-      .where('tenant_id', '=', input.tenant_id)
-      .returningAll()
-      .executeTakeFirst();
+      .where('tenant_id', '=', input.tenant_id);
+    if (expectedManifest !== undefined) {
+      query = query.where('manifest', '=', json(expectedManifest));
+    }
+    const row = await query.returningAll().executeTakeFirst();
     return row === undefined ? undefined : toRecord(row);
   }
 }

packages/trueforge/src/db/sandboxProviderStore.ts

@@ -34,6 +34,7 @@ export interface UpdateSandboxStatusInput {
   status: SandboxBuildStatus;
   status_reason: string | null;
   build_metadata: SandboxBuildMetadata | null;
+  expected_manifest?: SandboxProviderManifest | undefined;
 }
 
 export interface ISandboxProviderStore<TTransaction = never> {

packages/trueforge/src/db/sqlite/sandbox-provider-store/SqliteSandboxProviderStore.ts

@@ -1,4 +1,4 @@
-import type { ExpressionBuilder, Kysely, Transaction } from 'kysely';
+import { sql, type ExpressionBuilder, type Kysely, type Transaction } from 'kysely';
 import type { SandboxBuildMetadata, SandboxProviderManifest } from '../../../schemas/sandboxProvider';
 import {
   type ISandboxProviderStore,
@@ -91,16 +91,19 @@ export class SqliteSandboxProviderStore implements ISandboxProviderStore<Transac
     transaction?: Transaction<Database>,
   ): Promise<SandboxProviderRecord | undefined> {
     const db = transaction ?? this.#db;
-    return await db
+    const expectedManifest = input.expected_manifest;
+    let query = db
       .updateTable('sandbox_provider')
       .set({
         status: input.status,
         status_reason: input.status_reason,
         build_metadata: input.build_metadata !== null ? jsonbBind(input.build_metadata) : null,
         updated_at: nowIso(),
       })
-      .where('tenant_id', '=', input.tenant_id)
-      .returning(recordColumns)
-      .executeTakeFirst();
+      .where('tenant_id', '=', input.tenant_id);
+    if (expectedManifest !== undefined) {
+      query = query.where(sql<boolean>`manifest = ${jsonbBind(expectedManifest)}`);
+    }
+    return await query.returning(recordColumns).executeTakeFirst();
   }
 }

packages/trueforge/src/runtime/sessionResources.ts

@@ -25,7 +25,7 @@ import { isMcpAuthRequired, resolveMcpAuth } from '../mcp/auth/mcpDcr';
 import type { IOAuthTokenStore } from '../mcp/auth/types';
 import { LocalSandboxProvider } from '../sandbox/local/provider/LocalSandboxProvider';
 import { getCachedLocalSandboxSupport, isLocalSandboxFallbackEnabled } from '../sandbox/localRuntime';
-import { toDaytonaSandboxProvider } from '../sandbox/providerUtils';
+import { recordDaytonaAccessFailure, toDaytonaSandboxProvider } from '../sandbox/providerUtils';
 import type { ReasoningEffort } from '../schemas/modelProvider';
 
 export interface McpConnection {
@@ -248,6 +248,15 @@ export async function resolveSandboxProvider({
       tenant_id,
       logger,
       build_metadata: record.build_metadata,
+      onError: async error => {
+        await recordDaytonaAccessFailure({
+          store,
+          tenant_id,
+          error,
+          build_metadata: record.build_metadata,
+          expected_manifest: record.manifest,
+        });
+      },
     });
   }
   if (!configuration.STANDALONE) {

packages/trueforge/src/sandbox/providerUtils.ts

@@ -18,11 +18,15 @@ import {
 
 /** Daytona rejected the credentials (401 unauthorized); retrying the same key cannot succeed. */
 export function isDaytonaAuthError(error: unknown): boolean {
-  return error instanceof DaytonaError && error.statusCode === 401;
+  return error instanceof DaytonaError
+    ? error.statusCode === 401
+    : error instanceof Error && error.cause !== undefined && isDaytonaAuthError(error.cause);
 }
 
 export function isDaytonaPermissionError(error: unknown): boolean {
-  return error instanceof DaytonaError && error.statusCode === 403;
+  return error instanceof DaytonaError
+    ? error.statusCode === 403
+    : error instanceof Error && error.cause !== undefined && isDaytonaPermissionError(error.cause);
 }
 
 /**
@@ -38,11 +42,13 @@ export function toDaytonaSandboxProvider({
   tenant_id,
   logger,
   build_metadata,
+  onError,
 }: {
   manifest: SandboxProviderManifest;
   tenant_id: string;
   logger: Logger;
   build_metadata?: SandboxBuildMetadata | null;
+  onError?: ((error: unknown) => Promise<void>) | undefined;
 }): DaytonaSandboxProvider {
   const { apiKey, ...settings } = toDaytonaSandboxProviderInput(manifest);
   return new DaytonaSandboxProvider({
@@ -54,6 +60,7 @@ export function toDaytonaSandboxProvider({
     buildRef: build_metadata?.['build_ref'],
     fileMaxBytesForDownload: configuration.SANDBOX_FILE_MAX_BYTES_FOR_DOWNLOAD,
     logger,
+    onError,
   });
 }
 
@@ -74,6 +81,36 @@ function sandboxStatusFromRecord(record: SandboxProviderRecord): SandboxStatus {
   };
 }
 
+export async function recordDaytonaAccessFailure({
+  store,
+  tenant_id,
+  error,
+  build_metadata,
+  expected_manifest,
+}: {
+  store: ISandboxProviderStore;
+  tenant_id: string;
+  error: unknown;
+  build_metadata?: SandboxBuildMetadata | null;
+  expected_manifest?: SandboxProviderManifest | undefined;
+}): Promise<SandboxStatus | undefined> {
+  const status_reason = isDaytonaAuthError(error)
+    ? 'Daytona rejected the API key. Check the configured credentials.'
+    : isDaytonaPermissionError(error)
+      ? 'Daytona denied access. Check the API key permissions.'
+      : undefined;
+  if (status_reason === undefined) {
+    return undefined;
+  }
+  const next: SandboxStatus = { status: 'failed', status_reason, build_metadata: build_metadata ?? null };
+  const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_manifest });
+  if (updated !== undefined) {
+    return sandboxStatusFromRecord(updated);
+  }
+  const current = await store.getSandboxProvider(tenant_id);
+  return current === undefined ? undefined : sandboxStatusFromRecord(current);
+}
+
 // Daytona deactivates idle snapshots after 14 days; revalidate at 13 to stay a day ahead.
 const READY_REVALIDATE_INTERVAL_MS = 13 * 24 * 60 * 60 * 1000;
 
@@ -109,13 +146,35 @@ export async function checkSnapshotStatus({
     build_metadata: record.build_metadata,
   });
   let build: SandboxBuild;
-  if (record.status === 'ready') {
-    // this is because image may have deactivated
-    build = await withTimeout(provider.buildImage(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox buildImage');
-  } else {
-    build = await withTimeout(provider.getImageBuildStatus(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox getImageBuildStatus');
+  try {
+    if (record.status === 'ready') {
+      // this is because image may have deactivated
+      build = await withTimeout(provider.buildImage(), STATUS_REFRESH_TIMEOUT_MS, 'sandbox buildImage');
+    } else {
+      build = await withTimeout(
+        provider.getImageBuildStatus(),
+        STATUS_REFRESH_TIMEOUT_MS,
+        'sandbox getImageBuildStatus',
+      );
+    }
+  } catch (error) {
+    const failed = await recordDaytonaAccessFailure({
+      store,
+      tenant_id,
+      error,
+      build_metadata: record.build_metadata,
+      expected_manifest: record.manifest,
+    });
+    if (failed !== undefined) {
+      return failed;
+    }
+    throw error;
   }
   const next = toSandboxStatus(build);
-  const updated = await store.updateSandboxStatus({ tenant_id, ...next });
-  return updated ? sandboxStatusFromRecord(updated) : next;
+  const updated = await store.updateSandboxStatus({ tenant_id, ...next, expected_manifest: record.manifest });
+  if (updated !== undefined) {
+    return sandboxStatusFromRecord(updated);
+  }
+  const current = await store.getSandboxProvider(tenant_id);
+  return current === undefined ? undefined : sandboxStatusFromRecord(current);
 }

packages/trueforge/tests/db/sandboxProviderStoreContractSuite.ts

@@ -113,4 +113,20 @@ export function runSandboxProviderStoreContractSuite(getStore: () => ISandboxPro
       }),
     ).toBeUndefined();
   });
+
+  it('updateSandboxStatus ignores a stale provider manifest', async () => {
+    const store = getStore();
+    await store.upsertSandboxProvider(upsertInput());
+
+    expect(
+      await store.updateSandboxStatus({
+        tenant_id: TENANT,
+        status: 'failed',
+        status_reason: 'stale failure',
+        build_metadata: BUILD_METADATA,
+        expected_manifest: manifest({ exec_timeout_ms: 120000 }),
+      }),
+    ).toBeUndefined();
+    expect((await store.getSandboxProvider(TENANT))?.status).toBe('pending');
+  });
 }