From 52de06f8623d04cba1d16714c8f8293915d67942 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:47:17 +0000 Subject: [PATCH 1/4] fix(cloud): nudge clients when a provision cannot be queued queueProvision moved the row to failed but sent no nudge, so other members' sidebars kept the workspace in provisioning until some other change. The provision job and the wake path already nudge after they fail a row; queueProvision has the organization id since #9936. Co-Authored-By: Claude --- .../src/router/cloud-workspace/start.test.ts | 28 +++++++++++++++++++ .../trpc/src/router/cloud-workspace/start.ts | 1 + 2 files changed, 29 insertions(+) create mode 100644 packages/trpc/src/router/cloud-workspace/start.test.ts diff --git a/packages/trpc/src/router/cloud-workspace/start.test.ts b/packages/trpc/src/router/cloud-workspace/start.test.ts new file mode 100644 index 00000000000..8ac000c1d9a --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/start.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from "bun:test"; +import * as Sentry from "@sentry/core"; +import { stub } from "../../../test/stub"; +import * as realtime from "../../lib/realtime"; +import * as chatMoves from "../chat-move/utils/queueChatMove"; +import * as jobs from "./jobs"; +import { queueProvision } from "./start"; +import * as transition from "./transition"; + +describe("a provision that cannot be queued", () => { + const nudged: unknown[] = []; + stub(jobs, { + publishCloudWorkspaceJob: async () => { + throw new Error("queue down"); + }, + }); + stub(transition, { transitionCloudWorkspace: async () => true }); + stub(realtime, { nudge: (...args: unknown[]) => nudged.push(args) }); + stub(chatMoves, { failChatMovesTo: async () => {} }); + stub(Sentry, { captureException: () => "" }); + + test("tells the organization's clients that the row failed", async () => { + await expect( + queueProvision({ cloudWorkspaceId: "ws_1" }, "org_1"), + ).rejects.toThrow(); + expect(nudged).toEqual([["org_1", "cloud_workspaces"]]); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/start.ts b/packages/trpc/src/router/cloud-workspace/start.ts index 780aab0e025..a6c1c13f5a4 100644 --- a/packages/trpc/src/router/cloud-workspace/start.ts +++ b/packages/trpc/src/router/cloud-workspace/start.ts @@ -325,6 +325,7 @@ export async function queueProvision( from: ["provisioning"], to: "failed", }); + nudge(organizationId, "cloud_workspaces"); await failChatMovesTo( job.cloudWorkspaceId, "The cloud workspace could not start", From 7af5dc2d8c4eacd931a7f03979c5f59f5d0a9131 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:48:14 +0000 Subject: [PATCH 2/4] refactor(cloud): one failCloudWorkspace next to transitionCloudWorkspace provision and queueProvision each wrote the same block: move the row to failed, nudge, write workspace.failed, capture the error if the row moved. That block now lives once in transition.ts. Same from, to and set; every caller keeps its console.error line word for word. No behaviour change. markSandboxUnavailable in wake.ts is the third copy; it moves to this function after #9930, which edits wake.ts, merges. Co-Authored-By: Claude --- .../src/router/cloud-workspace/provision.ts | 36 ++++++------------ .../trpc/src/router/cloud-workspace/start.ts | 22 +++-------- .../src/router/cloud-workspace/transition.ts | 37 +++++++++++++++++++ 3 files changed, 54 insertions(+), 41 deletions(-) diff --git a/packages/trpc/src/router/cloud-workspace/provision.ts b/packages/trpc/src/router/cloud-workspace/provision.ts index c5028193409..ccb6535e53c 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.ts @@ -2,11 +2,10 @@ import { randomBytes } from "node:crypto"; import * as Sentry from "@sentry/core"; import { db } from "@superset/db/client"; import { cloudWorkspaces } from "@superset/db/schema"; -import { isCloudUsageRefusal } from "@superset/shared/billing"; import type { CloudAgentLaunch } from "@superset/shared/cloud-agent-launch"; import { CLOUD_WORKSPACE_PROVISION_TRANSACTION } from "@superset/shared/constants"; import { eq } from "drizzle-orm"; -import { controlTags, logControl } from "../../lib/control-log"; +import { controlTags } from "../../lib/control-log"; import { nudge } from "../../lib/realtime"; import { buildSandboxClaim, @@ -24,7 +23,7 @@ import { import { generateCloudWorkspaceName } from "./generate-name"; import { attachBootSession, authorizeBoot, boxFor, releaseBoot } from "./meter"; import { recordSandboxAwake } from "./sandbox-expiry"; -import { transitionCloudWorkspace } from "./transition"; +import { failCloudWorkspace, transitionCloudWorkspace } from "./transition"; export const FALLBACK_NAME = "Cloud workspace"; @@ -235,13 +234,15 @@ async function provision( teardownError, ); }); - const changed = await transitionCloudWorkspace({ - id: row.id, - from: ["provisioning", "awake", "asleep"], - to: "failed", - set: keepForDiagnosis ? { providerSandboxId } : {}, - }); - nudge(row.organizationId, "cloud_workspaces"); + await failCloudWorkspace( + { cloudWorkspaceId: row.id, organizationId: row.organizationId, bootId }, + { + from: ["provisioning", "awake", "asleep"], + set: keepForDiagnosis ? { providerSandboxId } : {}, + site: "provision", + error, + }, + ); await failChatMovesTo(row.id, "The cloud workspace could not start").catch( (moveError) => console.error( @@ -250,21 +251,6 @@ async function provision( ), ); console.error(`[cloud-workspace] provisioning failed for ${row.id}`, error); - const scope = { - cloudWorkspaceId: row.id, - organizationId: row.organizationId, - bootId, - }; - const refused = isCloudUsageRefusal(error); - logControl( - "workspace.failed", - scope, - { "failure.site": "provision", "status.changed": changed }, - refused ? "warn" : "error", - ); - if (changed && !refused) { - Sentry.captureException(error, { tags: controlTags(scope) }); - } return "failed"; } } diff --git a/packages/trpc/src/router/cloud-workspace/start.ts b/packages/trpc/src/router/cloud-workspace/start.ts index a6c1c13f5a4..556fd49d5f7 100644 --- a/packages/trpc/src/router/cloud-workspace/start.ts +++ b/packages/trpc/src/router/cloud-workspace/start.ts @@ -16,7 +16,7 @@ import { waitUntil } from "@vercel/functions"; import { and, eq, inArray, isNull } from "drizzle-orm"; import { anchorAttachments } from "../../lib/attachments"; import { resolveMachineSize } from "../../lib/billing/plan"; -import { controlTags, logControl } from "../../lib/control-log"; +import { logControl } from "../../lib/control-log"; import { githubRepositoriesOutOfReach, githubUserTokenFor, @@ -41,7 +41,7 @@ import { sandboxNameFor, } from "./provision"; import { linkTask } from "./record"; -import { transitionCloudWorkspace } from "./transition"; +import { failCloudWorkspace, transitionCloudWorkspace } from "./transition"; /** An environment this user may start a workspace from, or a user-facing NOT_FOUND. */ export async function loadUsableEnvironment(args: { @@ -320,12 +320,10 @@ export async function queueProvision( } catch (error) { // Nothing was provisioned, so there is no sandbox to tear down — // but the row must not sit in `provisioning` with no job coming. - const changed = await transitionCloudWorkspace({ - id: job.cloudWorkspaceId, - from: ["provisioning"], - to: "failed", - }); - nudge(organizationId, "cloud_workspaces"); + await failCloudWorkspace( + { cloudWorkspaceId: job.cloudWorkspaceId, organizationId }, + { from: ["provisioning"], site: "queue", error }, + ); await failChatMovesTo( job.cloudWorkspaceId, "The cloud workspace could not start", @@ -339,14 +337,6 @@ export async function queueProvision( `[cloud-workspace] could not queue provisioning for ${job.cloudWorkspaceId}`, error, ); - const scope = { cloudWorkspaceId: job.cloudWorkspaceId, organizationId }; - logControl( - "workspace.failed", - scope, - { "failure.site": "queue", "status.changed": changed }, - "error", - ); - if (changed) Sentry.captureException(error, { tags: controlTags(scope) }); throw userError({ code: "INTERNAL_SERVER_ERROR", message: "Could not start cloud workspace provisioning", diff --git a/packages/trpc/src/router/cloud-workspace/transition.ts b/packages/trpc/src/router/cloud-workspace/transition.ts index 985f22c5467..a39fcd29d30 100644 --- a/packages/trpc/src/router/cloud-workspace/transition.ts +++ b/packages/trpc/src/router/cloud-workspace/transition.ts @@ -1,7 +1,15 @@ +import * as Sentry from "@sentry/core"; import { db } from "@superset/db/client"; import type { CloudWorkspaceStatus } from "@superset/db/schema"; import { cloudWorkspaces } from "@superset/db/schema"; +import { isCloudUsageRefusal } from "@superset/shared/billing"; import { and, eq, inArray } from "drizzle-orm"; +import { + type ControlScope, + controlTags, + logControl, +} from "../../lib/control-log"; +import { nudge } from "../../lib/realtime"; /** * Moves a cloud workspace between statuses only from the ones it may leave. @@ -28,3 +36,32 @@ export async function transitionCloudWorkspace(args: { .returning({ id: cloudWorkspaces.id }); return moved.length > 0; } + +/** An Autumn refusal is an expected state: it is logged as a warning and opens no Sentry issue. */ +export async function failCloudWorkspace( + scope: ControlScope, + args: { + from: readonly CloudWorkspaceStatus[]; + set?: Partial; + site: "provision" | "queue"; + error: unknown; + }, +): Promise { + const changed = await transitionCloudWorkspace({ + id: scope.cloudWorkspaceId, + from: args.from, + to: "failed", + set: args.set, + }); + nudge(scope.organizationId, "cloud_workspaces"); + const refused = isCloudUsageRefusal(args.error); + logControl( + "workspace.failed", + scope, + { "failure.site": args.site, "status.changed": changed }, + refused ? "warn" : "error", + ); + if (changed && !refused) { + Sentry.captureException(args.error, { tags: controlTags(scope) }); + } +} From 7a6a49e1f8125b8bed66c33a1d020ec6b70f1390 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:49:03 +0000 Subject: [PATCH 3/4] refactor(cloud): workspace.failed only when the row moved failCloudWorkspace wrote workspace.failed at error level even when the transition matched no row, so the log claimed a failure another write had already recorded (or a delete had overtaken). It now writes the record, and captures, only when this call moved the row to failed, and the status.changed attribute is gone because the event now means it. control-log.test.ts tested that logControl calls Sentry.logger; the provision tests check the whole record, so it is removed. provision.test gains the case where the row did not move. Co-Authored-By: Claude --- packages/trpc/src/lib/control-log.test.ts | 32 -------- .../router/cloud-workspace/provision.test.ts | 79 ++++++++++++------- .../src/router/cloud-workspace/transition.ts | 5 +- 3 files changed, 52 insertions(+), 64 deletions(-) delete mode 100644 packages/trpc/src/lib/control-log.test.ts diff --git a/packages/trpc/src/lib/control-log.test.ts b/packages/trpc/src/lib/control-log.test.ts deleted file mode 100644 index b6135997448..00000000000 --- a/packages/trpc/src/lib/control-log.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import * as Sentry from "@sentry/core"; -import { stub } from "../../test/stub"; -import { logControl } from "./control-log"; - -describe("logControl", () => { - const logged: unknown[] = []; - stub(Sentry.logger, { - warn: (...args: unknown[]) => logged.push(args), - }); - - test("writes the event as the body with the control source and ids", () => { - logControl( - "meter.cap_reached", - { cloudWorkspaceId: "ws_1", organizationId: "org_1", bootId: "boot_1" }, - { stop_in_ms: 300_000 }, - "warn", - ); - expect(logged).toEqual([ - [ - "meter.cap_reached", - { - "log.source": "control", - "cloud_workspace.id": "ws_1", - "organization.id": "org_1", - "boot.id": "boot_1", - stop_in_ms: 300_000, - }, - ], - ]); - }); -}); diff --git a/packages/trpc/src/router/cloud-workspace/provision.test.ts b/packages/trpc/src/router/cloud-workspace/provision.test.ts index 92542b96999..9e5526ba659 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.test.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.test.ts @@ -9,39 +9,44 @@ import * as chatMoves from "../chat-move/utils/queueChatMove"; import { provisionCloudWorkspace } from "./provision"; import * as transition from "./transition"; +const logged: unknown[] = []; +const captured: unknown[] = []; +let claimError: unknown; +stub(db.query.cloudWorkspaces, { + findFirst: async () => ({ + id: "ws_1", + organizationId: "org_1", + status: "provisioning", + providerSandboxId: "ws-box", + }), +}); +stub(sandbox, { + buildSandboxClaim: async () => { + throw claimError; + }, + deleteSandbox: async () => {}, +}); +stub(realtime, { nudge: () => {} }); +stub(chatMoves, { failChatMovesTo: async () => {} }); +stub(Sentry.logger, { + warn: (...args: unknown[]) => logged.push(["warn", ...args]), + error: (...args: unknown[]) => logged.push(["error", ...args]), +}); +stub(Sentry, { + captureException: (...args: unknown[]) => captured.push(args), +}); + describe("a provision that Autumn refuses", () => { - const logged: unknown[] = []; - const captured: unknown[] = []; - stub(db.query.cloudWorkspaces, { - findFirst: async () => ({ - id: "ws_1", - organizationId: "org_1", - status: "provisioning", - providerSandboxId: "ws-box", - }), - }); - stub(sandbox, { - buildSandboxClaim: async () => { - throw userError({ - code: "FORBIDDEN", - message: "limit reached", - i18nKey: "serverError.cloudWorkspace.usageLimitReached", - }); - }, - deleteSandbox: async () => {}, - }); stub(transition, { transitionCloudWorkspace: async () => true }); - stub(realtime, { nudge: () => {} }); - stub(chatMoves, { failChatMovesTo: async () => {} }); - stub(Sentry.logger, { - warn: (...args: unknown[]) => logged.push(["warn", ...args]), - error: (...args: unknown[]) => logged.push(["error", ...args]), - }); - stub(Sentry, { - captureException: (...args: unknown[]) => captured.push(args), - }); test("fails the workspace with a warning and opens no Sentry issue", async () => { + logged.length = 0; + captured.length = 0; + claimError = userError({ + code: "FORBIDDEN", + message: "limit reached", + i18nKey: "serverError.cloudWorkspace.usageLimitReached", + }); expect(await provisionCloudWorkspace({ cloudWorkspaceId: "ws_1" })).toBe( "failed", ); @@ -54,10 +59,24 @@ describe("a provision that Autumn refuses", () => { "cloud_workspace.id": "ws_1", "organization.id": "org_1", "failure.site": "provision", - "status.changed": true, }, ], ]); expect(captured).toEqual([]); }); }); + +describe("a provision whose row another write already moved", () => { + stub(transition, { transitionCloudWorkspace: async () => false }); + + test("writes no workspace.failed and opens no Sentry issue", async () => { + logged.length = 0; + captured.length = 0; + claimError = new Error("create failed"); + expect(await provisionCloudWorkspace({ cloudWorkspaceId: "ws_1" })).toBe( + "failed", + ); + expect(logged).toEqual([]); + expect(captured).toEqual([]); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/transition.ts b/packages/trpc/src/router/cloud-workspace/transition.ts index a39fcd29d30..69e7a0dd2f9 100644 --- a/packages/trpc/src/router/cloud-workspace/transition.ts +++ b/packages/trpc/src/router/cloud-workspace/transition.ts @@ -54,14 +54,15 @@ export async function failCloudWorkspace( set: args.set, }); nudge(scope.organizationId, "cloud_workspaces"); + if (!changed) return; const refused = isCloudUsageRefusal(args.error); logControl( "workspace.failed", scope, - { "failure.site": args.site, "status.changed": changed }, + { "failure.site": args.site }, refused ? "warn" : "error", ); - if (changed && !refused) { + if (!refused) { Sentry.captureException(args.error, { tags: controlTags(scope) }); } } From 522fcde0ea4808117027c74d0ddb8aa8da8c0e33 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:49:54 +0000 Subject: [PATCH 4/4] refactor(cloud): one way to tag captures, and feature ids tied to the db enum - renew-chatgpt-sign-ins builds its capture tags with controlTags, like the cloud workspace routes, instead of by hand. - BOX_GB_MINUTES_FEATURE_ID and ACTIVE_CPU_MINUTES_FEATURE_ID satisfy the cloud_usage_feature enum from 0196, so a rename on either side fails typecheck instead of writing rows Autumn does not know. No billing call changes. Co-Authored-By: Claude --- packages/auth/src/lib/billing/autumn/features.ts | 8 ++++++-- packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts | 9 +++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/auth/src/lib/billing/autumn/features.ts b/packages/auth/src/lib/billing/autumn/features.ts index 4d98c6abe79..df596f7295d 100644 --- a/packages/auth/src/lib/billing/autumn/features.ts +++ b/packages/auth/src/lib/billing/autumn/features.ts @@ -1,5 +1,9 @@ -export const BOX_GB_MINUTES_FEATURE_ID = "box_gb_minutes"; -export const ACTIVE_CPU_MINUTES_FEATURE_ID = "active_cpu_minutes"; +import type { CloudUsageFeature } from "@superset/db/enums"; + +export const BOX_GB_MINUTES_FEATURE_ID = + "box_gb_minutes" satisfies CloudUsageFeature; +export const ACTIVE_CPU_MINUTES_FEATURE_ID = + "active_cpu_minutes" satisfies CloudUsageFeature; export const SEATS_FEATURE_ID = "seats"; export const ENVIRONMENTS_FEATURE_ID = "environments"; export const PRO_ACCESS_FEATURE_ID = "pro_access"; diff --git a/packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts b/packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts index d6b4222697b..f7d4449be84 100644 --- a/packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts +++ b/packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts @@ -11,6 +11,7 @@ import { agentCredentials, cloudWorkspaces } from "@superset/db/schema"; import { CHATGPT_PROVIDER } from "@superset/shared/agent-credentials"; import { and, eq, gt, inArray, isNull } from "drizzle-orm"; import { renewChatgptSignIns } from "../../router/agent-credential"; +import { controlTags } from "../control-log"; import { buildSandboxClaim } from "./claim"; import { applySandboxPolicy } from "./vercel"; @@ -68,10 +69,10 @@ export async function renewChatgptSignInsAndBoxes(args: { if (outcome === "applied") boxesUpdated++; } catch (error) { Sentry.captureException(error, { - tags: { - "cloud_workspace.id": row.id, - "organization.id": row.organizationId, - }, + tags: controlTags({ + cloudWorkspaceId: row.id, + organizationId: row.organizationId, + }), }); } }