diff --git a/packages/auth/src/lib/billing/autumn/features.ts b/packages/auth/src/lib/billing/autumn/features.ts index 4d98c6abe79..df596f7295d 100644 --- a/packages/auth/src/lib/billing/autumn/features.ts +++ b/packages/auth/src/lib/billing/autumn/features.ts @@ -1,5 +1,9 @@ -export const BOX_GB_MINUTES_FEATURE_ID = "box_gb_minutes"; -export const ACTIVE_CPU_MINUTES_FEATURE_ID = "active_cpu_minutes"; +import type { CloudUsageFeature } from "@superset/db/enums"; + +export const BOX_GB_MINUTES_FEATURE_ID = + "box_gb_minutes" satisfies CloudUsageFeature; +export const ACTIVE_CPU_MINUTES_FEATURE_ID = + "active_cpu_minutes" satisfies CloudUsageFeature; export const SEATS_FEATURE_ID = "seats"; export const ENVIRONMENTS_FEATURE_ID = "environments"; export const PRO_ACCESS_FEATURE_ID = "pro_access"; diff --git a/packages/trpc/src/lib/control-log.test.ts b/packages/trpc/src/lib/control-log.test.ts deleted file mode 100644 index b6135997448..00000000000 --- a/packages/trpc/src/lib/control-log.test.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import * as Sentry from "@sentry/core"; -import { stub } from "../../test/stub"; -import { logControl } from "./control-log"; - -describe("logControl", () => { - const logged: unknown[] = []; - stub(Sentry.logger, { - warn: (...args: unknown[]) => logged.push(args), - }); - - test("writes the event as the body with the control source and ids", () => { - logControl( - "meter.cap_reached", - { cloudWorkspaceId: "ws_1", organizationId: "org_1", bootId: "boot_1" }, - { stop_in_ms: 300_000 }, - "warn", - ); - expect(logged).toEqual([ - [ - "meter.cap_reached", - { - "log.source": "control", - "cloud_workspace.id": "ws_1", - "organization.id": "org_1", - "boot.id": "boot_1", - stop_in_ms: 300_000, - }, - ], - ]); - }); -}); diff --git a/packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts b/packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts index d6b4222697b..f7d4449be84 100644 --- a/packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts +++ b/packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts @@ -11,6 +11,7 @@ import { agentCredentials, cloudWorkspaces } from "@superset/db/schema"; import { CHATGPT_PROVIDER } from "@superset/shared/agent-credentials"; import { and, eq, gt, inArray, isNull } from "drizzle-orm"; import { renewChatgptSignIns } from "../../router/agent-credential"; +import { controlTags } from "../control-log"; import { buildSandboxClaim } from "./claim"; import { applySandboxPolicy } from "./vercel"; @@ -68,10 +69,10 @@ export async function renewChatgptSignInsAndBoxes(args: { if (outcome === "applied") boxesUpdated++; } catch (error) { Sentry.captureException(error, { - tags: { - "cloud_workspace.id": row.id, - "organization.id": row.organizationId, - }, + tags: controlTags({ + cloudWorkspaceId: row.id, + organizationId: row.organizationId, + }), }); } } diff --git a/packages/trpc/src/router/cloud-workspace/provision.test.ts b/packages/trpc/src/router/cloud-workspace/provision.test.ts index 92542b96999..9e5526ba659 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.test.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.test.ts @@ -9,39 +9,44 @@ import * as chatMoves from "../chat-move/utils/queueChatMove"; import { provisionCloudWorkspace } from "./provision"; import * as transition from "./transition"; +const logged: unknown[] = []; +const captured: unknown[] = []; +let claimError: unknown; +stub(db.query.cloudWorkspaces, { + findFirst: async () => ({ + id: "ws_1", + organizationId: "org_1", + status: "provisioning", + providerSandboxId: "ws-box", + }), +}); +stub(sandbox, { + buildSandboxClaim: async () => { + throw claimError; + }, + deleteSandbox: async () => {}, +}); +stub(realtime, { nudge: () => {} }); +stub(chatMoves, { failChatMovesTo: async () => {} }); +stub(Sentry.logger, { + warn: (...args: unknown[]) => logged.push(["warn", ...args]), + error: (...args: unknown[]) => logged.push(["error", ...args]), +}); +stub(Sentry, { + captureException: (...args: unknown[]) => captured.push(args), +}); + describe("a provision that Autumn refuses", () => { - const logged: unknown[] = []; - const captured: unknown[] = []; - stub(db.query.cloudWorkspaces, { - findFirst: async () => ({ - id: "ws_1", - organizationId: "org_1", - status: "provisioning", - providerSandboxId: "ws-box", - }), - }); - stub(sandbox, { - buildSandboxClaim: async () => { - throw userError({ - code: "FORBIDDEN", - message: "limit reached", - i18nKey: "serverError.cloudWorkspace.usageLimitReached", - }); - }, - deleteSandbox: async () => {}, - }); stub(transition, { transitionCloudWorkspace: async () => true }); - stub(realtime, { nudge: () => {} }); - stub(chatMoves, { failChatMovesTo: async () => {} }); - stub(Sentry.logger, { - warn: (...args: unknown[]) => logged.push(["warn", ...args]), - error: (...args: unknown[]) => logged.push(["error", ...args]), - }); - stub(Sentry, { - captureException: (...args: unknown[]) => captured.push(args), - }); test("fails the workspace with a warning and opens no Sentry issue", async () => { + logged.length = 0; + captured.length = 0; + claimError = userError({ + code: "FORBIDDEN", + message: "limit reached", + i18nKey: "serverError.cloudWorkspace.usageLimitReached", + }); expect(await provisionCloudWorkspace({ cloudWorkspaceId: "ws_1" })).toBe( "failed", ); @@ -54,10 +59,24 @@ describe("a provision that Autumn refuses", () => { "cloud_workspace.id": "ws_1", "organization.id": "org_1", "failure.site": "provision", - "status.changed": true, }, ], ]); expect(captured).toEqual([]); }); }); + +describe("a provision whose row another write already moved", () => { + stub(transition, { transitionCloudWorkspace: async () => false }); + + test("writes no workspace.failed and opens no Sentry issue", async () => { + logged.length = 0; + captured.length = 0; + claimError = new Error("create failed"); + expect(await provisionCloudWorkspace({ cloudWorkspaceId: "ws_1" })).toBe( + "failed", + ); + expect(logged).toEqual([]); + expect(captured).toEqual([]); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/provision.ts b/packages/trpc/src/router/cloud-workspace/provision.ts index c5028193409..ccb6535e53c 100644 --- a/packages/trpc/src/router/cloud-workspace/provision.ts +++ b/packages/trpc/src/router/cloud-workspace/provision.ts @@ -2,11 +2,10 @@ import { randomBytes } from "node:crypto"; import * as Sentry from "@sentry/core"; import { db } from "@superset/db/client"; import { cloudWorkspaces } from "@superset/db/schema"; -import { isCloudUsageRefusal } from "@superset/shared/billing"; import type { CloudAgentLaunch } from "@superset/shared/cloud-agent-launch"; import { CLOUD_WORKSPACE_PROVISION_TRANSACTION } from "@superset/shared/constants"; import { eq } from "drizzle-orm"; -import { controlTags, logControl } from "../../lib/control-log"; +import { controlTags } from "../../lib/control-log"; import { nudge } from "../../lib/realtime"; import { buildSandboxClaim, @@ -24,7 +23,7 @@ import { import { generateCloudWorkspaceName } from "./generate-name"; import { attachBootSession, authorizeBoot, boxFor, releaseBoot } from "./meter"; import { recordSandboxAwake } from "./sandbox-expiry"; -import { transitionCloudWorkspace } from "./transition"; +import { failCloudWorkspace, transitionCloudWorkspace } from "./transition"; export const FALLBACK_NAME = "Cloud workspace"; @@ -235,13 +234,15 @@ async function provision( teardownError, ); }); - const changed = await transitionCloudWorkspace({ - id: row.id, - from: ["provisioning", "awake", "asleep"], - to: "failed", - set: keepForDiagnosis ? { providerSandboxId } : {}, - }); - nudge(row.organizationId, "cloud_workspaces"); + await failCloudWorkspace( + { cloudWorkspaceId: row.id, organizationId: row.organizationId, bootId }, + { + from: ["provisioning", "awake", "asleep"], + set: keepForDiagnosis ? { providerSandboxId } : {}, + site: "provision", + error, + }, + ); await failChatMovesTo(row.id, "The cloud workspace could not start").catch( (moveError) => console.error( @@ -250,21 +251,6 @@ async function provision( ), ); console.error(`[cloud-workspace] provisioning failed for ${row.id}`, error); - const scope = { - cloudWorkspaceId: row.id, - organizationId: row.organizationId, - bootId, - }; - const refused = isCloudUsageRefusal(error); - logControl( - "workspace.failed", - scope, - { "failure.site": "provision", "status.changed": changed }, - refused ? "warn" : "error", - ); - if (changed && !refused) { - Sentry.captureException(error, { tags: controlTags(scope) }); - } return "failed"; } } diff --git a/packages/trpc/src/router/cloud-workspace/start.test.ts b/packages/trpc/src/router/cloud-workspace/start.test.ts new file mode 100644 index 00000000000..8ac000c1d9a --- /dev/null +++ b/packages/trpc/src/router/cloud-workspace/start.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, test } from "bun:test"; +import * as Sentry from "@sentry/core"; +import { stub } from "../../../test/stub"; +import * as realtime from "../../lib/realtime"; +import * as chatMoves from "../chat-move/utils/queueChatMove"; +import * as jobs from "./jobs"; +import { queueProvision } from "./start"; +import * as transition from "./transition"; + +describe("a provision that cannot be queued", () => { + const nudged: unknown[] = []; + stub(jobs, { + publishCloudWorkspaceJob: async () => { + throw new Error("queue down"); + }, + }); + stub(transition, { transitionCloudWorkspace: async () => true }); + stub(realtime, { nudge: (...args: unknown[]) => nudged.push(args) }); + stub(chatMoves, { failChatMovesTo: async () => {} }); + stub(Sentry, { captureException: () => "" }); + + test("tells the organization's clients that the row failed", async () => { + await expect( + queueProvision({ cloudWorkspaceId: "ws_1" }, "org_1"), + ).rejects.toThrow(); + expect(nudged).toEqual([["org_1", "cloud_workspaces"]]); + }); +}); diff --git a/packages/trpc/src/router/cloud-workspace/start.ts b/packages/trpc/src/router/cloud-workspace/start.ts index 780aab0e025..556fd49d5f7 100644 --- a/packages/trpc/src/router/cloud-workspace/start.ts +++ b/packages/trpc/src/router/cloud-workspace/start.ts @@ -16,7 +16,7 @@ import { waitUntil } from "@vercel/functions"; import { and, eq, inArray, isNull } from "drizzle-orm"; import { anchorAttachments } from "../../lib/attachments"; import { resolveMachineSize } from "../../lib/billing/plan"; -import { controlTags, logControl } from "../../lib/control-log"; +import { logControl } from "../../lib/control-log"; import { githubRepositoriesOutOfReach, githubUserTokenFor, @@ -41,7 +41,7 @@ import { sandboxNameFor, } from "./provision"; import { linkTask } from "./record"; -import { transitionCloudWorkspace } from "./transition"; +import { failCloudWorkspace, transitionCloudWorkspace } from "./transition"; /** An environment this user may start a workspace from, or a user-facing NOT_FOUND. */ export async function loadUsableEnvironment(args: { @@ -320,11 +320,10 @@ export async function queueProvision( } catch (error) { // Nothing was provisioned, so there is no sandbox to tear down — // but the row must not sit in `provisioning` with no job coming. - const changed = await transitionCloudWorkspace({ - id: job.cloudWorkspaceId, - from: ["provisioning"], - to: "failed", - }); + await failCloudWorkspace( + { cloudWorkspaceId: job.cloudWorkspaceId, organizationId }, + { from: ["provisioning"], site: "queue", error }, + ); await failChatMovesTo( job.cloudWorkspaceId, "The cloud workspace could not start", @@ -338,14 +337,6 @@ export async function queueProvision( `[cloud-workspace] could not queue provisioning for ${job.cloudWorkspaceId}`, error, ); - const scope = { cloudWorkspaceId: job.cloudWorkspaceId, organizationId }; - logControl( - "workspace.failed", - scope, - { "failure.site": "queue", "status.changed": changed }, - "error", - ); - if (changed) Sentry.captureException(error, { tags: controlTags(scope) }); throw userError({ code: "INTERNAL_SERVER_ERROR", message: "Could not start cloud workspace provisioning", diff --git a/packages/trpc/src/router/cloud-workspace/transition.ts b/packages/trpc/src/router/cloud-workspace/transition.ts index 985f22c5467..69e7a0dd2f9 100644 --- a/packages/trpc/src/router/cloud-workspace/transition.ts +++ b/packages/trpc/src/router/cloud-workspace/transition.ts @@ -1,7 +1,15 @@ +import * as Sentry from "@sentry/core"; import { db } from "@superset/db/client"; import type { CloudWorkspaceStatus } from "@superset/db/schema"; import { cloudWorkspaces } from "@superset/db/schema"; +import { isCloudUsageRefusal } from "@superset/shared/billing"; import { and, eq, inArray } from "drizzle-orm"; +import { + type ControlScope, + controlTags, + logControl, +} from "../../lib/control-log"; +import { nudge } from "../../lib/realtime"; /** * Moves a cloud workspace between statuses only from the ones it may leave. @@ -28,3 +36,33 @@ export async function transitionCloudWorkspace(args: { .returning({ id: cloudWorkspaces.id }); return moved.length > 0; } + +/** An Autumn refusal is an expected state: it is logged as a warning and opens no Sentry issue. */ +export async function failCloudWorkspace( + scope: ControlScope, + args: { + from: readonly CloudWorkspaceStatus[]; + set?: Partial; + site: "provision" | "queue"; + error: unknown; + }, +): Promise { + const changed = await transitionCloudWorkspace({ + id: scope.cloudWorkspaceId, + from: args.from, + to: "failed", + set: args.set, + }); + nudge(scope.organizationId, "cloud_workspaces"); + if (!changed) return; + const refused = isCloudUsageRefusal(args.error); + logControl( + "workspace.failed", + scope, + { "failure.site": args.site }, + refused ? "warn" : "error", + ); + if (!refused) { + Sentry.captureException(args.error, { tags: controlTags(scope) }); + } +}