Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions packages/auth/src/lib/billing/autumn/features.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
export const BOX_GB_MINUTES_FEATURE_ID = "box_gb_minutes";
export const ACTIVE_CPU_MINUTES_FEATURE_ID = "active_cpu_minutes";
import type { CloudUsageFeature } from "@superset/db/enums";

export const BOX_GB_MINUTES_FEATURE_ID =
"box_gb_minutes" satisfies CloudUsageFeature;
export const ACTIVE_CPU_MINUTES_FEATURE_ID =
"active_cpu_minutes" satisfies CloudUsageFeature;
export const SEATS_FEATURE_ID = "seats";
export const ENVIRONMENTS_FEATURE_ID = "environments";
export const PRO_ACCESS_FEATURE_ID = "pro_access";
Expand Down
32 changes: 0 additions & 32 deletions packages/trpc/src/lib/control-log.test.ts

This file was deleted.

9 changes: 5 additions & 4 deletions packages/trpc/src/lib/sandbox/renew-chatgpt-sign-ins.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import { agentCredentials, cloudWorkspaces } from "@superset/db/schema";
import { CHATGPT_PROVIDER } from "@superset/shared/agent-credentials";
import { and, eq, gt, inArray, isNull } from "drizzle-orm";
import { renewChatgptSignIns } from "../../router/agent-credential";
import { controlTags } from "../control-log";
import { buildSandboxClaim } from "./claim";
import { applySandboxPolicy } from "./vercel";

Expand Down Expand Up @@ -68,10 +69,10 @@ export async function renewChatgptSignInsAndBoxes(args: {
if (outcome === "applied") boxesUpdated++;
} catch (error) {
Sentry.captureException(error, {
tags: {
"cloud_workspace.id": row.id,
"organization.id": row.organizationId,
},
tags: controlTags({
cloudWorkspaceId: row.id,
organizationId: row.organizationId,
}),
});
}
}
Expand Down
79 changes: 49 additions & 30 deletions packages/trpc/src/router/cloud-workspace/provision.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,39 +9,44 @@ import * as chatMoves from "../chat-move/utils/queueChatMove";
import { provisionCloudWorkspace } from "./provision";
import * as transition from "./transition";

const logged: unknown[] = [];
const captured: unknown[] = [];
let claimError: unknown;
stub(db.query.cloudWorkspaces, {
findFirst: async () => ({
id: "ws_1",
organizationId: "org_1",
status: "provisioning",
providerSandboxId: "ws-box",
}),
});
stub(sandbox, {
buildSandboxClaim: async () => {
throw claimError;
},
deleteSandbox: async () => {},
});
stub(realtime, { nudge: () => {} });
stub(chatMoves, { failChatMovesTo: async () => {} });
stub(Sentry.logger, {
warn: (...args: unknown[]) => logged.push(["warn", ...args]),
error: (...args: unknown[]) => logged.push(["error", ...args]),
});
stub(Sentry, {
captureException: (...args: unknown[]) => captured.push(args),
});

describe("a provision that Autumn refuses", () => {
const logged: unknown[] = [];
const captured: unknown[] = [];
stub(db.query.cloudWorkspaces, {
findFirst: async () => ({
id: "ws_1",
organizationId: "org_1",
status: "provisioning",
providerSandboxId: "ws-box",
}),
});
stub(sandbox, {
buildSandboxClaim: async () => {
throw userError({
code: "FORBIDDEN",
message: "limit reached",
i18nKey: "serverError.cloudWorkspace.usageLimitReached",
});
},
deleteSandbox: async () => {},
});
stub(transition, { transitionCloudWorkspace: async () => true });
stub(realtime, { nudge: () => {} });
stub(chatMoves, { failChatMovesTo: async () => {} });
stub(Sentry.logger, {
warn: (...args: unknown[]) => logged.push(["warn", ...args]),
error: (...args: unknown[]) => logged.push(["error", ...args]),
});
stub(Sentry, {
captureException: (...args: unknown[]) => captured.push(args),
});

test("fails the workspace with a warning and opens no Sentry issue", async () => {
logged.length = 0;
captured.length = 0;
claimError = userError({
code: "FORBIDDEN",
message: "limit reached",
i18nKey: "serverError.cloudWorkspace.usageLimitReached",
});
expect(await provisionCloudWorkspace({ cloudWorkspaceId: "ws_1" })).toBe(
"failed",
);
Expand All @@ -54,10 +59,24 @@ describe("a provision that Autumn refuses", () => {
"cloud_workspace.id": "ws_1",
"organization.id": "org_1",
"failure.site": "provision",
"status.changed": true,
},
],
]);
expect(captured).toEqual([]);
});
});

describe("a provision whose row another write already moved", () => {
stub(transition, { transitionCloudWorkspace: async () => false });

test("writes no workspace.failed and opens no Sentry issue", async () => {
logged.length = 0;
captured.length = 0;
claimError = new Error("create failed");
expect(await provisionCloudWorkspace({ cloudWorkspaceId: "ws_1" })).toBe(
"failed",
);
expect(logged).toEqual([]);
expect(captured).toEqual([]);
});
});
36 changes: 11 additions & 25 deletions packages/trpc/src/router/cloud-workspace/provision.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,10 @@ import { randomBytes } from "node:crypto";
import * as Sentry from "@sentry/core";
import { db } from "@superset/db/client";
import { cloudWorkspaces } from "@superset/db/schema";
import { isCloudUsageRefusal } from "@superset/shared/billing";
import type { CloudAgentLaunch } from "@superset/shared/cloud-agent-launch";
import { CLOUD_WORKSPACE_PROVISION_TRANSACTION } from "@superset/shared/constants";
import { eq } from "drizzle-orm";
import { controlTags, logControl } from "../../lib/control-log";
import { controlTags } from "../../lib/control-log";
import { nudge } from "../../lib/realtime";
import {
buildSandboxClaim,
Expand All @@ -24,7 +23,7 @@ import {
import { generateCloudWorkspaceName } from "./generate-name";
import { attachBootSession, authorizeBoot, boxFor, releaseBoot } from "./meter";
import { recordSandboxAwake } from "./sandbox-expiry";
import { transitionCloudWorkspace } from "./transition";
import { failCloudWorkspace, transitionCloudWorkspace } from "./transition";

export const FALLBACK_NAME = "Cloud workspace";

Expand Down Expand Up @@ -235,13 +234,15 @@ async function provision(
teardownError,
);
});
const changed = await transitionCloudWorkspace({
id: row.id,
from: ["provisioning", "awake", "asleep"],
to: "failed",
set: keepForDiagnosis ? { providerSandboxId } : {},
});
nudge(row.organizationId, "cloud_workspaces");
await failCloudWorkspace(
{ cloudWorkspaceId: row.id, organizationId: row.organizationId, bootId },
{
from: ["provisioning", "awake", "asleep"],
set: keepForDiagnosis ? { providerSandboxId } : {},
site: "provision",
error,
},
);
await failChatMovesTo(row.id, "The cloud workspace could not start").catch(
(moveError) =>
console.error(
Expand All @@ -250,21 +251,6 @@ async function provision(
),
);
console.error(`[cloud-workspace] provisioning failed for ${row.id}`, error);
const scope = {
cloudWorkspaceId: row.id,
organizationId: row.organizationId,
bootId,
};
const refused = isCloudUsageRefusal(error);
logControl(
"workspace.failed",
scope,
{ "failure.site": "provision", "status.changed": changed },
refused ? "warn" : "error",
);
if (changed && !refused) {
Sentry.captureException(error, { tags: controlTags(scope) });
}
return "failed";
}
}
28 changes: 28 additions & 0 deletions packages/trpc/src/router/cloud-workspace/start.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
import { describe, expect, test } from "bun:test";
import * as Sentry from "@sentry/core";
import { stub } from "../../../test/stub";
import * as realtime from "../../lib/realtime";
import * as chatMoves from "../chat-move/utils/queueChatMove";
import * as jobs from "./jobs";
import { queueProvision } from "./start";
import * as transition from "./transition";

describe("a provision that cannot be queued", () => {
const nudged: unknown[] = [];
stub(jobs, {
publishCloudWorkspaceJob: async () => {
throw new Error("queue down");
},
});
stub(transition, { transitionCloudWorkspace: async () => true });
stub(realtime, { nudge: (...args: unknown[]) => nudged.push(args) });
stub(chatMoves, { failChatMovesTo: async () => {} });
stub(Sentry, { captureException: () => "" });

test("tells the organization's clients that the row failed", async () => {
await expect(
queueProvision({ cloudWorkspaceId: "ws_1" }, "org_1"),
).rejects.toThrow();
expect(nudged).toEqual([["org_1", "cloud_workspaces"]]);
});
});
21 changes: 6 additions & 15 deletions packages/trpc/src/router/cloud-workspace/start.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import { waitUntil } from "@vercel/functions";
import { and, eq, inArray, isNull } from "drizzle-orm";
import { anchorAttachments } from "../../lib/attachments";
import { resolveMachineSize } from "../../lib/billing/plan";
import { controlTags, logControl } from "../../lib/control-log";
import { logControl } from "../../lib/control-log";
import {
githubRepositoriesOutOfReach,
githubUserTokenFor,
Expand All @@ -41,7 +41,7 @@ import {
sandboxNameFor,
} from "./provision";
import { linkTask } from "./record";
import { transitionCloudWorkspace } from "./transition";
import { failCloudWorkspace, transitionCloudWorkspace } from "./transition";

/** An environment this user may start a workspace from, or a user-facing NOT_FOUND. */
export async function loadUsableEnvironment(args: {
Expand Down Expand Up @@ -320,11 +320,10 @@ export async function queueProvision(
} catch (error) {
// Nothing was provisioned, so there is no sandbox to tear down —
// but the row must not sit in `provisioning` with no job coming.
const changed = await transitionCloudWorkspace({
id: job.cloudWorkspaceId,
from: ["provisioning"],
to: "failed",
});
await failCloudWorkspace(
{ cloudWorkspaceId: job.cloudWorkspaceId, organizationId },
{ from: ["provisioning"], site: "queue", error },
);
await failChatMovesTo(
job.cloudWorkspaceId,
"The cloud workspace could not start",
Expand All @@ -338,14 +337,6 @@ export async function queueProvision(
`[cloud-workspace] could not queue provisioning for ${job.cloudWorkspaceId}`,
error,
);
const scope = { cloudWorkspaceId: job.cloudWorkspaceId, organizationId };
logControl(
"workspace.failed",
scope,
{ "failure.site": "queue", "status.changed": changed },
"error",
);
if (changed) Sentry.captureException(error, { tags: controlTags(scope) });
throw userError({
code: "INTERNAL_SERVER_ERROR",
message: "Could not start cloud workspace provisioning",
Expand Down
38 changes: 38 additions & 0 deletions packages/trpc/src/router/cloud-workspace/transition.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
import * as Sentry from "@sentry/core";
import { db } from "@superset/db/client";
import type { CloudWorkspaceStatus } from "@superset/db/schema";
import { cloudWorkspaces } from "@superset/db/schema";
import { isCloudUsageRefusal } from "@superset/shared/billing";
import { and, eq, inArray } from "drizzle-orm";
import {
type ControlScope,
controlTags,
logControl,
} from "../../lib/control-log";
import { nudge } from "../../lib/realtime";

/**
* Moves a cloud workspace between statuses only from the ones it may leave.
Expand All @@ -28,3 +36,33 @@ export async function transitionCloudWorkspace(args: {
.returning({ id: cloudWorkspaces.id });
return moved.length > 0;
}

/** An Autumn refusal is an expected state: it is logged as a warning and opens no Sentry issue. */
export async function failCloudWorkspace(
scope: ControlScope,
args: {
from: readonly CloudWorkspaceStatus[];
set?: Partial<typeof cloudWorkspaces.$inferInsert>;
site: "provision" | "queue";
error: unknown;
},
): Promise<void> {
const changed = await transitionCloudWorkspace({
id: scope.cloudWorkspaceId,
from: args.from,
to: "failed",
set: args.set,
});
nudge(scope.organizationId, "cloud_workspaces");
if (!changed) return;
const refused = isCloudUsageRefusal(args.error);
logControl(
"workspace.failed",
scope,
{ "failure.site": args.site },
refused ? "warn" : "error",
);
if (!refused) {
Sentry.captureException(args.error, { tags: controlTags(scope) });
}
}
Loading