Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions docs/cloud-sandbox-considerations.md
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,19 @@ host-service's pressure report (`runtime/sandbox-box-pressure`), which
cannot report a box whose host-service has stalled. Still open: automatic resume of an agent whose own process was
reaped while its terminal survived.

## Box logs leave the box

`superset-logship` sends `boot.log`, `memguard.log`, `desktop.log` and
`host-service.log` to the sandbox Sentry project when the claim sets
`SUPERSET_SANDBOX_BOX_LOGS=1`. That needs both PostHog flags:
`sandbox-box-logs-fleet` and the organization's `sandbox-box-logs`. The setup
script and the clone output are never read. **Open:** host-service lines can
still carry customer text: worktree paths, branch names, repository URLs, and
git or `gh` errors that print the command. The shipper scrubs token formats;
host-service no longer prints hook commands. Before the flag is on for a customer organization,
the policy wording (L1) must be live, and someone must decide which of those
lines to drop.

## Untested behaviour

These are unknowns, not known failures — but each could change the design, and
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
Expand Down Expand Up @@ -60,6 +60,26 @@ describe("runSandboxArchiveHook", () => {
expect(result.status === "ok" && result.outputTail).toBe("cloud\ndone\n");
});

it("logs the outcome without a multi-line command", async () => {
writeConfig(
'[archive.cloud]\nrun = """\necho ARCHIVE-LINE-ONE\necho ARCHIVE-LINE-TWO\n"""\n',
);
const printed: unknown[] = [];
const spy = spyOn(console, "log").mockImplementation((...args) => {
printed.push(...args);
});
try {
expect((await runSandboxArchiveHook(identity())).status).toBe("ok");
} finally {
spy.mockRestore();
}
expect(printed).toEqual([
expect.stringMatching(
/^\[sandbox\] archive hook exited 0 \(command of \d+ chars\)$/,
),
]);
});

it("is skipped when there is nothing to run", async () => {
writeConfig('[archive]\nrun = "echo x"\n[archive.cloud]\nrun = []\n');
expect(await runSandboxArchiveHook(identity())).toEqual({
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import { getManagedEnv } from "../sandbox-managed-env/sandbox-managed-env.ts";
import { chainPhases, loadProjectConfig } from "../setup/config";
import type { SandboxIdentity } from "./sandbox-self-seed";
import { buildStartHookEnv } from "./utils/buildStartHookEnv";
import { hookOutcomeLine } from "./utils/hookOutcomeLine";

/** The same limit as a local archive, so `[archive]` has one documented cap. */
export const ARCHIVE_HOOK_TIMEOUT_MS = TEARDOWN_TIMEOUT_MS;
Expand Down Expand Up @@ -96,7 +97,7 @@ export async function runSandboxArchiveHook(
});
});
child.on("close", (code) => {
console.log(`[sandbox] archive hook exited ${code}: ${command}`);
console.log(hookOutcomeLine("archive", `exited ${code}`, command));
finish(
code === 0 && !timedOut
? { status: "ok", outputTail: output }
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { afterEach, beforeEach, describe, expect, test } from "bun:test";
import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
import { spawnSync } from "node:child_process";
import {
existsSync,
mkdirSync,
mkdtempSync,
readFileSync,
Expand Down Expand Up @@ -161,6 +162,62 @@ describe("start hook run terminal", () => {
expect(readFileSync(files.runTerminal, "utf8").trim()).toBe("term-1");
});

test("a multi-line setup command never reaches the log; its exit is stamped in the boot log", async () => {
const bootLog = join(root, "boot.log");
const previousBootLog = process.env.SUPERSET_SANDBOX_BOOT_LOG;
process.env.SUPERSET_SANDBOX_BOOT_LOG = bootLog;
const printed: unknown[] = [];
const spies = (["log", "warn", "error"] as const).map((method) =>
spyOn(console, method).mockImplementation((...args) => {
printed.push(...args);
}),
);
try {
const identity = identityWith(
'[setup.cloud]\nrun = """\necho LINE-ONE\necho LINE-TWO-SECRET\nexit 3\n"""\n',
);
const outcome = await runSandboxStartHook(ctx, identity, {
files,
starter: fakeStarter().starter,
});
expect(outcome).toMatchObject({ started: false, exitCode: 3 });
await until(() => readFileSync(bootLog, "utf8").includes("hook."));
} finally {
for (const spy of spies) spy.mockRestore();
if (previousBootLog === undefined)
delete process.env.SUPERSET_SANDBOX_BOOT_LOG;
else process.env.SUPERSET_SANDBOX_BOOT_LOG = previousBootLog;
}
expect(JSON.stringify(printed)).not.toMatch(/LINE-ONE|LINE-TWO-SECRET/);
expect(printed.join("\n")).toMatch(
/\[sandbox\] start hook failed \(exit 3\) \(command of \d+ chars\)/,
);
expect(readFileSync(bootLog, "utf8")).toMatch(
/^\d{13} hook\.failed exit=3 ms=\d+\n$/,
);
});

test("a setup that succeeds is stamped exited 0 in the boot log", async () => {
const bootLog = join(root, "boot.log");
const previousBootLog = process.env.SUPERSET_SANDBOX_BOOT_LOG;
process.env.SUPERSET_SANDBOX_BOOT_LOG = bootLog;
try {
const identity = identityWith('[setup.cloud]\nrun = "true"\n');
await runSandboxStartHook(ctx, identity, {
files,
starter: fakeStarter().starter,
});
await until(() => existsSync(bootLog));
} finally {
if (previousBootLog === undefined)
delete process.env.SUPERSET_SANDBOX_BOOT_LOG;
else process.env.SUPERSET_SANDBOX_BOOT_LOG = previousBootLog;
}
expect(readFileSync(bootLog, "utf8")).toMatch(
/^\d{13} hook\.exited exit=0 ms=\d+\n$/,
);
});

test("a forced re-run keeps the run terminal that is still live", async () => {
const identity = identityWith(
'[terminals.dev]\nrun = "bun dev"\ndefault = true\n',
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import { importCloudAttachments } from "../../trpc/router/attachments/attachment
import { createChatSessionInWorkspace } from "../../trpc/router/sessions/session-service";
import { ensurePresetRow } from "../../trpc/router/settings/agent-configs";
import type { HostServiceContext } from "../../types";
import { recordBootStamp } from "../boot-stamps";
import {
getManagedEnv,
waitForManagedEnv,
Expand All @@ -50,6 +51,7 @@ import {
withPendingLaunchReport,
} from "./launchMarker";
import { buildStartHookEnv } from "./utils/buildStartHookEnv";
import { hookOutcomeLine } from "./utils/hookOutcomeLine";
import { seedSandboxAgents } from "./utils/seedSandboxAgents";
import { terminalFallbackMode } from "./utils/terminalFallbackMode";

Expand Down Expand Up @@ -356,7 +358,11 @@ export async function runSandboxStartHook(
}
writeFileSync(files.marker, `${started.pid ?? ""}\n`);
console.log(
`[sandbox] start hook running in terminal ${started.terminalId}: ${plan.run.command}`,
hookOutcomeLine(
"start",
`running in terminal ${started.terminalId}`,
plan.run.command,
),
);
return {
started: true,
Expand All @@ -375,8 +381,15 @@ export async function runSandboxStartHook(
});
child.unref();
const pid = child.pid ?? 0;
startHookState = { state: "running", command, pid, since: Date.now() };
child.on("exit", (code) => {
const since = Date.now();
startHookState = { state: "running", command, pid, since };
child.on("exit", (code, signal) => {
const ms = Date.now() - since;
recordBootStamp(
code === 0
? `hook.exited exit=0 ms=${ms}`
: `hook.failed exit=${code ?? signal} ms=${ms}`,
);
startHookState = {
state: "exited",
command,
Expand All @@ -399,7 +412,9 @@ export async function runSandboxStartHook(
});
});
if (failure !== null && failure !== 0) {
console.error(`[sandbox] start hook failed (exit ${failure}): ${command}`);
console.error(
hookOutcomeLine("start", `failed (exit ${failure})`, command),
);
return {
started: false,
reason: "failed",
Expand All @@ -411,7 +426,7 @@ export async function runSandboxStartHook(
// Written only now: a hook that failed to start must be runnable again on
// this boot, and the marker is what makes the next call a no-op.
writeFileSync(files.marker, `${pid}\n`);
console.log(`[sandbox] start hook running (pid ${pid}): ${command}`);
console.log(hookOutcomeLine("start", `running (pid ${pid})`, command));
return { started: true, pid, command };
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
/** host-service.log leaves a cloud box, so a hook line names the outcome and never the repository's command. */
export function hookOutcomeLine(
hook: "start" | "archive",
outcome: string,
command: string,
): string {
return `[sandbox] ${hook} hook ${outcome} (command of ${command.length} chars)`;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
export { hookOutcomeLine } from "./hookOutcomeLine";
2 changes: 1 addition & 1 deletion packages/sandbox/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ The rule of thumb: if you can answer "where is it on the box", put it there.
| `bun run build` | Hashes `bundle/`, renders `contract.sh` from `@superset/shared/sandbox/contract`, derives the step versions, writes `dist/bundle/<sha>/` and the tarball. Deterministic: the same tree is the same sha on any machine. |
| `bun run build --publish` | Also uploads the bundle and any asset the bucket lacks, from the local cache or else its row's `source`, checked against the row's sha256. Refuses to publish a bundle that references an asset the bucket does not have. |
| `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. |
| boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs `apply-rootfs` and the steps marked `beforeHost` in `steps.json` (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to). Then, detached: `sync-assets` and the other steps, then the desktop, with the checkout beside them. `checkout.ready` waits for both, and the stamp `steps.end` says the steps are done. Once host-service answers, the environment has been pushed and the checkout is ready, the runner asks host-service to run the repo's `start` hook with that environment. |
| boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs `apply-rootfs` and the steps marked `beforeHost` in `steps.json` (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to). When `sandbox.conf` has `SUPERSET_SANDBOX_BOX_LOGS=1`, it also starts `superset-logship`, which sends the boot, memguard, desktop and host-service logs to Sentry Logs from a byte offset saved on disk. Then, detached: `sync-assets` and the other steps, then the desktop, with the checkout beside them. `checkout.ready` waits for both, and the stamp `steps.end` says the steps are done. Once host-service answers, the environment has been pushed and the checkout is ready, the runner asks host-service to run the repo's `start` hook with that environment. |
| `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex, Chrome, opencode and pi rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-<bundle sha>` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. |
| `bun run agent-tools --bundle <sha>` | Writes that bundle's agent CLIs to `agent-tools-<region>-<sha 12>`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. The drive carries claude, codex and pdev (also in the image) and opencode and pi (only on the drive). `bun run release` does this for every region, six at a time, before it records the bundle. It fails when a region with an environment has no drive, or a drive without claude, codex or pdev. Afterwards a `--production` release deletes the drives that neither an environment row (archived ones too) nor one of the 3 newest released bundles names; one Vercel project holds every database's drives, so no other release deletes any. |
| `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle and its agents drive in the default region, the same probe, a stop and a wake. |
Expand Down
28 changes: 20 additions & 8 deletions packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot
Original file line number Diff line number Diff line change
Expand Up @@ -237,30 +237,42 @@ if [ -x /usr/local/bin/superset-memguard ]; then
stamp "memguard.start pid=$!"
fi

# Sends this box's own logs to Sentry, only when the claim turned it on; see the script.
if [ "${SUPERSET_SANDBOX_BOX_LOGS:-}" = 1 ] && [ -n "${HOST_SERVICE_SENTRY_DSN:-}" ]; then
install -d -o "$SUPERSET_USER" -g "$SUPERSET_USER" "${SUPERSET_STATE_DIR}/logship"
env -u HOST_SERVICE_SECRET nice -n 10 setpriv --reuid="$SUPERSET_USER" --regid="$SUPERSET_USER" --init-groups \
/usr/local/bin/superset-logship >> "${SUPERSET_LOG_DIR}/logship.log" 2>&1 9>&- &
stamp "logship.start pid=$!"
fi

# The fixed swapfile of an earlier bundle; boxes have no swap now.
[ -f /swapfile ] && ! grep -q "^/swapfile " /proc/swaps && rm -f /swapfile

# The repository's start hook: the services a workspace needs every boot.
# host-service runs it, because the environment the hook needs is pushed
# into host-service after boot and never leaves it; this runner decides
# when: once host-service answers, the push has landed and the checkout is in.
# boot.log leaves the box, so it gets the outcome only: no command, no output.
start_hook() {
local api="http://127.0.0.1:${SUPERSET_HOST_SERVICE_PORT}/trpc/sandbox"
local auth="Authorization: Bearer ${HOST_SERVICE_SECRET}"
local i
local i out started reason pid
for i in $(seq 1 1200); do
[ -f "${SUPERSET_RUN_DIR}/host-service.ready" ] &&
curl -fs -H "$auth" "${api}.status" 2>/dev/null | grep -q '"environmentPushed":true' && break
sleep 0.1
done
if [ "$i" -ge 1200 ]; then stamp "hook.skipped no environment push within 120s"; return 0; fi
local out
out="$(curl -fs -X POST -H "$auth" -H 'Content-Type: application/json' -d '{}' "${api}.runStartHook" 2>&1)" ||
{ stamp "hook.failed ${out:0:160}"; return 0; }
case "$out" in
*'"started":true'*) stamp "hook.started ${out}" ;;
*'"no-hook"'*) stamp "hook.none" ;;
*) stamp "hook.skipped ${out:0:160}" ;;
# host-service stamps how a setup exited; this stamps only whether it started.
out="$(curl -fs -X POST -H "$auth" -H 'Content-Type: application/json' -d '{}' "${api}.runStartHook" 2>/dev/null |
jq -r '.result.data.json | [(.started | tostring), (.reason // ""), (.pid // "" | tostring)] | join("|")' 2>/dev/null)"
IFS='|' read -r started reason pid <<< "$out"
case "${started}:${reason}" in
true:*) stamp "hook.started pid=${pid}" ;;
false:no-hook) stamp "hook.none" ;;
false:failed) stamp "hook.start.failed" ;;
false:*) stamp "hook.skipped ${reason//[^a-z-]/}" ;;
*) stamp "hook.start.failed no answer from host-service" ;;
esac
}

Expand Down
Loading
Loading