diff --git a/docs/cloud-sandbox-considerations.md b/docs/cloud-sandbox-considerations.md index 170cb936a90..7c1364c4275 100644 --- a/docs/cloud-sandbox-considerations.md +++ b/docs/cloud-sandbox-considerations.md @@ -228,6 +228,19 @@ host-service's pressure report (`runtime/sandbox-box-pressure`), which cannot report a box whose host-service has stalled. Still open: automatic resume of an agent whose own process was reaped while its terminal survived. +## Box logs leave the box + +`superset-logship` sends `boot.log`, `memguard.log`, `desktop.log` and +`host-service.log` to the sandbox Sentry project when the claim sets +`SUPERSET_SANDBOX_BOX_LOGS=1`. That needs both PostHog flags: +`sandbox-box-logs-fleet` and the organization's `sandbox-box-logs`. The setup +script and the clone output are never read. **Open:** host-service lines can +still carry customer text: worktree paths, branch names, repository URLs, and +git or `gh` errors that print the command. The shipper scrubs token formats; +host-service no longer prints hook commands. Before the flag is on for a customer organization, +the policy wording (L1) must be live, and someone must decide which of those +lines to drop. + ## Untested behaviour These are unknowns, not known failures — but each could change the design, and diff --git a/packages/host-service/src/runtime/sandbox-self-seed/sandbox-archive-hook.test.ts b/packages/host-service/src/runtime/sandbox-self-seed/sandbox-archive-hook.test.ts index f08c5bef032..8dc7445d573 100644 --- a/packages/host-service/src/runtime/sandbox-self-seed/sandbox-archive-hook.test.ts +++ b/packages/host-service/src/runtime/sandbox-self-seed/sandbox-archive-hook.test.ts @@ -1,4 +1,4 @@ -import { afterEach, beforeEach, describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, spyOn } from "bun:test"; import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -60,6 +60,26 @@ describe("runSandboxArchiveHook", () => { expect(result.status === "ok" && result.outputTail).toBe("cloud\ndone\n"); }); + it("logs the outcome without a multi-line command", async () => { + writeConfig( + '[archive.cloud]\nrun = """\necho ARCHIVE-LINE-ONE\necho ARCHIVE-LINE-TWO\n"""\n', + ); + const printed: unknown[] = []; + const spy = spyOn(console, "log").mockImplementation((...args) => { + printed.push(...args); + }); + try { + expect((await runSandboxArchiveHook(identity())).status).toBe("ok"); + } finally { + spy.mockRestore(); + } + expect(printed).toEqual([ + expect.stringMatching( + /^\[sandbox\] archive hook exited 0 \(command of \d+ chars\)$/, + ), + ]); + }); + it("is skipped when there is nothing to run", async () => { writeConfig('[archive]\nrun = "echo x"\n[archive.cloud]\nrun = []\n'); expect(await runSandboxArchiveHook(identity())).toEqual({ diff --git a/packages/host-service/src/runtime/sandbox-self-seed/sandbox-archive-hook.ts b/packages/host-service/src/runtime/sandbox-self-seed/sandbox-archive-hook.ts index d53681a416b..35c610bcf34 100644 --- a/packages/host-service/src/runtime/sandbox-self-seed/sandbox-archive-hook.ts +++ b/packages/host-service/src/runtime/sandbox-self-seed/sandbox-archive-hook.ts @@ -6,6 +6,7 @@ import { getManagedEnv } from "../sandbox-managed-env/sandbox-managed-env.ts"; import { chainPhases, loadProjectConfig } from "../setup/config"; import type { SandboxIdentity } from "./sandbox-self-seed"; import { buildStartHookEnv } from "./utils/buildStartHookEnv"; +import { hookOutcomeLine } from "./utils/hookOutcomeLine"; /** The same limit as a local archive, so `[archive]` has one documented cap. */ export const ARCHIVE_HOOK_TIMEOUT_MS = TEARDOWN_TIMEOUT_MS; @@ -96,7 +97,7 @@ export async function runSandboxArchiveHook( }); }); child.on("close", (code) => { - console.log(`[sandbox] archive hook exited ${code}: ${command}`); + console.log(hookOutcomeLine("archive", `exited ${code}`, command)); finish( code === 0 && !timedOut ? { status: "ok", outputTail: output } diff --git a/packages/host-service/src/runtime/sandbox-self-seed/sandbox-self-seed.test.ts b/packages/host-service/src/runtime/sandbox-self-seed/sandbox-self-seed.test.ts index 9f730c8e3ab..b2a154accec 100644 --- a/packages/host-service/src/runtime/sandbox-self-seed/sandbox-self-seed.test.ts +++ b/packages/host-service/src/runtime/sandbox-self-seed/sandbox-self-seed.test.ts @@ -1,6 +1,7 @@ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { spawnSync } from "node:child_process"; import { + existsSync, mkdirSync, mkdtempSync, readFileSync, @@ -161,6 +162,62 @@ describe("start hook run terminal", () => { expect(readFileSync(files.runTerminal, "utf8").trim()).toBe("term-1"); }); + test("a multi-line setup command never reaches the log; its exit is stamped in the boot log", async () => { + const bootLog = join(root, "boot.log"); + const previousBootLog = process.env.SUPERSET_SANDBOX_BOOT_LOG; + process.env.SUPERSET_SANDBOX_BOOT_LOG = bootLog; + const printed: unknown[] = []; + const spies = (["log", "warn", "error"] as const).map((method) => + spyOn(console, method).mockImplementation((...args) => { + printed.push(...args); + }), + ); + try { + const identity = identityWith( + '[setup.cloud]\nrun = """\necho LINE-ONE\necho LINE-TWO-SECRET\nexit 3\n"""\n', + ); + const outcome = await runSandboxStartHook(ctx, identity, { + files, + starter: fakeStarter().starter, + }); + expect(outcome).toMatchObject({ started: false, exitCode: 3 }); + await until(() => readFileSync(bootLog, "utf8").includes("hook.")); + } finally { + for (const spy of spies) spy.mockRestore(); + if (previousBootLog === undefined) + delete process.env.SUPERSET_SANDBOX_BOOT_LOG; + else process.env.SUPERSET_SANDBOX_BOOT_LOG = previousBootLog; + } + expect(JSON.stringify(printed)).not.toMatch(/LINE-ONE|LINE-TWO-SECRET/); + expect(printed.join("\n")).toMatch( + /\[sandbox\] start hook failed \(exit 3\) \(command of \d+ chars\)/, + ); + expect(readFileSync(bootLog, "utf8")).toMatch( + /^\d{13} hook\.failed exit=3 ms=\d+\n$/, + ); + }); + + test("a setup that succeeds is stamped exited 0 in the boot log", async () => { + const bootLog = join(root, "boot.log"); + const previousBootLog = process.env.SUPERSET_SANDBOX_BOOT_LOG; + process.env.SUPERSET_SANDBOX_BOOT_LOG = bootLog; + try { + const identity = identityWith('[setup.cloud]\nrun = "true"\n'); + await runSandboxStartHook(ctx, identity, { + files, + starter: fakeStarter().starter, + }); + await until(() => existsSync(bootLog)); + } finally { + if (previousBootLog === undefined) + delete process.env.SUPERSET_SANDBOX_BOOT_LOG; + else process.env.SUPERSET_SANDBOX_BOOT_LOG = previousBootLog; + } + expect(readFileSync(bootLog, "utf8")).toMatch( + /^\d{13} hook\.exited exit=0 ms=\d+\n$/, + ); + }); + test("a forced re-run keeps the run terminal that is still live", async () => { const identity = identityWith( '[terminals.dev]\nrun = "bun dev"\ndefault = true\n', diff --git a/packages/host-service/src/runtime/sandbox-self-seed/sandbox-self-seed.ts b/packages/host-service/src/runtime/sandbox-self-seed/sandbox-self-seed.ts index 8dc96b497da..06acf7d7a02 100644 --- a/packages/host-service/src/runtime/sandbox-self-seed/sandbox-self-seed.ts +++ b/packages/host-service/src/runtime/sandbox-self-seed/sandbox-self-seed.ts @@ -32,6 +32,7 @@ import { importCloudAttachments } from "../../trpc/router/attachments/attachment import { createChatSessionInWorkspace } from "../../trpc/router/sessions/session-service"; import { ensurePresetRow } from "../../trpc/router/settings/agent-configs"; import type { HostServiceContext } from "../../types"; +import { recordBootStamp } from "../boot-stamps"; import { getManagedEnv, waitForManagedEnv, @@ -50,6 +51,7 @@ import { withPendingLaunchReport, } from "./launchMarker"; import { buildStartHookEnv } from "./utils/buildStartHookEnv"; +import { hookOutcomeLine } from "./utils/hookOutcomeLine"; import { seedSandboxAgents } from "./utils/seedSandboxAgents"; import { terminalFallbackMode } from "./utils/terminalFallbackMode"; @@ -356,7 +358,11 @@ export async function runSandboxStartHook( } writeFileSync(files.marker, `${started.pid ?? ""}\n`); console.log( - `[sandbox] start hook running in terminal ${started.terminalId}: ${plan.run.command}`, + hookOutcomeLine( + "start", + `running in terminal ${started.terminalId}`, + plan.run.command, + ), ); return { started: true, @@ -375,8 +381,15 @@ export async function runSandboxStartHook( }); child.unref(); const pid = child.pid ?? 0; - startHookState = { state: "running", command, pid, since: Date.now() }; - child.on("exit", (code) => { + const since = Date.now(); + startHookState = { state: "running", command, pid, since }; + child.on("exit", (code, signal) => { + const ms = Date.now() - since; + recordBootStamp( + code === 0 + ? `hook.exited exit=0 ms=${ms}` + : `hook.failed exit=${code ?? signal} ms=${ms}`, + ); startHookState = { state: "exited", command, @@ -399,7 +412,9 @@ export async function runSandboxStartHook( }); }); if (failure !== null && failure !== 0) { - console.error(`[sandbox] start hook failed (exit ${failure}): ${command}`); + console.error( + hookOutcomeLine("start", `failed (exit ${failure})`, command), + ); return { started: false, reason: "failed", @@ -411,7 +426,7 @@ export async function runSandboxStartHook( // Written only now: a hook that failed to start must be runnable again on // this boot, and the marker is what makes the next call a no-op. writeFileSync(files.marker, `${pid}\n`); - console.log(`[sandbox] start hook running (pid ${pid}): ${command}`); + console.log(hookOutcomeLine("start", `running (pid ${pid})`, command)); return { started: true, pid, command }; } diff --git a/packages/host-service/src/runtime/sandbox-self-seed/utils/hookOutcomeLine/hookOutcomeLine.ts b/packages/host-service/src/runtime/sandbox-self-seed/utils/hookOutcomeLine/hookOutcomeLine.ts new file mode 100644 index 00000000000..034f3b2779e --- /dev/null +++ b/packages/host-service/src/runtime/sandbox-self-seed/utils/hookOutcomeLine/hookOutcomeLine.ts @@ -0,0 +1,8 @@ +/** host-service.log leaves a cloud box, so a hook line names the outcome and never the repository's command. */ +export function hookOutcomeLine( + hook: "start" | "archive", + outcome: string, + command: string, +): string { + return `[sandbox] ${hook} hook ${outcome} (command of ${command.length} chars)`; +} diff --git a/packages/host-service/src/runtime/sandbox-self-seed/utils/hookOutcomeLine/index.ts b/packages/host-service/src/runtime/sandbox-self-seed/utils/hookOutcomeLine/index.ts new file mode 100644 index 00000000000..beed4d16fe0 --- /dev/null +++ b/packages/host-service/src/runtime/sandbox-self-seed/utils/hookOutcomeLine/index.ts @@ -0,0 +1 @@ +export { hookOutcomeLine } from "./hookOutcomeLine"; diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index 664f3bd39a7..7785deac5fb 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -53,7 +53,7 @@ The rule of thumb: if you can answer "where is it on the box", put it there. | `bun run build` | Hashes `bundle/`, renders `contract.sh` from `@superset/shared/sandbox/contract`, derives the step versions, writes `dist/bundle//` and the tarball. Deterministic: the same tree is the same sha on any machine. | | `bun run build --publish` | Also uploads the bundle and any asset the bucket lacks, from the local cache or else its row's `source`, checked against the row's sha256. Refuses to publish a bundle that references an asset the bucket does not have. | | `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. | -| boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs `apply-rootfs` and the steps marked `beforeHost` in `steps.json` (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to). Then, detached: `sync-assets` and the other steps, then the desktop, with the checkout beside them. `checkout.ready` waits for both, and the stamp `steps.end` says the steps are done. Once host-service answers, the environment has been pushed and the checkout is ready, the runner asks host-service to run the repo's `start` hook with that environment. | +| boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs `apply-rootfs` and the steps marked `beforeHost` in `steps.json` (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to). When `sandbox.conf` has `SUPERSET_SANDBOX_BOX_LOGS=1`, it also starts `superset-logship`, which sends the boot, memguard, desktop and host-service logs to Sentry Logs from a byte offset saved on disk. Then, detached: `sync-assets` and the other steps, then the desktop, with the checkout beside them. `checkout.ready` waits for both, and the stamp `steps.end` says the steps are done. Once host-service answers, the environment has been pushed and the checkout is ready, the runner asks host-service to run the repo's `start` hook with that environment. | | `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex, Chrome, opencode and pi rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | | `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. The drive carries claude, codex and pdev (also in the image) and opencode and pi (only on the drive). `bun run release` does this for every region, six at a time, before it records the bundle. It fails when a region with an environment has no drive, or a drive without claude, codex or pdev. Afterwards a `--production` release deletes the drives that neither an environment row (archived ones too) nor one of the 3 newest released bundles names; one Vercel project holds every database's drives, so no other release deletes any. | | `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle and its agents drive in the default region, the same probe, a stop and a wake. | diff --git a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot index 4f8e743efa3..3077db2b102 100755 --- a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot +++ b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot @@ -237,6 +237,14 @@ if [ -x /usr/local/bin/superset-memguard ]; then stamp "memguard.start pid=$!" fi +# Sends this box's own logs to Sentry, only when the claim turned it on; see the script. +if [ "${SUPERSET_SANDBOX_BOX_LOGS:-}" = 1 ] && [ -n "${HOST_SERVICE_SENTRY_DSN:-}" ]; then + install -d -o "$SUPERSET_USER" -g "$SUPERSET_USER" "${SUPERSET_STATE_DIR}/logship" + env -u HOST_SERVICE_SECRET nice -n 10 setpriv --reuid="$SUPERSET_USER" --regid="$SUPERSET_USER" --init-groups \ + /usr/local/bin/superset-logship >> "${SUPERSET_LOG_DIR}/logship.log" 2>&1 9>&- & + stamp "logship.start pid=$!" +fi + # The fixed swapfile of an earlier bundle; boxes have no swap now. [ -f /swapfile ] && ! grep -q "^/swapfile " /proc/swaps && rm -f /swapfile @@ -244,23 +252,27 @@ fi # host-service runs it, because the environment the hook needs is pushed # into host-service after boot and never leaves it; this runner decides # when: once host-service answers, the push has landed and the checkout is in. +# boot.log leaves the box, so it gets the outcome only: no command, no output. start_hook() { local api="http://127.0.0.1:${SUPERSET_HOST_SERVICE_PORT}/trpc/sandbox" local auth="Authorization: Bearer ${HOST_SERVICE_SECRET}" - local i + local i out started reason pid for i in $(seq 1 1200); do [ -f "${SUPERSET_RUN_DIR}/host-service.ready" ] && curl -fs -H "$auth" "${api}.status" 2>/dev/null | grep -q '"environmentPushed":true' && break sleep 0.1 done if [ "$i" -ge 1200 ]; then stamp "hook.skipped no environment push within 120s"; return 0; fi - local out - out="$(curl -fs -X POST -H "$auth" -H 'Content-Type: application/json' -d '{}' "${api}.runStartHook" 2>&1)" || - { stamp "hook.failed ${out:0:160}"; return 0; } - case "$out" in - *'"started":true'*) stamp "hook.started ${out}" ;; - *'"no-hook"'*) stamp "hook.none" ;; - *) stamp "hook.skipped ${out:0:160}" ;; + # host-service stamps how a setup exited; this stamps only whether it started. + out="$(curl -fs -X POST -H "$auth" -H 'Content-Type: application/json' -d '{}' "${api}.runStartHook" 2>/dev/null | + jq -r '.result.data.json | [(.started | tostring), (.reason // ""), (.pid // "" | tostring)] | join("|")' 2>/dev/null)" + IFS='|' read -r started reason pid <<< "$out" + case "${started}:${reason}" in + true:*) stamp "hook.started pid=${pid}" ;; + false:no-hook) stamp "hook.none" ;; + false:failed) stamp "hook.start.failed" ;; + false:*) stamp "hook.skipped ${reason//[^a-z-]/}" ;; + *) stamp "hook.start.failed no answer from host-service" ;; esac } diff --git a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-logship b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-logship new file mode 100755 index 00000000000..5be7ba3b74d --- /dev/null +++ b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-logship @@ -0,0 +1,271 @@ +#!/usr/bin/env bash +# Sends the platform's own box logs to Sentry Logs. Started by superset-boot as +# the sandbox user beside memguard, only when sandbox.conf has +# SUPERSET_SANDBOX_BOX_LOGS=1; it does not need host-service. Each pass, every few seconds: +# +# read boot, memguard, desktop and host-service logs from the byte offset +# saved for each under the state directory. start-hook.log and +# checkout.log are the customer's output and are never read. Offsets +# are trusted only if this script also ran in the previous boot; else +# each file starts at its first line of the current boot. +# filter drop host-service vitals and memguard rank lines; scrub token +# formats; collapse a run of identical lines read in one pass into one +# record with a count. A file that is not a regular file is never read. +# cap a budget of serialized record bytes per box per clock hour; one record +# says it was reached, and nothing else is sent that hour. +# send one envelope per file per pass. The offset moves only on a 2xx, so a +# box that sleeps first sends those lines at its next wake. A refusal +# waits (Retry-After, else doubling up to an hour) and keeps nothing in +# memory: the log files are the only queue. +# +# --once runs one pass and exits. It prints one line per refusal, never per line of log. +set -uo pipefail +. "${SUPERSET_CONTRACT:-/etc/superset/contract.sh}" + +[ "${SUPERSET_SANDBOX_BOX_LOGS:-}" = 1 ] || exit 0 +DSN="${HOST_SERVICE_SENTRY_DSN:-}" +if ! [[ "$DSN" =~ ^(https?)://([^@/:]+)(:[^@/]*)?@([^/]+)(/.*)?/([0-9]+)/?$ ]]; then + echo "logship: no usable DSN; not sending"; exit 0 +fi +ENVELOPE_URL="${BASH_REMATCH[1]}://${BASH_REMATCH[4]}${BASH_REMATCH[5]}/api/${BASH_REMATCH[6]}/envelope/" +AUTH="X-Sentry-Auth: Sentry sentry_version=7, sentry_key=${BASH_REMATCH[2]}, sentry_client=superset-logship/1" + +SOURCES=(boot memguard desktop host-service) +STATE="${SUPERSET_STATE_DIR}/logship" +CAP="${SUPERSET_LOGSHIP_CAP_BYTES:-262144}" +INTERVAL_S="${SUPERSET_LOGSHIP_INTERVAL_S:-5}" +MAX_READ=65536 +MAX_RECORDS=100 +MAX_BODY=8192 +MAX_WAIT_S=3600 + +mkdir -p "$STATE" +exec 7> "${STATE}/lock" +flock -n 7 || exit 0 + +now_ms() { echo "${SUPERSET_LOGSHIP_NOW_MS:-$(date +%s%3N)}"; } +TRACE_ID="$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')" + +read_state() { local v; v="$(cat "${STATE}/$1" 2>/dev/null)" && echo "$v" || echo "$2"; } +write_state() { printf '%s\n' "$2" > "${STATE}/$1.tmp" && mv -f "${STATE}/$1.tmp" "${STATE}/$1"; } + +# `R ` for each record and `D ` for each dropped +# line, whose time a following line without one inherits; then `E ` for +# how far the file is read. A trailing line with no newline yet waits a pass. +read -r -d '' SPLIT <<'AWK' +function handle(line, len, key, i, n, d) { + if (stop) return + key = line + sub(/^[0-9]+ /, "", key) + sub(/^[0-9][0-9-]*T[0-9:.]*Z /, "", key) + if ((src == "host-service" && index(key, "[host-service:vitals]") == 1) || + (src == "memguard" && index(key, "memguard rank ") == 1)) { + if (!have) { printf "D\t%s\n", substr(line, 1, 30); pos += len; return } + dropped = dropped substr(line, 1, 30) "\n"; pos += len; return + } + if (have && key == prevkey) { count++; dropped = dropped substr(line, 1, 30) "\n"; pos += len; return } + if (emitted + have >= maxrec) { flush(); stop = 1; return } + flush() + have = 1; prevkey = key; prevline = line; prevoff = pos; count = 1; pos += len +} +function flush() { + if (!have) return + printf "R\t%d\t%d\t%s\n", prevoff, count, substr(prevline, 1, maxbody) + emitted++; have = 0 + if (dropped != "") { n = split(dropped, d, "\n"); for (i = 1; i < n; i++) printf "D\t%s\n", d[i]; dropped = "" } +} +{ if (NR > 1) handle(buf, buflen); buf = $0; buflen = length($0) + 1 } +END { + if (NR >= 1) { + if (!partial) handle(buf, buflen) + else if (NR == 1 && full) handle(buf, length(buf)) + } + flush() + printf "E\t%d\n", pos +} +AWK + +# The byte of the first line stamped at or after the boot start (`ms`, and `iso` +# for ISO stamps, which compare as strings). +read -r -d '' FIRST <<'AWK' +$0 ~ /^[0-9]+ / && $1 + 0 >= ms { found = 1; exit } +$0 ~ /^[0-9][0-9-]*T[0-9:.]*Z / && substr($0, 1, 24) >= iso { found = 1; exit } +{ pos += length($0) + 1 } +END { pos += 0; if (pos > size) pos = size; print pos } +AWK + +read -r -d '' SCRUB <<'SED' +s/gh[pousr]_[A-Za-z0-9]{20,}/[redacted]/g +s/github_pat_[A-Za-z0-9_]{20,}/[redacted]/g +s/glpat-[A-Za-z0-9_-]{20,}/[redacted]/g +s/npm_[A-Za-z0-9]{30,}/[redacted]/g +s/sk-ant-[A-Za-z0-9_-]{10,}/[redacted]/g +s/sk-(proj-)?[A-Za-z0-9_-]{20,}/[redacted]/g +s/[rs]k_(live|test)_[A-Za-z0-9]{10,}/[redacted]/g +s/xox[abposr]-[A-Za-z0-9-]{10,}/[redacted]/g +s/(AKIA|ASIA)[A-Z0-9]{16}/[redacted]/g +s/eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/[redacted]/g +s#(://)[^/@[:space:]:]+:[^/@[:space:]]+@#\1[redacted]@#g +s#([Bb]earer|[Bb]asic|[Tt]oken) [A-Za-z0-9._~+/=-]{8,}#\1 [redacted]#g +s/([A-Za-z0-9_-]*(token|secret|password|passwd|api[_-]?key|authorization|credential)[A-Za-z0-9_-]*["']?[[:space:]]*[=:][[:space:]]*["']?)[^[:space:]"'&,;}]{4,}/\1[redacted]/Ig +SED + +# Records within the hour's budget as a Sentry log item, after one metadata line: +# bytes used, whether the cap is now reached, the offset read to, records kept, +# and the time of the last line read. Times are epoch milliseconds. +read -r -d '' BUILD <<'JQ' +def stamp($l): + if ($l | test("^[0-9]{13} ")) then ($l[0:13] | tonumber) + elif ($l | test("^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\\.[0-9]+)?Z ")) then + (($l[0:19] + "Z") | fromdateiso8601) * 1000 + + ((($l | capture("^.{19}(?\\.[0-9]+)?").f // "0") | tonumber) * 1000 + 0.5 | floor) + else null end; +def str($v): {value: $v, type: "string"}; +def int($v): {value: $v, type: "integer"}; +def common: { + "log.source": str($src), + "cloud_workspace.id": str($ws), + "organization.id": str($org) +}; +def record($r; $t): + ([$starts[] | select(. <= $t)] | max) as $boot + | { + timestamp: ($t / 1000), trace_id: $trace, level: "info", body: $r.b, + attributes: (common + + {"log.file.offset": int($r.o | tonumber)} + + (if $boot == null then {} else {"box.boot_started_at": int($boot)} end) + + (if $r.c == "1" then {} else {"log.repeat_count": int($r.c | tonumber)} end)) + }; +[inputs] as $lines +| ($lines | map(select(startswith("E\t")))[0][2:] | tonumber) as $read_to +| reduce ($lines[] | select(startswith("R\t") or startswith("D\t"))) as $l + ({items: [], used: $used, over: ($capped == "1"), prev: $prev}; + if ($l | startswith("D\t")) then .prev = (stamp($l[2:]) // .prev) + else + ($l | capture("^R\t(?[0-9]+)\t(?[0-9]+)\t(?.*)$")) as $r + | (stamp($r.b) // .prev // $now) as $t + | .prev = $t + | record($r; $t) as $rec + | ($rec | tojson | utf8bytelength) as $n + | if .over or (.used + $n > $cap) then .over = true + else .items += [$rec] | .used += $n end + end) +| if .over and ($capped == "0") then + { + timestamp: ($now / 1000), trace_id: $trace, level: "info", + body: "log shipper: cap of \($cap) bytes this hour reached; nothing more is sent until the next hour", + attributes: common + } as $note + | .items += [$note] | .used += ($note | tojson | utf8bytelength) + else . end +| .capped = (if .over then "1" else $capped end) +| ([.used, .capped, $read_to, (.items | length), (.prev // "")] | @tsv), ({items: .items} | tojson) +JQ + +# 0 sent or nothing to send, 1 refused (the pass stops and waits). +send() { + local body="$1" count="$2" out code wait fails + out="$(printf '%s\n%s\n%s\n' "{\"dsn\":\"${DSN}\"}" \ + "{\"type\":\"log\",\"item_count\":${count},\"content_type\":\"application/vnd.sentry.items.log+json\"}" "$body" | + curl -s -o /dev/null -D - -w '%{http_code}' --max-time 15 \ + -H "$AUTH" -H 'Content-Type: application/x-sentry-envelope' --data-binary @- "$ENVELOPE_URL" | tr -d '\r')" + code="${out##*$'\n'}" + case "$code" in + 2??) rm -f "${STATE}/retry"; return 0 ;; + # A batch Sentry cannot parse would be refused forever; it is dropped. + 400 | 413) echo "$(now_ms) logship dropped a batch: ${code}"; return 0 ;; + esac + fails="$(read_state retry "0 0" | cut -d' ' -f2)" + fails=$((fails + 1)) + wait="$(awk -F': *' 'tolower($1) == "retry-after" && $2 ~ /^[0-9]+$/ { print $2; exit }' <<< "$out")" + if [ -z "$wait" ]; then + case "$code" in + 401 | 403) wait=$MAX_WAIT_S ;; + *) wait=$((INTERVAL_S * (1 << (fails < 10 ? fails : 10)))) ;; + esac + fi + [ "$wait" -gt "$MAX_WAIT_S" ] && wait=$MAX_WAIT_S + write_state retry "$(( $(now_ms) + wait * 1000 )) ${fails}" + echo "$(now_ms) logship refused: ${code:-none}; waiting ${wait}s" + return 1 +} + +ship_file() { + local src="$1" file="${SUPERSET_LOG_DIR}/${src}.log" size off partial=0 full=0 used capped read_to count last payload + # The box's user owns the log directory and could link a log to any file it can read. + [ -f "$file" ] && [ ! -L "$file" ] || return 0 + size="$(stat -c %s "$file")" + off="$(read_state "${src}.offset" "")" + # Turning the shipper on sends no history: each file starts at the current boot. + if [ -z "$off" ]; then + off="$size" + [ -n "$BOOT_MS" ] && off="$(LC_ALL=C awk -v ms="$BOOT_MS" -v iso="$BOOT_ISO" -v size="$size" "$FIRST" "$file")" + fi + [ "$size" -lt "$off" ] && off=0 + [ "$size" -eq "$off" ] && { write_state "${src}.offset" "$off"; return 0; } + tail -c +"$((off + 1))" "$file" | head -c "$MAX_READ" > "${STATE}/chunk" + [ -n "$(tail -c 1 "${STATE}/chunk")" ] && partial=1 + [ "$(stat -c %s "${STATE}/chunk")" -ge "$MAX_READ" ] && full=1 + { IFS=$'\t' read -r used capped read_to count last; read -r payload; } < <( + LC_ALL=C awk -v src="$src" -v pos="$off" -v partial="$partial" -v full="$full" \ + -v maxrec="$MAX_RECORDS" -v maxbody="$MAX_BODY" "$SPLIT" "${STATE}/chunk" | + LC_ALL=C sed -E "$SCRUB" | + jq -R -n -r --arg src "$src" --arg ws "${SUPERSET_SANDBOX_WORKSPACE_ID:-}" \ + --arg org "${SUPERSET_SANDBOX_ORGANIZATION_ID:-}" --argjson prev "$(read_state "${src}.time" null)" \ + --arg trace "$TRACE_ID" --argjson now "$NOW" --argjson starts "$BOOT_STARTS" \ + --argjson cap "$CAP" --argjson used "$USED" --arg capped "$CAPPED" "$BUILD" + ) + [ -n "${read_to:-}" ] || return 0 + if [ "$count" -gt 0 ]; then + send "$payload" "$count" || return 1 + fi + USED="$used"; CAPPED="$capped" + write_state hour "${HOUR} ${USED} ${CAPPED}" + [ -n "${last:-}" ] && write_state "${src}.time" "$last" + write_state "${src}.offset" "$read_to" +} + +pass() { + NOW="$(now_ms)" + local retry_at h + retry_at="$(read_state retry "0 0" | cut -d' ' -f1)" + [ "$NOW" -lt "$retry_at" ] && return 0 + HOUR=$((NOW / 3600000)) + read -r h USED CAPPED < <(read_state hour "0 0 0") + [ "$h" = "$HOUR" ] || { USED=0; CAPPED=0; } + local src size off idle=1 + for src in "${SOURCES[@]}"; do + [ -L "${SUPERSET_LOG_DIR}/${src}.log" ] && continue + size="$(stat -c %s "${SUPERSET_LOG_DIR}/${src}.log" 2>/dev/null)" || continue + off="$(read_state "${src}.offset" "")" + [ "$size" = "$off" ] || { idle=0; break; } + done + [ "$idle" = 1 ] && return 0 + read_boots + for src in "${SOURCES[@]}"; do + ship_file "$src" || break + done + rm -f "${STATE}/chunk" +} + +read_boots() { + BOOT_STARTS="$(grep -oE '^[0-9]+ boot\.start' "$SUPERSET_BOOT_LOG" 2>/dev/null | cut -d' ' -f1 | jq -sc .)" + BOOT_MS="$(jq -r '.[-1] // ""' <<< "$BOOT_STARTS")" + BOOT_ISO="" + [ -n "$BOOT_MS" ] && BOOT_ISO="$(date -u -d "@$((BOOT_MS / 1000))" +%Y-%m-%dT%H:%M:%S).$(printf '%03d' $((BOOT_MS % 1000)))Z" +} + +# Offsets saved in a boot this script did not run in (the flag was off, or an older +# runner left the state behind) would send what that boot wrote; start again. +read_boots +if [ "$(read_state boot "")" != "${BOOT_MS:-}" ] && + [ "$(read_state boot "")" != "$(jq -r '.[-2] // "none"' <<< "$BOOT_STARTS")" ]; then + rm -f "${STATE}"/*.offset "${STATE}"/*.time +fi +write_state boot "${BOOT_MS:-}" + +[ "${1:-}" = --once ] && { pass; exit 0; } +while true; do + pass + sleep "$INTERVAL_S" +done diff --git a/packages/sandbox/src/boot-twice.ts b/packages/sandbox/src/boot-twice.ts index 945d72279fa..64b37c49a05 100644 --- a/packages/sandbox/src/boot-twice.ts +++ b/packages/sandbox/src/boot-twice.ts @@ -321,6 +321,17 @@ try { ).trim(), ); + // The stub identity has no SUPERSET_SANDBOX_BOX_LOGS, like every box whose flag is off. + const shipper = exec( + `echo $(grep -c logship ${SANDBOX_PATHS.bootLog}) $(pgrep -fc '[s]uperset-logship') $(ls ${SANDBOX_PATHS.state}/logship 2>/dev/null | wc -l)`, + { check: false }, + ).trim(); + expect( + "box logs off: no shipper stamp, process or state", + shipper === "0 0 0", + shipper, + ); + // Second boot: stop everything the way a wake would (no processes survive), then boot again. exec( `pkill -u ubuntu -TERM || true; sleep 1; pkill -u ubuntu -KILL || true; pkill -x Xvnc || true; rm -rf ${SANDBOX_PATHS.run}`, diff --git a/packages/sandbox/src/logship.test.ts b/packages/sandbox/src/logship.test.ts new file mode 100644 index 00000000000..23672b2381f --- /dev/null +++ b/packages/sandbox/src/logship.test.ts @@ -0,0 +1,482 @@ +import { afterAll, beforeEach, describe, expect, test } from "bun:test"; +import { + appendFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const BIN = join(import.meta.dir, "../bundle/rootfs/usr/local/bin"); +const SHIPPER = join(BIN, "superset-logship"); +const boot = readFileSync(join(BIN, "superset-boot"), "utf8"); + +type Item = { + timestamp: number; + trace_id: string; + level: string; + body: string; + attributes: Record; +}; +type Received = { auth: string | null; item: string; items: Item[] }; + +let status = 200; +let headers: Record = {}; +let received: Received[] = []; +const sentry = Bun.serve({ + port: 0, + async fetch(req) { + const [, item = "", payload = "{}"] = (await req.text()).split("\n"); + received.push({ + auth: req.headers.get("x-sentry-auth"), + item, + items: JSON.parse(payload).items ?? [], + }); + return new Response("", { status, headers }); + }, +}); +afterAll(() => sentry.stop(true)); + +const HOUR_MS = 3_600_000; +const NOW = 1_794_960_000_000; +/** 2026-11-17T21:13:20.000Z, the first boot of every test. */ +const BOOT1 = 1_794_950_000_000; +const iso = (ms: number) => new Date(ms).toISOString(); +const WS = "11111111-1111-4111-8111-111111111111"; +const ORG = "22222222-2222-4222-8222-222222222222"; + +let root = ""; +const logs = () => join(root, "log"); +const state = () => join(root, "state", "logship"); +const write = (name: string, text: string) => + writeFileSync(join(logs(), name), text); +const append = (name: string, text: string) => + appendFileSync(join(logs(), name), text); +const bootStart = (ms: number) => + append("boot.log", `${ms} boot.start pid=1\n`); + +beforeEach(() => { + if (root) rmSync(root, { recursive: true, force: true }); + root = mkdtempSync(join(tmpdir(), "logship-")); + mkdirSync(logs()); + writeFileSync( + join(root, "contract.sh"), + `SUPERSET_LOG_DIR='${logs()}'\nSUPERSET_STATE_DIR='${join(root, "state")}'\nSUPERSET_BOOT_LOG='${logs()}/boot.log'\n`, + ); + status = 200; + headers = {}; + received = []; +}); + +async function ship( + env: Record = {}, + args: string[] = ["--once"], + stopWhen?: () => boolean, +): Promise { + const proc = Bun.spawn(["bash", SHIPPER, ...args], { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + SUPERSET_CONTRACT: join(root, "contract.sh"), + SUPERSET_SANDBOX_BOX_LOGS: "1", + HOST_SERVICE_SENTRY_DSN: `http://publickey@127.0.0.1:${sentry.port}/42`, + SUPERSET_SANDBOX_WORKSPACE_ID: WS, + SUPERSET_SANDBOX_ORGANIZATION_ID: ORG, + SUPERSET_LOGSHIP_NOW_MS: String(NOW), + ...env, + }, + stdout: "pipe", + stderr: "pipe", + }); + if (stopWhen) { + while (!stopWhen() && proc.exitCode === null) await Bun.sleep(5); + // Ten more loop intervals: a pass that ignored the wait would send again. + await Bun.sleep(200); + proc.kill("SIGTERM"); + } + await proc.exited; + const stderr = await new Response(proc.stderr).text(); + if (stderr) throw new Error(`superset-logship wrote to stderr: ${stderr}`); + return await new Response(proc.stdout).text(); +} + +const items = () => received.flatMap((r) => r.items); +const bodies = () => items().map((i) => i.body); +const attr = (i: Item | undefined, key: string) => i?.attributes[key]?.value; +const bySource = (source: string) => + items().filter((i) => attr(i, "log.source") === source); +const isCap = (i: Item) => i.body.startsWith("log shipper: cap of"); + +describe("superset-logship", () => { + test("sends each line with exactly the record contract's attributes", async () => { + bootStart(BOOT1); + append("boot.log", `${BOOT1 + 123} host.ready\n`); + write( + "host-service.log", + `${iso(BOOT1 + 5_000).replace("000Z", "822Z")} [host-service] boom\n at stack (file.js:1)\n`, + ); + await ship(); + + expect(received[0]?.auth).toContain("sentry_key=publickey"); + expect(JSON.parse(received[0]?.item ?? "{}")).toMatchObject({ + type: "log", + content_type: "application/vnd.sentry.items.log+json", + }); + const [start, ready] = bySource("boot"); + expect(start).toMatchObject({ + level: "info", + timestamp: BOOT1 / 1000, + body: `${BOOT1} boot.start pid=1`, + }); + expect(start?.trace_id).toMatch(/^[0-9a-f]{32}$/); + expect(start?.attributes).toEqual({ + "log.source": { value: "boot", type: "string" }, + "cloud_workspace.id": { value: WS, type: "string" }, + "organization.id": { value: ORG, type: "string" }, + "log.file.offset": { value: 0, type: "integer" }, + "box.boot_started_at": { value: BOOT1, type: "integer" }, + }); + expect(attr(ready, "log.file.offset")).toBe(31); + expect(ready?.timestamp).toBe((BOOT1 + 123) / 1000); + + const [line, stack] = bySource("host-service"); + expect(line?.timestamp).toBe((BOOT1 + 5_822) / 1000); + expect(attr(line, "box.boot_started_at")).toBe(BOOT1); + // No time of its own: the line before it gives both its time and boot. + expect(stack?.timestamp).toBe(line?.timestamp as number); + expect(attr(stack, "box.boot_started_at")).toBe(BOOT1); + }); + + test("never reads the setup script's or the clone's output", async () => { + bootStart(BOOT1); + write("start-hook.log", `${BOOT1 + 1} CUSTOMER-SETUP-OUTPUT\n`); + write("checkout.log", `${BOOT1 + 1} CUSTOMER-CLONE-OUTPUT\n`); + await ship(); + expect(JSON.stringify(received)).not.toContain("CUSTOMER"); + expect(readdirSync(state())).not.toContain("start-hook.offset"); + }); + + test("drops vitals and memguard rank lines and keeps kills", async () => { + bootStart(BOOT1); + const t = (s: number) => iso(BOOT1 + s * 1000); + write( + "host-service.log", + `${t(1)} [host-service:vitals] {"rss":120}\n${t(2)} [host-service] listening\n`, + ); + write( + "memguard.log", + `${BOOT1 + 10} memguard rank pid=9 node 0->500\n${BOOT1 + 500} memguard reap pid=9 node rss=900MB adj=500\n`, + ); + await ship(); + expect(bodies()).toEqual([ + `${BOOT1} boot.start pid=1`, + `${BOOT1 + 500} memguard reap pid=9 node rss=900MB adj=500`, + `${t(2)} [host-service] listening`, + ]); + }); + + test("reads only regular files, never a link the box's user put in the log directory", async () => { + bootStart(BOOT1); + const secret = join(root, "secret.txt"); + writeFileSync(secret, `${BOOT1 + 1} desktop SECRET-FILE\n`); + symlinkSync(secret, join(logs(), "desktop.log")); + await ship(); + expect(JSON.stringify(received)).not.toContain("SECRET-FILE"); + expect(existsSync(join(state(), "desktop.offset"))).toBe(false); + }); + + test("scrubs token formats before a line leaves", async () => { + bootStart(BOOT1); + const secrets = [ + "ghp_abcdefghijklmnopqrstuvwxyz0123456789", + "ghs_ABCDEFGHIJKLMNOPQRSTUVWXYZ012345", + "github_pat_11ABCDEFG0123456789_abcdefghijklmnopqrstuvwxyz", + "sk-ant-api03-abcdefghijklmnop_qrstuv", + "sk-proj-abcdefghijklmnopqrstuvwxyz012345", + "xoxb-123456789012-abcdefghijkl", + "AKIAABCDEFGHIJKLMNOP", + "sk_live_abcdefghijklmnop", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N", + "npm_abcdefghijklmnopqrstuvwxyz0123456789", + ]; + const t = (s: number) => iso(BOOT1 + s * 1000); + write( + "host-service.log", + [ + ...secrets.map((s, n) => `${t(n)} value ${s} end`), + `${t(20)} clone https://x-access-token:opaque-token-value@github.com/acme/api.git`, + `${t(21)} Authorization: Bearer abc.def-ghi_jkl123`, + `${t(22)} {"api_key":"plain-value-1234","HOST_SERVICE_SECRET=s3cr3tvalue"}`, + `${t(23)} password=hunter2hunter2 next`, + "", + ].join("\n"), + ); + await ship(); + const sent = bodies().join("\n"); + for (const secret of [ + ...secrets, + "opaque-token-value", + "abc.def-ghi_jkl123", + "plain-value-1234", + "s3cr3tvalue", + "hunter2", + ]) { + expect(sent).not.toContain(secret); + } + expect(sent).toContain("https://[redacted]@github.com/acme/api.git"); + expect(sent).toContain("Authorization: [redacted]"); + expect(bySource("host-service")).toHaveLength(secrets.length + 4); + }); + + test("collapses a run of identical lines read in one pass into one record with a count, across dropped lines", async () => { + bootStart(BOOT1); + write( + "desktop.log", + `${BOOT1 + 1} desktop websockify retry\n${BOOT1 + 2} desktop websockify retry\n${BOOT1 + 3} desktop websockify retry\n${BOOT1 + 4} desktop X ready\n`, + ); + const t = (s: number) => iso(BOOT1 + s * 1000); + write( + "host-service.log", + `${t(1)} sweep done\n${t(2)} [host-service:vitals] {}\n${t(3)} sweep done\n`, + ); + await ship(); + const [retry, ready] = bySource("desktop"); + expect(bySource("desktop")).toHaveLength(2); + expect(retry?.body).toBe(`${BOOT1 + 1} desktop websockify retry`); + expect(attr(retry, "log.repeat_count")).toBe(3); + expect(attr(ready, "log.repeat_count")).toBeUndefined(); + expect(attr(ready, "log.file.offset")).toBe(117); + expect( + bySource("host-service").map((i) => attr(i, "log.repeat_count")), + ).toEqual([2]); + expect( + readFileSync(join(state(), "host-service.time"), "utf8").trim(), + ).toBe(String(BOOT1 + 3000)); + }); + + test("the hour's cap counts each record's serialized size, with one cap record", async () => { + bootStart(BOOT1); + const line = (n: number) => `${BOOT1 + 10 + n} desktop line ${n}\n`; + write( + "desktop.log", + Array.from({ length: 10 }, (_, n) => line(n)).join(""), + ); + const cap = { SUPERSET_LOGSHIP_CAP_BYTES: "1000" }; + await ship(cap); + const kept = items().filter((i) => !isCap(i)); + const size = (i: Item) => Buffer.byteLength(JSON.stringify(i)); + const keptBytes = kept.reduce((n, i) => n + size(i), 0); + expect(keptBytes).toBeLessThanOrEqual(1000); + expect(keptBytes + size(kept[0] as Item)).toBeGreaterThan(1000); + // Counting bodies alone would have kept every line. + expect(kept.length).toBeLessThan(11); + expect(items().filter(isCap)).toHaveLength(1); + + const hour = readFileSync(join(state(), "hour"), "utf8").trim().split(" "); + expect(Number(hour[1])).toBe(keptBytes + size(items().find(isCap) as Item)); + expect(hour[2]).toBe("1"); + + // A record small enough to fit what is left still waits for the next hour. + append("desktop.log", `${BOOT1 + 99} x\n`); + received = []; + await ship(cap); + expect(received).toHaveLength(0); + + append("desktop.log", line(11)); + await ship({ ...cap, SUPERSET_LOGSHIP_NOW_MS: String(NOW + HOUR_MS) }); + expect(bodies()).toEqual([line(11).trimEnd()]); + }); + + test("sends lines written before a sleep at the next wake, once, and waits for a line's end", async () => { + bootStart(BOOT1); + append("boot.log", `${BOOT1 + 100} host.ready\n`); + await ship(); + expect(bySource("boot")).toHaveLength(2); + + // Written after the last pass, then the box sleeps: no process survives. + append( + "boot.log", + `${BOOT1 + 200} desktop.start\n${BOOT1 + 300} checkout.en`, + ); + received = []; + await ship(); + expect(bodies()).toEqual([`${BOOT1 + 200} desktop.start`]); + + // The wake: a new boot appends to the same file. + const boot2 = BOOT1 + 1_000_000; + append("boot.log", `d\n${boot2} boot.start pid=2\n`); + received = []; + await ship(); + expect(bodies()).toEqual([ + `${BOOT1 + 300} checkout.end`, + `${boot2} boot.start pid=2`, + ]); + expect(items().map((i) => attr(i, "box.boot_started_at"))).toEqual([ + BOOT1, + boot2, + ]); + + received = []; + await ship(); + expect(received).toHaveLength(0); + expect(readFileSync(join(state(), "boot.offset"), "utf8").trim()).toBe( + String(statSync(join(logs(), "boot.log")).size), + ); + }); + + test("turned on for the first time, every file starts at the current boot", async () => { + const boot2 = BOOT1 + 1_000_000; + bootStart(BOOT1); + append("boot.log", `${BOOT1 + 1} host.ready\n`); + bootStart(boot2); + write( + "host-service.log", + `${iso(BOOT1 + 5)} old\n old stack\n${iso(boot2 - 1)} last old\n old stack\n${iso(boot2 + 7)} new\n new stack\n`, + ); + write("memguard.log", `${BOOT1 + 9} memguard old\n${boot2} memguard new\n`); + write("desktop.log", `${BOOT1 + 9} desktop old\n`); + await ship(); + expect(bodies()).toEqual([ + `${boot2} boot.start pid=1`, + `${boot2} memguard new`, + `${iso(boot2 + 7)} new`, + " new stack", + ]); + expect(readFileSync(join(state(), "desktop.offset"), "utf8").trim()).toBe( + String(statSync(join(logs(), "desktop.log")).size), + ); + }); + + test("on, then off for a boot, then on again: nothing written while off is sent", async () => { + bootStart(BOOT1); + write("host-service.log", `${iso(BOOT1 + 1)} on in boot 1\n`); + await ship(); + expect(bodies()).toEqual([ + `${BOOT1} boot.start pid=1`, + `${iso(BOOT1 + 1)} on in boot 1`, + ]); + + // Boot 2 runs with the flag off. Its runner should remove the state; an + // older runner leaves it, so the shipper must not trust it either. + const boot2 = BOOT1 + 1_000_000; + bootStart(boot2); + append( + "host-service.log", + `${iso(boot2 + 1)} WHILE-OFF\n WHILE-OFF stack\n`, + ); + write("desktop.log", `${boot2 + 2} desktop WHILE-OFF\n`); + + const boot3 = BOOT1 + 2_000_000; + bootStart(boot3); + append("host-service.log", `${iso(boot3 + 1)} on in boot 3\n`); + received = []; + await ship(); + expect(JSON.stringify(received)).not.toContain("WHILE-OFF"); + expect(bodies()).toEqual([ + `${boot3} boot.start pid=1`, + `${iso(boot3 + 1)} on in boot 3`, + ]); + }); + + test.each([ + [403, {}], + [429, { "Retry-After": "60" }], + ])("a %i from Sentry waits quietly and keeps every offset", async (code, reply) => { + status = code; + headers = reply; + bootStart(BOOT1); + const out = await ship( + { SUPERSET_LOGSHIP_INTERVAL_S: "0.02" }, + [], + () => received.length > 0, + ); + expect(received).toHaveLength(1); + expect(out.trim().split("\n")).toHaveLength(1); + expect(out).toContain(`refused: ${code}`); + expect(existsSync(join(state(), "boot.offset"))).toBe(false); + const files = readdirSync(state()).sort(); + expect(files).toEqual(["boot", "lock", "retry"]); + + await ship({ SUPERSET_LOGSHIP_NOW_MS: String(NOW + 30_000) }); + expect(received).toHaveLength(1); + await ship({ SUPERSET_LOGSHIP_NOW_MS: String(NOW + HOUR_MS) }); + expect(received).toHaveLength(2); + expect(readdirSync(state()).sort()).toEqual(files); + + status = 200; + await ship({ SUPERSET_LOGSHIP_NOW_MS: String(NOW + 2 * HOUR_MS) }); + expect(received).toHaveLength(3); + expect(existsSync(join(state(), "boot.offset"))).toBe(true); + }); + + test("does nothing when the claim did not turn it on", async () => { + bootStart(BOOT1); + await ship({ SUPERSET_SANDBOX_BOX_LOGS: "" }); + await ship({ HOST_SERVICE_SENTRY_DSN: "" }); + expect(received).toHaveLength(0); + expect(existsSync(join(root, "state"))).toBe(false); + }); +}); + +describe("the start hook's stamps", () => { + const fn = /\nstart_hook\(\) \{\n[\s\S]*?\n\}\n/.exec(boot)?.[0] ?? ""; + const CUSTOMER = "CUSTOMER-OUTPUT secret-db-url pnpm run acme"; + + async function runHook(answer: object): Promise { + const host = Bun.serve({ + port: 0, + fetch(req) { + const json = new URL(req.url).pathname.endsWith(".runStartHook") + ? answer + : { environmentPushed: true }; + return Response.json({ result: { data: { json } } }); + }, + }); + const run = join(root, "run"); + mkdirSync(run, { recursive: true }); + writeFileSync(join(run, "host-service.ready"), ""); + const log = join(root, `boot-${crypto.randomUUID()}.log`); + const proc = Bun.spawn( + [ + "bash", + "-c", + `set -uo pipefail\nstamp() { printf '%s %s\\n' "$(date +%s%3N)" "$*" >> "$LOG"; }\n${fn}\nstart_hook`, + ], + { + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + LOG: log, + SUPERSET_HOST_SERVICE_PORT: String(host.port), + SUPERSET_RUN_DIR: run, + HOST_SERVICE_SECRET: "boot-secret", + }, + }, + ); + await proc.exited; + host.stop(true); + return readFileSync(log, "utf8"); + } + + test("a hook that fails to start is stamped without its command or output", async () => { + expect(fn).not.toBe(""); + const log = await runHook({ + started: false, + reason: "failed", + command: CUSTOMER, + exitCode: 2, + log: CUSTOMER, + }); + expect(log).toMatch(/^\d{13} hook\.start\.failed\n$/); + }); + + test("a started hook is stamped with its pid only", async () => { + const log = await runHook({ started: true, pid: 4321, command: CUSTOMER }); + expect(log).toMatch(/^\d{13} hook\.started pid=4321\n$/); + }); +}); diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index d1f814f070d..c41ad6448d2 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -200,6 +200,10 @@ export const FEATURE_FLAGS = { * on every create and wake. Unloaded or offline means no drive. */ SANDBOX_AGENT_TOOLS_DRIVE: "sandbox-agent-tools-drive", + /** Sends an organization's box logs to Sentry; needs `SANDBOX_BOX_LOGS_FLEET` too. */ + SANDBOX_BOX_LOGS: "sandbox-box-logs", + /** One answer for every box (asked for the subject `sandbox-fleet`, no group): off stops box logs at each box's next wake. */ + SANDBOX_BOX_LOGS_FLEET: "sandbox-box-logs-fleet", /** * Moving a workspace and its chats to the cloud or to another computer: * shows the Move items, and gates diff --git a/packages/shared/src/sandbox/contract.ts b/packages/shared/src/sandbox/contract.ts index 9fd3d568b03..a383b7579a5 100644 --- a/packages/shared/src/sandbox/contract.ts +++ b/packages/shared/src/sandbox/contract.ts @@ -113,6 +113,8 @@ export const sandboxIdentitySchema = z.object({ SUPERSET_SANDBOX_PROVIDER: z.string().min(1), HOST_SERVICE_SENTRY_DSN: z.string().optional(), HOST_SERVICE_SENTRY_ENVIRONMENT: z.string().optional(), + /** Starts `superset-logship`, which sends the platform's box logs to Sentry. Absent is off. */ + SUPERSET_SANDBOX_BOX_LOGS: z.literal("1").optional(), /** A built-in agent to run once on first boot; see cloud-agent-launch. */ SUPERSET_SANDBOX_AGENT: z.string().optional(), SUPERSET_SANDBOX_AGENT_PROMPT: z.string().optional(), diff --git a/packages/trpc/src/lib/sandbox/agent-tools.test.ts b/packages/trpc/src/lib/sandbox/agent-tools.test.ts deleted file mode 100644 index 44ece4b0191..00000000000 --- a/packages/trpc/src/lib/sandbox/agent-tools.test.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { describe, expect, test } from "bun:test"; -import { stub } from "../../../test/stub"; -import { posthog } from "../analytics"; -import { agentToolsDriveEnabled } from "./agent-tools"; - -let answer: () => Promise = () => Promise.resolve(true); -const asked: unknown[][] = []; -stub(posthog, { - isFeatureEnabled: (...args: unknown[]) => { - asked.push(args); - return answer(); - }, -}); - -const ORG = "6f1c2b8e-1d2a-4c55-9a3e-0b7f5d2c9e11"; - -describe("agentToolsDriveEnabled", () => { - test("asks the org's flag", async () => { - answer = () => Promise.resolve(true); - expect(await agentToolsDriveEnabled(ORG)).toBe(true); - expect(asked.at(-1)?.[0]).toBe("sandbox-agent-tools-drive"); - expect(asked.at(-1)?.[2]).toMatchObject({ - groups: { organization: ORG }, - }); - }); - - test("fails closed when the flag is off, unloaded or PostHog is down", async () => { - for (const next of [ - () => Promise.resolve(false), - () => Promise.resolve(undefined), - () => Promise.reject(new Error("offline")), - ]) { - answer = next; - expect(await agentToolsDriveEnabled(ORG)).toBe(false); - } - }); -}); diff --git a/packages/trpc/src/lib/sandbox/agent-tools.ts b/packages/trpc/src/lib/sandbox/agent-tools.ts deleted file mode 100644 index bb14bb7bc0a..00000000000 --- a/packages/trpc/src/lib/sandbox/agent-tools.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { FEATURE_FLAGS } from "@superset/shared/constants"; -import { posthog } from "../analytics"; - -/** Fails closed: PostHog unreachable means no drive. */ -export async function agentToolsDriveEnabled( - organizationId: string, -): Promise { - const enabled = await posthog - .isFeatureEnabled(FEATURE_FLAGS.SANDBOX_AGENT_TOOLS_DRIVE, organizationId, { - groups: { organization: organizationId }, - sendFeatureFlagEvents: false, - }) - .catch(() => undefined); - return enabled === true; -} diff --git a/packages/trpc/src/lib/sandbox/claim.ts b/packages/trpc/src/lib/sandbox/claim.ts index 8e51b6e5e59..21d8bab8258 100644 --- a/packages/trpc/src/lib/sandbox/claim.ts +++ b/packages/trpc/src/lib/sandbox/claim.ts @@ -13,6 +13,7 @@ import { type CloudAgentLaunch, cloudAgentLaunchToEnv, } from "@superset/shared/cloud-agent-launch"; +import { FEATURE_FLAGS } from "@superset/shared/constants"; import { SANDBOX_CONTRACT_VERSION, type SandboxIdentity, @@ -25,11 +26,12 @@ import { resolveAgentSignIns, } from "../../router/agent-credential"; import { resolveEnvironment } from "../../router/environment/resolve-environment"; +import { posthog } from "../analytics"; import { resolveMachineSize } from "../billing/plan"; import { githubUserConnectionFor, githubUserTokenFor } from "../github-user"; import { sandboxHostSecretFor } from "./access"; -import { agentToolsDriveEnabled } from "./agent-tools"; import { deriveSandboxCredentials, gitAuthorFor } from "./credentials"; +import { boxLogsEnabled, organizationFlagEnabled } from "./feature-flags"; import { creatorPlugins } from "./plugins"; import { readRepoHooks } from "./repo-hooks"; import { @@ -88,11 +90,14 @@ export async function buildSandboxClaim(args: { repositories: SandboxRepository[]; agentCredentialDigest: string; }> { - const [environment, signIns] = await Promise.all([ + const [environment, signIns, boxLogs] = await Promise.all([ resolveEnvironment(args.row.environmentId, args.row.organizationId), args.row.createdByUserId ? resolveAgentSignIns({ userId: args.row.createdByUserId }) : Promise.resolve({ env: {}, codexChatgpt: null }), + env.SENTRY_DSN_SANDBOX + ? boxLogsEnabled(posthog, args.row.organizationId) + : false, ]); if (!environment) throw new Error("Environment not found"); const checkouts = await workspaceRepositories({ @@ -147,6 +152,7 @@ export async function buildSandboxClaim(args: { env.NEXT_PUBLIC_SENTRY_ENVIRONMENT ?? "development", } : {}), + ...(boxLogs ? { SUPERSET_SANDBOX_BOX_LOGS: "1" as const } : {}), ...(cloudAgentLaunchToEnv(args.launch) as Partial), }; const { networkPolicy, managedEnv, files } = await deriveSandboxCredentials({ @@ -175,7 +181,11 @@ export async function buildSandboxClaim(args: { files, ports: repoHooks?.ports, vcpus: await boxVcpus(args.row), - agentTools: await agentToolsDriveEnabled(args.row.organizationId), + agentTools: await organizationFlagEnabled( + posthog, + FEATURE_FLAGS.SANDBOX_AGENT_TOOLS_DRIVE, + args.row.organizationId, + ), }, environment: { sourceKind: environment.sourceKind, diff --git a/packages/trpc/src/lib/sandbox/feature-flags.test.ts b/packages/trpc/src/lib/sandbox/feature-flags.test.ts new file mode 100644 index 00000000000..6ec74b5f421 --- /dev/null +++ b/packages/trpc/src/lib/sandbox/feature-flags.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, test } from "bun:test"; +import { boxLogsEnabled, organizationFlagEnabled } from "./feature-flags"; + +const ORG = "6f1c2b8e-1d2a-4c55-9a3e-0b7f5d2c9e11"; + +function flags(answers: Record Promise>) { + const asked: unknown[][] = []; + return { + asked, + isFeatureEnabled: (...args: unknown[]) => { + asked.push(args); + return answers[args[0] as string]?.() ?? Promise.resolve(undefined); + }, + }; +} +const on = () => Promise.resolve(true); +const off = () => Promise.resolve(false); + +describe("organizationFlagEnabled", () => { + test("asks the flag for the organization and its group", async () => { + const posthog = flags({ "some-flag": on }); + expect(await organizationFlagEnabled(posthog, "some-flag", ORG)).toBe(true); + expect(posthog.asked).toEqual([ + [ + "some-flag", + ORG, + { groups: { organization: ORG }, sendFeatureFlagEvents: false }, + ], + ]); + }); + + test("fails closed when the flag is off, unloaded or PostHog is down", async () => { + for (const answer of [ + off, + () => Promise.resolve(undefined), + () => Promise.reject(new Error("offline")), + ]) { + expect( + await organizationFlagEnabled( + flags({ "some-flag": answer }), + "some-flag", + ORG, + ), + ).toBe(false); + } + }); +}); + +describe("boxLogsEnabled", () => { + test("is on when the fleet switch and the organization's flag are both on", async () => { + const posthog = flags({ + "sandbox-box-logs-fleet": on, + "sandbox-box-logs": on, + }); + expect(await boxLogsEnabled(posthog, ORG)).toBe(true); + expect(posthog.asked.map((args) => args.slice(0, 2))).toEqual([ + ["sandbox-box-logs-fleet", "sandbox-fleet"], + ["sandbox-box-logs", ORG], + ]); + expect(posthog.asked[0]?.[2]).toEqual({ sendFeatureFlagEvents: false }); + }); + + test("with the fleet switch off, asks nothing else and is off", async () => { + const posthog = flags({ + "sandbox-box-logs-fleet": off, + "sandbox-box-logs": on, + }); + expect(await boxLogsEnabled(posthog, ORG)).toBe(false); + expect(posthog.asked).toHaveLength(1); + }); +}); diff --git a/packages/trpc/src/lib/sandbox/feature-flags.ts b/packages/trpc/src/lib/sandbox/feature-flags.ts new file mode 100644 index 00000000000..14945e395df --- /dev/null +++ b/packages/trpc/src/lib/sandbox/feature-flags.ts @@ -0,0 +1,39 @@ +import { FEATURE_FLAGS } from "@superset/shared/constants"; +import type { PostHog } from "posthog-node"; + +type Flags = Pick; + +/** Fails closed: an unloaded flag or an unreachable PostHog is off. */ +export async function organizationFlagEnabled( + flags: Flags, + flag: string, + organizationId: string, +): Promise { + const enabled = await flags + .isFeatureEnabled(flag, organizationId, { + groups: { organization: organizationId }, + sendFeatureFlagEvents: false, + }) + .catch(() => undefined); + return enabled === true; +} + +/** Both switches must be on; the organization's flag is asked only while the fleet switch is. */ +export async function boxLogsEnabled( + flags: Flags, + organizationId: string, +): Promise { + const fleet = await flags + .isFeatureEnabled(FEATURE_FLAGS.SANDBOX_BOX_LOGS_FLEET, "sandbox-fleet", { + sendFeatureFlagEvents: false, + }) + .catch(() => undefined); + return ( + fleet === true && + organizationFlagEnabled( + flags, + FEATURE_FLAGS.SANDBOX_BOX_LOGS, + organizationId, + ) + ); +}