diff --git a/docs/cloud-sandbox-mismatches.md b/docs/cloud-sandbox-mismatches.md index 644e4a0f857..988674e3af3 100644 --- a/docs/cloud-sandbox-mismatches.md +++ b/docs/cloud-sandbox-mismatches.md @@ -930,7 +930,16 @@ sandbox gets one only from `Sandbox.create`, `Sandbox.fork` or and `wakeSandbox` both pass it. A drive lives in one region and mounts only in that region. A session without the mount leaves links in `/usr/local/bin` that point nowhere; `superset-boot` re-runs `install-agent-clis` to put the -image's agents back. +image's agents back. OpenCode and Pi exist only on the drive, so for them it +deletes the link instead. + +**An agent on the drive cannot update itself.** On a machine someone owns, +OpenCode and Pi update in place, beside their binary. On a box they run from +a read-only mount and are pinned by the bundle. `/etc/profile.d/superset.sh` +turns off their update checks (`OPENCODE_DISABLE_AUTOUPDATE`, +`PI_SKIP_VERSION_CHECK`), the same way `DISABLE_AUTOUPDATER` does for +Claude. They keep their state under `~/.local/share/opencode`, +`~/.config/opencode` and `~/.pi/agent`, as they do on a laptop. **A read-only snapshot mount taken while a writer holds the drive is corrupt.** Measured 2026-11-17: a box that mounted the snapshot while the builder still diff --git a/packages/sandbox/README.md b/packages/sandbox/README.md index 2fdb4b2c7ea..cbb3c63f7d8 100644 --- a/packages/sandbox/README.md +++ b/packages/sandbox/README.md @@ -22,6 +22,12 @@ checklist is `plans/20260913-sandbox-implementation-checklist.md`. `sync-assets` does not fetch them again, and fetches them again only when the step has to re-run. A file the box runs in place (`claude`, `superset`) is never discarded. +- **An agent only the agents drive carries** (`opencode`, `pi`) is a row + with no `dest`. `assets.tsv` leaves it out, so neither the image nor a + boot fetches it; `bun run build --publish` still mirrors it to the bucket, + and `bun run agent-tools` fetches it from there by sha. Its pin goes into + the bundle's `drive-assets.tsv`, which no box reads, so a bump alone still + names a new bundle and so a new drive. A step cannot declare such a row. - **Something the box has to execute to configure itself** is a step: `bundle/steps/.sh`, registered in `bundle/steps.json` with the assets it consumes, the rootfs files it reads (`inputs`) and the steps it must @@ -46,8 +52,8 @@ The rule of thumb: if you can answer "where is it on the box", put it there. | `bun run build --publish` | Also uploads the bundle and any asset the bucket lacks, from the local cache or else its row's `source`, checked against the row's sha256. Refuses to publish a bundle that references an asset the bucket does not have. | | `bun run image` | Builds the image: OS, apt lists, vendor tools, the `ubuntu` user, then installs the bundle at its sha and runs every pass once, so a fresh box finds every hash sidecar and step marker matching. A push logs Docker into the registry itself with a credential minted from `VERCEL_SANDBOX_TOKEN`, so there is no login to renew. `--local` builds `superset-sandbox:local` for the boot-twice test; `--dry` prints the Dockerfile. | | boot (`superset-boot`) | On the box, as root, run by the control plane on every create and wake, after it has written `sandbox.conf`, with the host secret in its env: clears `/run/superset`, installs the pinned bundle if the hash differs, runs the three passes (each a hash compare), starts host-service as `ubuntu` with the secret from the boot env, writes `resources.env`, starts `superset-memguard` (ranks every process for the OOM killer and reaps the largest before the kernel has to), then the desktop and the checkout. Once host-service answers, the environment has been pushed and the checkout is in, the runner asks host-service to run the repo's `start` hook with that environment. | -| `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex and Chrome rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | -| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. `bun run release` does this for every region with an environment, six at a time, before it records the bundle, and afterwards deletes the drives no environment names. | +| `bun run release` | Rewrites the host-service and CLI assets from this checkout and the claude, codex, Chrome, opencode and pi rows to their latest upstream release, publishes the bundle, pushes the image under a candidate tag `release-` (unless `--skip-image`), builds the internal golden from it with the environment's `setup` hook, probes a fork of it (`src/environments/probe.ts`) including a stop and wake, and only after the probe passes moves `latest` to the candidate and writes the environment rows. Every image environment resolves `superset-hostsvc` to `latest`, so a failed release changes no box. | +| `bun run agent-tools --bundle ` | Writes that bundle's agent CLIs to `agent-tools--`, a drive boxes mount read-only at `/opt/agents` when their organization has the `sandbox-agent-tools-drive` flag. A drive is written once and never again; it starts as a fork of the region's newest earlier drive, so only changed agents are fetched, and an agent that fails its download or `--version` check keeps the earlier drive's version or is left out. The drive carries claude, codex and pdev (also in the image) and opencode and pi (only on the drive). `bun run release` does this for every region with an environment, six at a time, before it records the bundle, and afterwards deletes the drives no environment names. | | `bun run src/real-sandbox.ts` | The Sandbox workflow dispatched with `real-sandbox`: one real sandbox from the registry image on this checkout's bundle, the same probe, a stop and a wake. | | Sandbox workflow, `production-release` | `bun run release --production` from `main` on a CI runner: the image, the internal golden and production's environment rows. Runs on a merge that touches the bundle and every night; a failed merge or nightly run opens an issue, or comments on the open one. | @@ -82,7 +88,8 @@ image or a boot writes is paid for in every snapshot. - An asset: its sha256, same sidecar rule; it is also the object's name in the bucket. - A step: `sha256(script + declared asset shas + declared input hashes + versions of steps it follows + salt)`. - The bundle: the sha256 of its tarball; the one pointer the box holds, written into `/etc/superset/sandbox.conf` by the control plane. -- host-service, the CLI, claude, codex and Chrome move at release time, so their committed rows are only the floor a pull request's boot test uses. The other rows move only when someone runs `bun run assets`. +- host-service, the CLI, claude, codex, Chrome, opencode and pi move at release time, so their committed rows are only the floor a pull request's boot test uses. The other rows move only when someone runs `bun run assets`. +- opencode and pi follow upstream like claude and codex, from GitHub releases. `bun run assets opencode` (or `pi`), which the release runs, takes the highest stable `vX.Y.Z` release that has the linux-x64 tarball, never a draft or prerelease, and not GitHub's `latest` marker, which for OpenCode names an older major line. The bytes must match the sha256 digest GitHub publishes for that asset; a missing or different digest refuses the bump, and the release keeps the committed pin with a `::warning::`. A version the agent-tools check rejects keeps the earlier drive's version of that agent. `bun run assets opencode ` pins a version by hand, with the same digest check. ## The runner in the image runs first diff --git a/packages/sandbox/bundle/assets.json b/packages/sandbox/bundle/assets.json index 177dce60011..f387dc1eca7 100644 --- a/packages/sandbox/bundle/assets.json +++ b/packages/sandbox/bundle/assets.json @@ -166,6 +166,13 @@ "version": "1.29.0+bfc1ec72a", "node": 24 }, + "opencode": { + "sha256": "fc0f81522aceeabac47693d88389efa2bc517ce74c2c38fd6967819fcc6e4d3b", + "suffix": ".tar.gz", + "mode": "0644", + "source": "https://github.com/anomalyco/opencode/releases/download/v2.3.0/opencode-linux-x64.tar.gz", + "version": "2.3.0" + }, "pdev": { "sha256": "5a51852a4de0a40e20c0137b873f83e1b26255b4c5d01b6b0a2f608e152d8535", "suffix": ".tar.gz", @@ -174,6 +181,13 @@ "source": "https://github.com/ParadevAI/pdev/releases/download/v0.2.5/pdev-0.2.5-linux-x64.tar.gz", "version": "0.2.5" }, + "pi": { + "sha256": "811c6f2c9e7aaee252b5404252e556fb713aa209c1f7f924226aa77d4ef70cb5", + "suffix": ".tar.gz", + "mode": "0644", + "source": "https://github.com/earendil-works/pi/releases/download/v1.5.2/pi-linux-x64.tar.gz", + "version": "1.5.2" + }, "wallpaper-0": { "sha256": "497a4edb230a58ea385c8b63fe8135e47554cb4a5169de3f6bbae293a1f5435a", "suffix": ".jpg", diff --git a/packages/sandbox/bundle/rootfs/etc/profile.d/superset.sh b/packages/sandbox/bundle/rootfs/etc/profile.d/superset.sh index a2e27de7f73..6eae7dfec23 100644 --- a/packages/sandbox/bundle/rootfs/etc/profile.d/superset.sh +++ b/packages/sandbox/bundle/rootfs/etc/profile.d/superset.sh @@ -10,6 +10,7 @@ export DISPLAY="${DISPLAY:-${SUPERSET_DISPLAY:-:1}}" # The box's memory ceiling and CPUs, written by superset-boot from the cgroup. [ -r "${SUPERSET_RUN_DIR:-/run/superset}/resources.env" ] && . "${SUPERSET_RUN_DIR:-/run/superset}/resources.env" case ":$PATH:" in *":/usr/local/go/bin:"*) ;; *) export PATH="$PATH:/usr/local/go/bin" ;; esac -# The agent CLIs are pinned by the bundle (install-agent-clis); a self-update -# would drift from the pin and only prints a warning every session. -export DISABLE_AUTOUPDATER=1 +# The agent CLIs are pinned by the bundle (install-agent-clis) or run from the +# read-only agents drive; a self-update would drift from the pin and only +# prints a warning every session. +export DISABLE_AUTOUPDATER=1 OPENCODE_DISABLE_AUTOUPDATE=1 PI_SKIP_VERSION_CHECK=1 diff --git a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot index 6dd3e6e2bfd..78d57a589e6 100755 --- a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot +++ b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-boot @@ -124,9 +124,11 @@ stamp "steps.start"; "$BUNDLE/setup" run-steps >> "$SUPERSET_BOOT_LOG" 2>&1; sta link_agents() { local dir="${SUPERSET_AGENTS_DIR:-/opt/agents}" linked=0 name sha bin target local manifest="${dir}/manifest.json" - if find /usr/local/bin -maxdepth 1 -lname "${dir}/*" ! -exec test -e {} \; -print | grep -q .; then + if find /usr/local/bin -maxdepth 1 -lname "${dir}/*" ! -exec test -e {} \; -print -quit | grep -q .; then SUPERSET_BUNDLE_DIR="$BUNDLE" bash "$BUNDLE/steps/install-agent-clis.sh" >> "$SUPERSET_BOOT_LOG" 2>&1 || stamp "agents.restore.failed" + # What is still dangling has no image copy to restore: opencode, pi. + find /usr/local/bin -maxdepth 1 -lname "${dir}/*" ! -exec test -e {} \; -delete fi if ! jq -e 'type == "array"' "$manifest" > /dev/null 2>&1; then stamp "agents.unavailable no manifest at ${dir}" diff --git a/packages/sandbox/src/agent-tools.test.ts b/packages/sandbox/src/agent-tools.test.ts index 7dac023edce..c8db13b598d 100644 --- a/packages/sandbox/src/agent-tools.test.ts +++ b/packages/sandbox/src/agent-tools.test.ts @@ -21,7 +21,7 @@ import { publishAgentToolsDrives, renderBuilderScript, } from "./agent-tools"; -import { type Assets, sha256 } from "./manifest"; +import { type Assets, loadAssets, sha256 } from "./manifest"; const sha = (c: string) => c.repeat(64); const BUNDLE = "0123456789abcdef".repeat(4); @@ -35,6 +35,8 @@ const agentAssets: Assets = { codex: row(sha("b"), ".tar.gz"), pdev: row(sha("c"), ".tar.gz"), chrome: row(sha("d"), ".deb"), + opencode: { sha256: sha("e"), suffix: ".tar.gz", mode: "0644" }, + pi: { sha256: sha("f"), suffix: ".tar.gz", mode: "0644" }, }; describe("agent-tools manifest", () => { @@ -44,10 +46,29 @@ describe("agent-tools manifest", () => { { name: "claude", sha: sha("a"), bin: "claude" }, { name: "codex", sha: sha("b"), bin: "bin/codex" }, { name: "pdev", sha: sha("c"), bin: "pdev/pdev" }, + { name: "opencode", sha: sha("e"), bin: "opencode" }, + { name: "pi", sha: sha("f"), bin: "pi/pi" }, ]); expect(parseAgentToolsManifest(JSON.stringify(manifest))).toEqual(manifest); }); + test("the committed bundle has a row for every agent, and opencode and pi only on the drive", () => { + const assets = loadAssets( + join(import.meta.dir, "..", "bundle", "assets.json"), + ); + expect(agentToolsManifest(assets).map((entry) => entry.name)).toEqual([ + "claude", + "codex", + "pdev", + "opencode", + "pi", + ]); + for (const name of ["opencode", "pi"]) { + expect(assets[name]?.dest).toBeUndefined(); + expect(assets[name]?.source).toStartWith("https://github.com/"); + } + }); + test("a missing or bad manifest reads as empty", () => { expect(parseAgentToolsManifest(null)).toEqual([]); expect(parseAgentToolsManifest("{not json")).toEqual([]); @@ -142,11 +163,9 @@ describe("publishAgentToolsDrive", () => { expect(result.drive).toBe(drive); expect(result.outcome).toBe("published"); expect(result.forkedFrom).toBeUndefined(); - expect(result.agents.map((agent) => agent.outcome)).toEqual([ - "written", - "written", - "written", - ]); + expect(result.agents.map((agent) => agent.outcome)).toEqual( + Array(5).fill("written"), + ); expect(calls).toEqual([`create ${drive}`, `write ${drive}`]); expect(agentLine(scripts[drive], "claude")).toEndWith("'' '' '--version'"); }); diff --git a/packages/sandbox/src/agent-tools.ts b/packages/sandbox/src/agent-tools.ts index 1c422daa7c8..2d94e54cdc6 100644 --- a/packages/sandbox/src/agent-tools.ts +++ b/packages/sandbox/src/agent-tools.ts @@ -41,6 +41,8 @@ export const AGENT_TOOLS: readonly AgentTool[] = [ { name: "claude", bin: "claude" }, { name: "codex", bin: "bin/codex" }, { name: "pdev", bin: "pdev/pdev" }, + { name: "opencode", bin: "opencode" }, + { name: "pi", bin: "pi/pi" }, ]; export interface AgentTool { @@ -411,7 +413,7 @@ export function describeAgentToolsPublish( ] : agent.outcome === "dropped" ? [ - `::warning::agent-tools: ${drive}: ${agent.name} left out; its boxes keep the image's ${agent.name}`, + `::warning::agent-tools: ${drive}: ${agent.name} left out; its boxes keep the image's ${agent.name}, if the image has one`, ] : [], ), diff --git a/packages/sandbox/src/assets/produce.test.ts b/packages/sandbox/src/assets/produce.test.ts new file mode 100644 index 00000000000..ec2b2a62b64 --- /dev/null +++ b/packages/sandbox/src/assets/produce.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from "bun:test"; +import { type GithubRelease, pickRelease } from "./produce"; + +const ASSET = "tool-linux-x64.tar.gz"; + +function release( + tag: string, + overrides: Partial = {}, +): GithubRelease { + return { + tag_name: tag, + draft: false, + prerelease: false, + assets: [ + { + name: ASSET, + digest: `sha256:${"a".repeat(64)}`, + browser_download_url: `https://example.invalid/${tag}/${ASSET}`, + }, + ], + ...overrides, + }; +} + +const pick = (releases: GithubRelease[]) => + pickRelease(releases, ASSET)?.version; + +describe("pickRelease", () => { + test("takes the highest version, not the first listed or an older line's newer patch", () => { + expect( + pick([release("v1.19.6"), release("v2.3.0"), release("v2.2.9")]), + ).toBe("2.3.0"); + }); + + test("skips drafts, prereleases, other tags and releases without the asset", () => { + expect( + pick([ + release("v3.0.0", { draft: true }), + release("v2.9.0", { prerelease: true }), + release("v2.8.0-rc.1"), + release("process-daemon-v9.0.0"), + release("v2.7.0", { assets: [] }), + release("v2.6.0"), + ]), + ).toBe("2.6.0"); + }); + + test("no stable release picks nothing", () => { + expect(pick([release("v1.0.0", { prerelease: true })])).toBeUndefined(); + }); +}); diff --git a/packages/sandbox/src/assets/produce.ts b/packages/sandbox/src/assets/produce.ts index e1b319795aa..fc17f99335f 100644 --- a/packages/sandbox/src/assets/produce.ts +++ b/packages/sandbox/src/assets/produce.ts @@ -8,6 +8,8 @@ * bun run assets host-service the runtime tarball from this checkout * bun run assets claude [version] Claude Code's native binary (default: latest) * bun run assets codex [version] Codex's standalone package (default: latest) + * bun run assets opencode [version] OpenCode's linux-x64 release, drive only (default: newest stable) + * bun run assets pi [version] Pi's linux-x64 release, drive only (default: newest stable) * bun run assets all * * Each producer writes its files to .cache/assets/, then @@ -60,7 +62,9 @@ function rewriteRows(rows: Record): void { ); writeFileSync(ASSETS_JSON, `${JSON.stringify(sorted, null, "\t")}\n`); for (const [name, row] of Object.entries(rows)) - console.log(`${name}: ${assetObjectKey(row)} -> ${row.dest}`); + console.log( + `${name}: ${assetObjectKey(row)} -> ${row.dest ?? "agent-tools drive only"}`, + ); } async function fetchBytes(url: string): Promise { @@ -538,6 +542,105 @@ async function pdev(version?: string): Promise { }); } +export interface GithubRelease { + tag_name: string; + draft: boolean; + prerelease: boolean; + assets: { + name: string; + digest?: string | null; + browser_download_url: string; + }[]; +} + +/** + * The highest stable `vX.Y.Z` release that carries the asset. Not GitHub's + * `latest`: OpenCode's still names an older major line. + */ +export function pickRelease( + releases: readonly GithubRelease[], + assetName: string, +): { version: string; asset: GithubRelease["assets"][number] } | undefined { + return releases + .flatMap((release) => { + const asset = release.assets.find((a) => a.name === assetName); + return !release.draft && + !release.prerelease && + asset && + /^v\d+\.\d+\.\d+$/.test(release.tag_name) + ? [{ version: release.tag_name.slice(1), asset }] + : []; + }) + .sort((a, b) => Bun.semver.order(b.version, a.version))[0]; +} + +/** + * An agent that boxes get only from the agent-tools drive: the row has no + * `dest`, so neither the image nor a boot fetches it. Without a version it + * takes the newest stable release. The bytes must match GitHub's digest for + * the asset. + */ +async function driveOnlyAgent(args: { + name: string; + repo: string; + asset: string; + version?: string; +}): Promise { + const api = `https://api.github.com/repos/${args.repo}/releases`; + const picked = args.version + ? { + version: args.version, + asset: ( + JSON.parse( + await fetchText(`${api}/tags/v${args.version}`), + ) as GithubRelease + ).assets.find((a) => a.name === args.asset), + } + : pickRelease( + JSON.parse(await fetchText(`${api}?per_page=100`)) as GithubRelease[], + args.asset, + ); + if (!picked?.asset) + throw new Error( + `${args.name}: no ${args.asset} ${args.version ? `in v${args.version}` : "in a stable release"}`, + ); + const { version, asset } = picked; + if (!asset.digest?.startsWith("sha256:")) + throw new Error(`${args.name} ${version}: GitHub publishes no digest`); + const bytes = await fetchBytes(asset.browser_download_url); + const hash = sha256(bytes); + if (`sha256:${hash}` !== asset.digest) + throw new Error( + `${args.name} ${version}: sha256 ${hash}, release ${asset.digest}`, + ); + cache(bytes, ".tar.gz"); + rewriteRows({ + [args.name]: { + sha256: hash, + suffix: ".tar.gz", + mode: "0644", + source: asset.browser_download_url, + version, + }, + }); +} + +const opencode = (version?: string) => + driveOnlyAgent({ + name: "opencode", + repo: "anomalyco/opencode", + asset: "opencode-linux-x64.tar.gz", + version, + }); + +const pi = (version?: string) => + driveOnlyAgent({ + name: "pi", + repo: "earendil-works/pi", + asset: "pi-linux-x64.tar.gz", + version, + }); + export const producers: Record< string, (version?: string) => Promise | void @@ -552,6 +655,8 @@ export const producers: Record< claude, codex, pdev, + opencode, + pi, }; if (import.meta.main) { diff --git a/packages/sandbox/src/boot-twice.ts b/packages/sandbox/src/boot-twice.ts index 97f3f54d857..e715c0d1c0a 100644 --- a/packages/sandbox/src/boot-twice.ts +++ b/packages/sandbox/src/boot-twice.ts @@ -358,14 +358,20 @@ try { const stubSha = "f".repeat(64); stop(); exec( - `: > ${SANDBOX_PATHS.bootLog}; mkdir -p ${SANDBOX_PATHS.agents}/claude/${stubSha} && printf '#!/bin/sh\\necho drive-claude\\n' > ${SANDBOX_PATHS.agents}/claude/${stubSha}/claude && chmod 755 ${SANDBOX_PATHS.agents}/claude/${stubSha}/claude && printf '%s' '${JSON.stringify([{ name: "claude", sha: stubSha, bin: "claude" }])}' > ${SANDBOX_PATHS.agents}/manifest.json`, + `: > ${SANDBOX_PATHS.bootLog}; for name in claude pi; do mkdir -p ${SANDBOX_PATHS.agents}/$name/${stubSha} && printf '#!/bin/sh\\necho drive-%s\\n' $name > ${SANDBOX_PATHS.agents}/$name/${stubSha}/$name && chmod 755 ${SANDBOX_PATHS.agents}/$name/${stubSha}/$name; done && printf '%s' '${JSON.stringify( + [ + { name: "claude", sha: stubSha, bin: "claude" }, + { name: "pi", sha: stubSha, bin: "pi" }, + ], + )}' > ${SANDBOX_PATHS.agents}/manifest.json`, ); boot(3); const logDrive = exec(`cat ${SANDBOX_PATHS.bootLog}`); expect( - "drive: listed agent linked", - /agents\.linked 1/.test(logDrive) && - /drive-claude/.test(exec("claude", { check: false, user: "ubuntu" })), + "drive: listed agents linked", + /agents\.linked 2/.test(logDrive) && + /drive-claude/.test(exec("claude", { check: false, user: "ubuntu" })) && + /drive-pi/.test(exec("pi", { check: false, user: "ubuntu" })), logDrive.match(/agents\..*/)?.[0], ); stop(); @@ -391,6 +397,12 @@ try { ), exec("readlink -f /usr/local/bin/claude", { check: false }).trim(), ); + expect( + "no drive: a drive-only agent's link is gone", + exec("test -L /usr/local/bin/pi && echo dangling", { + check: false, + }).trim() === "", + ); // Third boot, with host-service deliberately broken: the box must stay // alive, the desktop must still come up, and the log must say so. diff --git a/packages/sandbox/src/build.test.ts b/packages/sandbox/src/build.test.ts new file mode 100644 index 00000000000..acd0f9bc611 --- /dev/null +++ b/packages/sandbox/src/build.test.ts @@ -0,0 +1,22 @@ +import { afterAll, expect, test } from "bun:test"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BUNDLE_SRC, buildBundle } from "./build"; +import { loadAssets } from "./manifest"; + +const dist = mkdtempSync(join(tmpdir(), "bundle-")); +afterAll(() => rmSync(dist, { recursive: true, force: true })); + +test("a drive-only pin change names a new bundle and re-runs no step", () => { + const assets = loadAssets(join(BUNDLE_SRC, "assets.json")); + const before = buildBundle(assets, dist); + const opencode = assets.opencode; + if (!opencode) throw new Error("assets.json has no opencode row"); + const after = buildBundle( + { ...assets, opencode: { ...opencode, sha256: "0".repeat(64) } }, + dist, + ); + expect(after.sha256).not.toBe(before.sha256); + expect(after.steps).toEqual(before.steps); +}); diff --git a/packages/sandbox/src/build.ts b/packages/sandbox/src/build.ts index 9e1d61d6ccc..02101fe5be4 100644 --- a/packages/sandbox/src/build.ts +++ b/packages/sandbox/src/build.ts @@ -5,6 +5,7 @@ * setup, steps/, rootfs/ copied * rootfs/etc/superset/contract.sh rendered from @superset/shared/sandbox/contract * tools.tsv, assets.tsv, steps.tsv derived + * drive-assets.tsv the drive-only pins; hashed, never read on the box * dist/bundle.tar.gz the same tree, named by its sha in the bucket * * bun run build.ts build only; prints the bundle sha @@ -36,6 +37,7 @@ import { loadAssets, loadSteps, renderAssetsTsv, + renderDriveAssetsTsv, renderStepsTsv, renderToolsTsv, sha256, @@ -58,8 +60,11 @@ export interface BuiltBundle { steps: DerivedStep[]; } -export function buildBundle(): BuiltBundle { - const staging = join(DIST, "staging"); +export function buildBundle( + assets: Assets = loadAssets(join(BUNDLE_SRC, "assets.json")), + dist = DIST, +): BuiltBundle { + const staging = join(dist, "staging"); rmSync(staging, { recursive: true, force: true }); mkdirSync(staging, { recursive: true }); cpSync(join(BUNDLE_SRC, "setup"), join(staging, "setup")); @@ -75,7 +80,6 @@ export function buildBundle(): BuiltBundle { renderContractShell(), ); - const assets = loadAssets(join(BUNDLE_SRC, "assets.json")); const tools = walkRootfs(join(staging, "rootfs")); const declaredSteps = loadSteps(join(BUNDLE_SRC, "steps.json")); const steps = deriveStepVersions({ @@ -90,19 +94,23 @@ export function buildBundle(): BuiltBundle { renderAssetsTsv(assets, declaredSteps), ); writeFileSync(join(staging, "steps.tsv"), renderStepsTsv(steps)); + writeFileSync( + join(staging, "drive-assets.tsv"), + renderDriveAssetsTsv(assets), + ); // A reproducible tarball, written here rather than by the system tar so the // sha is the tree's on every machine. - const tarball = join(DIST, "bundle.tar.gz"); + const tarball = join(dist, "bundle.tar.gz"); const bytes = tarGzDirectory(staging); writeFileSync(tarball, bytes); const hash = sha256(bytes); - const dir = join(DIST, "bundle", hash); - rmSync(join(DIST, "bundle"), { recursive: true, force: true }); - mkdirSync(join(DIST, "bundle"), { recursive: true }); + const dir = join(dist, "bundle", hash); + rmSync(join(dist, "bundle"), { recursive: true, force: true }); + mkdirSync(join(dist, "bundle"), { recursive: true }); cpSync(staging, dir, { recursive: true }); rmSync(staging, { recursive: true, force: true }); - writeFileSync(join(DIST, "bundle.sha256"), `${hash}\n`); + writeFileSync(join(dist, "bundle.sha256"), `${hash}\n`); return { sha256: hash, dir, tarball, assets, steps }; } diff --git a/packages/sandbox/src/environments/release.ts b/packages/sandbox/src/environments/release.ts index 72836cb16ee..23c65bef6db 100644 --- a/packages/sandbox/src/environments/release.ts +++ b/packages/sandbox/src/environments/release.ts @@ -5,8 +5,8 @@ * * 1. the host-service runtime asset (this checkout's host-service and * pty-daemon, natives installed for the box), this checkout's CLI, the - * latest claude, codex and Chrome (a fetch that fails keeps the - * committed pin) and the bundle, published + * latest claude, codex, Chrome, opencode and pi (a fetch or digest + * check that fails keeps the committed pin) and the bundle, published * 2. the image in Vercel Container Registry under a candidate tag, unless * --skip-image; `latest`, which every image environment resolves to, * stays on the previous release @@ -60,7 +60,7 @@ process.env.SKIP_ENV_VALIDATION ??= "1"; const SKIP_IMAGE = process.argv.includes("--skip-image"); const KEEP_OLD = process.argv.includes("--keep-old"); const PRODUCTION = process.argv.includes("--production"); -const FOLLOWED_UPSTREAM = ["claude", "codex", "chrome"]; +const FOLLOWED_UPSTREAM = ["claude", "codex", "chrome", "opencode", "pi"]; /** The environment the release rebuilds; without it, the release creates one named INTERNAL_NAME. */ const ENVIRONMENT_ID = process.env.SUPERSET_INTERNAL_ENVIRONMENT_ID; const INTERNAL_NAME = diff --git a/packages/sandbox/src/manifest.test.ts b/packages/sandbox/src/manifest.test.ts index 61fb8c7eb2f..1deb50ec649 100644 --- a/packages/sandbox/src/manifest.test.ts +++ b/packages/sandbox/src/manifest.test.ts @@ -141,6 +141,22 @@ describe("deriveStepVersions", () => { expect(build({ salt: "x" })[1]?.version).not.toBe(build({})[1]?.version); }); + test("refuses a step that consumes a drive-only row", () => { + const f = fixture(); + expect(() => + deriveStepVersions({ + steps: [ + { name: "one", assets: ["agent"], after: [], inputs: [], salt: "" }, + ], + assets: { + agent: { sha256: "d".repeat(64), suffix: ".tar.gz", mode: "0644" }, + }, + tools: walkRootfs(f.rootfs), + stepsDir: f.steps, + }), + ).toThrow(/drive-only/); + }); + test("refuses unknown assets, missing inputs and forward references", () => { const f = fixture(); const tools = walkRootfs(f.rootfs); @@ -288,6 +304,14 @@ describe("renderAssetsTsv", () => { ); }); + test("a drive-only row has no line, so no box fetches it", () => { + const tsv = renderAssetsTsv({ + ...assets, + agent: { sha256: "d".repeat(64), suffix: ".tar.gz", mode: "0644" }, + }); + expect(tsv).toBe(renderAssetsTsv(assets)); + }); + test("refuses a discard that is not the step's asset, or one two steps claim", () => { expect(() => discardingSteps([step("one", ["tool"], ["thing"])])).toThrow( /not one of its assets/, diff --git a/packages/sandbox/src/manifest.ts b/packages/sandbox/src/manifest.ts index 180aae173d2..666c224fff1 100644 --- a/packages/sandbox/src/manifest.ts +++ b/packages/sandbox/src/manifest.ts @@ -18,8 +18,11 @@ const assetSchema = z.object({ * Empty for an executable, which has none. */ suffix: z.string().regex(/^(\.[a-z0-9.]+)?$/), - /** Absolute path on the box. Archives are staged under the media dir. */ - dest: z.string().startsWith("/"), + /** + * Absolute path on the box. Archives are staged under the media dir. + * Absent for a drive-only row: no box fetches it, only the agent-tools drive. + */ + dest: z.string().startsWith("/").optional(), mode: z .string() .regex(/^0[0-7]{3}$/) @@ -123,17 +126,31 @@ export function discardingSteps(steps: Step[]): Map { return owners; } -/** The fifth column names the step that discards the row, empty for a row that stays. */ +/** + * The fifth column names the step that discards the row, empty for a row that + * stays. A drive-only row has no line: no box fetches it. + */ export function renderAssetsTsv(assets: Assets, steps: Step[] = []): string { const owners = discardingSteps(steps); - return Object.entries(assets) - .map(([key, a]) => - [a.mode, a.sha256, a.suffix, b64(a.dest), owners.get(key) ?? ""].join( + const rows = Object.entries(assets).flatMap(([key, a]) => + a.dest ? [{ key, ...a, dest: a.dest }] : [], + ); + return rows + .map((a) => + [a.mode, a.sha256, a.suffix, b64(a.dest), owners.get(a.key) ?? ""].join( "\t", ), ) .join("\n") - .concat(Object.keys(assets).length ? "\n" : ""); + .concat(rows.length ? "\n" : ""); +} + +/** Puts the drive-only pins in the bundle's hash, so a bump names a new drive. No box reads it. */ +export function renderDriveAssetsTsv(assets: Assets): string { + return Object.entries(assets) + .filter(([, a]) => !a.dest) + .map(([key, a]) => `${key}\t${a.sha256}\n`) + .join(""); } export interface DerivedStep extends Step { @@ -188,6 +205,10 @@ export function deriveStepVersions(args: { const asset = args.assets[key]; if (!asset) throw new Error(`step ${step.name} declares unknown asset ${key}`); + if (!asset.dest) + throw new Error( + `step ${step.name} declares ${key}, which is drive-only and never on the box`, + ); parts.push(`asset:${key}:${asset.sha256}`); } for (const input of step.inputs) {