diff --git a/.gitignore b/.gitignore index 82bd83e1e..982c5a8e9 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,8 @@ result-* # NOTE: generated clients (go/gen, packages/compass-client/src/gen) # are intentionally committed and CI drift-gated — do not ignore them. + +# runner-image build outputs: the staged nix closure (which carries the +# entrypoint symlink) and the OCI layout, realised by tools/runner-image/build.ts. +/runner-image/store/ +/runner-image/out/ diff --git a/.moon/workspace.yml b/.moon/workspace.yml index a718eec44..773d9d18a 100644 --- a/.moon/workspace.yml +++ b/.moon/workspace.yml @@ -39,12 +39,23 @@ projects: # cache lane or V2a boot bring-up. Same affected-detection posture as # compass-agent-image (see guest-image/moon.yml). compass-guest-image: 'guest-image' + # The Runner container image: the compass-runner binary, the KVM userland and + # the guest assets on a minimal hardened base. Its `build` is runInCI:false (it + # needs a buildkitd and a multi-GB realise), so registration here is for graph + # membership and affected-detection, not a gate leg; the gate coverage is the + # runner-image tool's pure-core suite below. See runner-image/moon.yml. + compass-runner-image: 'runner-image' # The local microVM boot-test lane (RIG-2591): realises the guest image + VMM # stack from nix and execs the KVM-gated `go test -tags microvm` suite the # untagged compass-go:test lane never builds. Registered so its typecheck + # unit tests ride the moon-driven CI sweep; the boot lane itself lives on # compass-go:test-microvm (runInCI:false — it needs KVM + a nix build). microvm-boot-test: 'tools/microvm-boot-test' + # The runner-image build lane: realises the nix closure, stages it, and drives + # the Dockerfile build. Registered so its typecheck + pure-core unit tests ride + # the CI sweep — that suite is what catches a build-arg or closure-root drift, + # since no compass CI step can build the image itself today. + runner-image: 'tools/runner-image' # The Compass native-app release bundle: a heavy nix build (realises the # WebKitGTK cc/pkg-config closure) that stages the versioned tarball. Same # affected-detection posture as compass-agent-image — registered here so the diff --git a/bun.lock b/bun.lock index 7e4f2f793..786a61944 100644 --- a/bun.lock +++ b/bun.lock @@ -236,6 +236,16 @@ "typescript": "catalog:", }, }, + "tools/runner-image": { + "name": "@compass/runner-image", + "bin": { + "runner-image-build": "./build.ts", + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:", + }, + }, "tools/sea-ref-gate": { "name": "@compass/sea-ref-gate", "bin": { @@ -491,6 +501,8 @@ "@compass/renovate-preflight": ["@compass/renovate-preflight@workspace:tools/renovate-preflight"], + "@compass/runner-image": ["@compass/runner-image@workspace:tools/runner-image"], + "@compass/sea-ref-gate": ["@compass/sea-ref-gate@workspace:tools/sea-ref-gate"], "@compass/sql-migration-gate": ["@compass/sql-migration-gate@workspace:tools/sql-migration-gate"], diff --git a/runner-image/.dockerignore b/runner-image/.dockerignore new file mode 100644 index 000000000..4080d8467 --- /dev/null +++ b/runner-image/.dockerignore @@ -0,0 +1,6 @@ +# Context prune for the runner image build. The context is runner-image/ itself, +# which holds only the Dockerfile, the staged store/ closure and the +# compass-runner entrypoint symlink — so this excludes the build OUTPUT (out/), +# which would otherwise be re-transferred into every rebuild's context and grow +# it without bound. +out diff --git a/runner-image/Dockerfile b/runner-image/Dockerfile new file mode 100644 index 000000000..166872001 --- /dev/null +++ b/runner-image/Dockerfile @@ -0,0 +1,101 @@ +# syntax=docker/dockerfile:1.7-labs +# The compass-runner container image (R1 of the Compass Runner containerization +# record). Carries the Runner binary, the KVM userland it exec's +# (cloud-hypervisor, virtiofsd, passt) and the three guest assets the microVM +# backend direct-boots, so a Runner pod needs no host toolchain — only /dev/kvm +# and the session-volume mount. +# +# A Dockerfile on a minimal base, not a nix2container image: the mechanism +# follows the image's RUNTIME, and the Runner is a prebuilt application that +# runs no toolchain. Full rationale in the design record's R1 task; nix still +# builds every carried artifact, and build.ts COPYs the result. + +# Distroless STATIC, not :base and not Alpine. The carried binaries are +# dynamically linked but nix-closed: each names an ABSOLUTE /nix/store +# interpreter and resolves every NEEDED library through its own RPATH, so the +# closure supplies its own ld.so and glibc and the base is never consulted for a +# library. That makes the base's own libc dead weight, and `static` is the +# smallest base that still provides what IS needed from it: /etc/passwd + +# /etc/group (the nonroot uid), CA certificates (the Runner dials the Server +# over TLS), and /tmp. Verified: with the closure present all four binaries +# execute; with ONLY cloud-hypervisor's own store path copied — its glibc +# omitted — the same exec fails "missing dynamic library", which is the control +# proving the loader comes from the closure rather than the base. +# +# The pinned digest is the nonroot variant (uid/gid 65532). The Runner needs no +# root: /dev/kvm access is granted by supplementary group at the pod layer. +# Pinned by digest, never by tag — GHCR-style tag mutability gives no +# immutability guarantee. +FROM gcr.io/distroless/static-debian12@sha256:c0f429e16b13e583da7e5a6ec20dd656d325d88e6819cafe0adb0828976529dc + +# The realised nix closure: the Runner binary, the KVM userland and the guest +# assets, each at its own absolute /nix/store path. ONE COPY of a +# build.ts-staged directory rather than a COPY per component, because the store +# paths reference each other by absolute path — splitting them across layers by +# component would interleave a binary and its glibc into different layers for no +# cache benefit, since a Go rebuild moves the Runner path and nothing else. +# +# Ownership is left at the default (root-owned, world-readable) and the image +# runs as nonroot: the closure is read-only at runtime, so a non-root process +# needs no write access to it. This also keeps the layer bit-identical to the +# staged tree, which is what makes the digest reproducible across rebuilds. +COPY store /nix/store + +# The stable entrypoint name is /nix/store/.compass-runner, a RELATIVE symlink +# build.ts stages inside store/ (so it arrives with the COPY above — there is no +# second COPY). ENTRYPOINT is exec-form and cannot expand a build ARG, while the +# Runner's own store path carries a hash that moves on every Go rebuild; the +# symlink gives the image a fixed entrypoint whose target moves with the build. +# +# It must be relative and inside the staged tree. An absolute symlink at the +# context root dangles on the BUILD HOST, and BuildKit checksums context entries +# before any COPY runs, so such a link fails the build outright with "not found" +# rather than resolving later inside the image. + +# Absolute store paths, resolved at build time from the staged closure by +# build.ts and passed in as build args. They are baked into ENV rather than +# hardcoded because every path carries a content hash that moves on any rebuild +# of its input. +# +# The Runner reads each of these as the documented env fallback for the matching +# --microvm-* flag (main.go registerBackendFlags), so an operator can still +# override any one on the command line without rebuilding the image. Setting +# them here is what makes the image self-describing: the microVM backend's +# preflight is fail-closed (it requires both a static support probe and a real +# boot canary), so a missing or wrong path fails at startup, loudly, rather than +# at first session. +ARG VMM_BIN +ARG VIRTIOFSD_BIN +ARG GUEST_KERNEL +ARG GUEST_ROOTFS +ARG GUEST_INITRD + +ENV COMPASS_RUNTIME_BACKEND=microvm \ + COMPASS_MICROVM_VMM=${VMM_BIN} \ + COMPASS_MICROVM_VIRTIOFSD=${VIRTIOFSD_BIN} \ + COMPASS_MICROVM_KERNEL=${GUEST_KERNEL} \ + COMPASS_MICROVM_ROOTFS=${GUEST_ROOTFS} \ + COMPASS_MICROVM_INITRD=${GUEST_INITRD} + +# passt is exec'd by name, not by configured path (unlike the VMM and virtiofsd, +# which have --microvm-* flags), so its bin dir must be on PATH. Prepended to +# the base PATH rather than replacing it. +# Consumed by the dockerfile frontend (paired with the export's +# rewrite-timestamp) to fix the config `created` field and the layer mtimes, so +# two builds of the same inputs yield the same digest. +ARG SOURCE_DATE_EPOCH + +ARG STACK_BIN_DIR +ENV PATH=${STACK_BIN_DIR}:/usr/local/bin:/usr/bin:/bin + +# ENTRYPOINT, not CMD: the image runs exactly one program, and an ENTRYPOINT +# lets a pod spec's `args` add flags without re-stating the binary path. Exec +# form, so the Runner is pid 1 and receives SIGTERM directly at pod shutdown — +# a shell wrapper would swallow it and leave the graceful drain to the kill +# timeout. +# Explicit, rather than inherited from the base's own config: a future digest +# bump that landed on the root `static` variant would otherwise silently produce +# a root-running container, and nothing here would catch it. +USER 65532:65532 + +ENTRYPOINT ["/nix/store/.compass-runner"] diff --git a/runner-image/moon.yml b/runner-image/moon.yml new file mode 100644 index 000000000..a96169df3 --- /dev/null +++ b/runner-image/moon.yml @@ -0,0 +1,72 @@ +# yaml-language-server: $schema=https://moonrepo.dev/schemas/project.json +# +# compass-runner-image (runner-image/): the Runner container image — the +# compass-runner binary, the KVM userland it exec's (cloud-hypervisor, +# virtiofsd, passt) and the three guest assets the microVM backend direct-boots. +# This project carries the Dockerfile and the staged build context; the +# realise-and-stage logic is tools/runner-image (TypeScript, not a shell script). +# +# A Dockerfile project, not a nix one like the sibling agent-image/ and +# guest-image/: their artifacts ARE nix derivations, this image's runtime is a +# distroless base. See the design record's R1 task. +# +# WHY THERE IS NO `ci` TASK. Unlike agent-image/ and guest-image/, this project +# deliberately does NOT ride the pre-merge gate: the build needs a reachable +# buildkitd (which no compass CI step carries today) and it +# stages a multi-GB closure dominated by the guest rootfs. Registering a `ci` +# task now would add an unrunnable task to the gate, so the gate coverage this +# project has today is the pure-core suite in tools/runner-image, which is where +# a mapping drift would actually be caught. The publish lane adds the first CI leg that builds this image. +layer: 'application' +language: 'bash' +# ci-group.nix with the sibling image projects: the CI matrix requires every +# project to carry exactly one group tag, and this project's real build closure +# is the nix one those legs already provision. It contributes no CI task today +# (`build` is runInCI:false), so the tag places it in the graph rather than +# adding a leg. +tags: ['ci-group.nix'] + +workspace: + inheritedTasks: + # Not a bun package — no package.json, no bun.lock — so the tag-bun `install` + # must never run here; `lint` and `format` are whole-repo tasks on the root + # project, never inherited. The same defensive guard the sibling image + # projects carry. + exclude: ['install', 'lint', 'format'] + +tasks: + build: + # Realise the closure and build the image. runInCI:false — the build needs a + # buildkitd and a multi-GB nix realise, so this is the dev-box/publish entry + # point, not a gate task. cache:false — nix owns the store-path caching and + # buildkit owns the layer caching, so moon must not false-green a build it + # did not re-run (the posture the sibling image projects take). + command: 'bun tools/runner-image/build.ts' + options: + cache: false + runInCI: false + runFromWorkspaceRoot: true + # The image's true build closure. A leading `/` is workspace-root-relative + # (the convention the sibling image projects use); the bare glob is this + # project's own tree. Each entry is an input whose change alters the built + # image: the Dockerfile and the build script directly, the Go module + + # runner command for the binary, and the guest-image/VMM-env closures for + # the carried assets. A glob that misses one would leave a stale image + # passing as current. + # Named tracked files, NOT a `**/*` glob. moon hashes with its native walker, + # which reads contents directly and does not skip gitignored paths — a bare + # glob here would hash the multi-GB staged closure this task itself writes + # into store/, on every affected-computation. + inputs: + - 'Dockerfile' + - '.dockerignore' + - 'moon.yml' + - '/tools/runner-image/**/*' + - '/flake.nix' + - '/flake.lock' + - '/go/go.mod' + - '/go/go.sum' + - '/go/cmd/compass-runner/**' + - '/go/internal/**' + - '/guest-image/**' + - '/tools/toolchain/microvm-vmm-env.nix' diff --git a/tools/runner-image/biome.json b/tools/runner-image/biome.json new file mode 100644 index 000000000..ece11dd9b --- /dev/null +++ b/tools/runner-image/biome.json @@ -0,0 +1,4 @@ +{ + "extends": "//", + "linter": { "rules": { "suspicious": { "noConsole": "off" } } } +} diff --git a/tools/runner-image/build-core.test.ts b/tools/runner-image/build-core.test.ts new file mode 100644 index 000000000..32fe265d0 --- /dev/null +++ b/tools/runner-image/build-core.test.ts @@ -0,0 +1,164 @@ +// Unit tests for the runner-image build lane's pure core. Each case pins a +// behaviour the image's correctness depends on, and each would fail on a +// plausible bug — not on a restatement of the implementation. + +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + buildArgs, + buildctlArgs, + closureRoots, + kernelImagePath, + outputSpec, + parseOutPaths, + type RunnerImageOutputs, +} from "./build-core.ts"; + +const outputs: RunnerImageOutputs = { + runner: "/nix/store/aaa-compass-runner", + stack: "/nix/store/bbb-compass-stack-env", + kernelDir: "/nix/store/ccc-linux", + rootfs: "/nix/store/ddd-rootfs.erofs", + initrd: "/nix/store/eee-initrd", +}; + +describe("parseOutPaths", () => { + test("drops the blank trailing line nix emits", () => { + expect(parseOutPaths("/nix/store/a\n/nix/store/b\n")).toEqual([ + "/nix/store/a", + "/nix/store/b", + ]); + }); + + test("yields nothing for empty stdout, so a silent build failure cannot read as one path", () => { + expect(parseOutPaths("")).toEqual([]); + expect(parseOutPaths("\n \n")).toEqual([]); + }); +}); + +describe("kernelImagePath", () => { + // The kernel derivation is a directory (bzImage + System.map); the rootfs and + // initrd derivations ARE files. Booting the directory would fail at the VMM. + test("points at the bzImage inside the kernel derivation directory", () => { + expect(kernelImagePath("/nix/store/ccc-linux")).toBe( + "/nix/store/ccc-linux/bzImage", + ); + }); +}); + +describe("buildArgs", () => { + test("maps each artifact to the env the Runner reads, kernel suffixed and the file assets bare", () => { + expect(buildArgs(outputs)).toEqual({ + VMM_BIN: "/nix/store/bbb-compass-stack-env/bin/cloud-hypervisor", + VIRTIOFSD_BIN: "/nix/store/bbb-compass-stack-env/bin/virtiofsd", + STACK_BIN_DIR: "/nix/store/bbb-compass-stack-env/bin", + GUEST_KERNEL: "/nix/store/ccc-linux/bzImage", + GUEST_ROOTFS: "/nix/store/ddd-rootfs.erofs", + GUEST_INITRD: "/nix/store/eee-initrd", + }); + }); +}); + +describe("closureRoots", () => { + // The roots are derivation out-paths: `nix path-info -r` takes store paths, + // and passing the bzImage FILE instead of its directory would stage a partial + // closure whose binaries fail to exec. + test("uses the kernel DIRECTORY, not the bzImage inside it", () => { + const roots = closureRoots(outputs); + expect(roots).toContain("/nix/store/ccc-linux"); + expect(roots).not.toContain("/nix/store/ccc-linux/bzImage"); + }); + + test("covers all five artifacts, so no carried binary is staged without its dependencies", () => { + expect(closureRoots(outputs).sort()).toEqual( + [ + outputs.runner, + outputs.stack, + outputs.kernelDir, + outputs.rootfs, + outputs.initrd, + ].sort(), + ); + }); +}); + +describe("outputSpec", () => { + test("oci writes a browsable layout, which is what the publish lane scans before pushing", () => { + expect(outputSpec("oci", "ignored:tag", "/tmp/out")).toBe( + "type=oci,dest=/tmp/out,tar=false,rewrite-timestamp=true", + ); + }); + + test("image names the tag for a local load", () => { + expect(outputSpec("image", "compass-runner:dev", "/tmp/out")).toBe( + "type=image,name=compass-runner:dev,rewrite-timestamp=true", + ); + }); + + // The digest-stability property the publish lane depends on: without this, + // two builds of a bit-identical staged tree still export different layer + // digests, because the context's mtimes ride into the layer tar. + test("both modes rewrite layer timestamps, so a rebuild is digest-stable", () => { + expect(outputSpec("oci", "t", "/tmp/out")).toContain( + "rewrite-timestamp=true", + ); + expect(outputSpec("image", "t", "/tmp/out")).toContain( + "rewrite-timestamp=true", + ); + }); + + test("oci mode never names the tag, so a dev tag cannot leak into a layout build", () => { + expect(outputSpec("oci", "compass-runner:dev", "/tmp/out")).not.toContain( + "compass-runner:dev", + ); + }); +}); + +describe("buildctlArgs", () => { + test("passes every build-arg the Dockerfile declares", () => { + const args = buildctlArgs( + "/repo/runner-image", + outputs, + "linux/amd64", + "type=oci,dest=/repo/runner-image/out,tar=false", + ); + const joined = args.join(" "); + for (const name of Object.keys(buildArgs(outputs))) { + expect(joined).toContain(`build-arg:${name}=`); + } + }); + + test("build-arg order is deterministic, so two identical builds produce identical argv", () => { + const once = buildctlArgs("/ctx", outputs, "linux/amd64", "type=oci"); + const twice = buildctlArgs("/ctx", outputs, "linux/amd64", "type=oci"); + expect(once).toEqual(twice); + }); + + // The load-bearing drift test: buildArgs is only correct RELATIVE to the + // Dockerfile's own ARG declarations, and nothing else compares the two. It is + // what caught a RUNNER_BIN that build-core supplied and the Dockerfile had + // stopped consuming. + test("supplies exactly the build-args the Dockerfile declares", () => { + const dockerfile = readFileSync( + join(import.meta.dir, "..", "..", "runner-image", "Dockerfile"), + "utf8", + ); + const declared = new Set( + [...dockerfile.matchAll(/^ARG\s+([A-Z_][A-Z0-9_]*)/gm)].map( + (m) => m[1] as string, + ), + ); + // SOURCE_DATE_EPOCH is consumed by the frontend itself and supplied by + // buildctlArgs directly, not through the artifact mapping. + declared.delete("SOURCE_DATE_EPOCH"); + expect(new Set(Object.keys(buildArgs(outputs)))).toEqual(declared); + }); + + test("uses the dockerfile.v0 frontend against the staged context", () => { + const args = buildctlArgs("/ctx", outputs, "linux/amd64", "type=oci"); + expect(args.slice(0, 3)).toEqual(["build", "--frontend", "dockerfile.v0"]); + expect(args).toContain("context=/ctx"); + expect(args).toContain("dockerfile=/ctx"); + }); +}); diff --git a/tools/runner-image/build-core.ts b/tools/runner-image/build-core.ts new file mode 100644 index 000000000..67fd4f838 --- /dev/null +++ b/tools/runner-image/build-core.ts @@ -0,0 +1,156 @@ +// The pure core of the runner-image build lane (R1). Every function here is a +// total map over its inputs with no I/O, so build-core.test.ts can drive each +// mapping — and each fail-closed edge — without nix, buildkit, or a subprocess. +// +// The lane's job is to turn realised nix store paths into the two things the +// container build needs: the set of paths to stage into the build context, and +// the build-args that bake those absolute paths into the image's env. + +/** The fixed build epoch. Nix normalises every store mtime to 1, so using the + * same value keeps the rewritten layer timestamps equal to what the staged tree + * already carries. */ +export const SOURCE_DATE_EPOCH = 1; + +/** The six artifacts the image carries, as realised store paths. */ +export interface RunnerImageOutputs { + /** The `compass-runner` package out-path (the binary is at `bin/compass-runner`). */ + runner: string; + /** The `compass-stack-env` symlinkJoin (cloud-hypervisor, virtiofsd, passt under `bin/`). */ + stack: string; + /** The guest kernel DERIVATION dir; the bootable artifact is `/bzImage`. */ + kernelDir: string; + /** The guest rootfs image — the derivation IS the file. */ + rootfs: string; + /** The guest initramfs — the derivation IS the file. */ + initrd: string; +} + +/** Split `nix build --print-out-paths` stdout into trimmed, non-empty store + * paths, one per line. Mirrors the microvm-boot-test lane's parser: the same + * command shape deserves the same reader, not a second convention. */ +export function parseOutPaths(stdout: string): string[] { + return stdout + .split("\n") + .map((line) => line.trim()) + .filter((line) => line !== ""); +} + +/** + * The absolute in-image path of the bootable kernel. The kernel derivation is a + * DIRECTORY (it also carries System.map); cloud-hypervisor direct-boots the + * bzImage inside it, with no bootloader. The rootfs and initrd derivations are + * files, so only this one gains a suffix — the asymmetry is the derivations', + * not a convention. + */ +export function kernelImagePath(kernelDir: string): string { + return `${kernelDir}/bzImage`; +} + +/** + * The `--opt build-arg:` values the Dockerfile bakes into ENV. Every value is an + * absolute /nix/store path resolved at build time, because each carries a + * content hash that moves whenever its input is rebuilt — hardcoding any of them + * in the Dockerfile would pin a stale artifact that the staged closure no longer + * contains. + * + * These land on the documented `COMPASS_MICROVM_*` env fallbacks for the + * matching `--microvm-*` flags, so an operator can still override any one at + * runtime without rebuilding. + */ +export function buildArgs(outputs: RunnerImageOutputs): Record { + return { + VMM_BIN: `${outputs.stack}/bin/cloud-hypervisor`, + VIRTIOFSD_BIN: `${outputs.stack}/bin/virtiofsd`, + // passt is exec'd by NAME rather than by a configured path (it has no + // --microvm-* flag), so the image needs its bin dir on PATH. + STACK_BIN_DIR: `${outputs.stack}/bin`, + GUEST_KERNEL: kernelImagePath(outputs.kernelDir), + GUEST_ROOTFS: outputs.rootfs, + GUEST_INITRD: outputs.initrd, + }; +} + +/** + * The closure ROOTS whose transitive dependencies must be staged into the build + * context. + * + * The roots are the derivation out-paths, NOT the inner file paths: `nix + * path-info -r` takes store paths, and the kernel's root is its directory even + * though the image references the bzImage inside it. + * + * Staging the transitive closure — rather than just these five — is + * load-bearing. Each carried binary names an absolute /nix/store interpreter and + * resolves every NEEDED library through its own RPATH, so a context missing the + * transitive glibc produces an image whose binaries fail to exec with "missing + * dynamic library". That is the measured negative control behind the base-image + * choice, not a hypothetical. + */ +export function closureRoots(outputs: RunnerImageOutputs): string[] { + return [ + outputs.runner, + outputs.stack, + outputs.kernelDir, + outputs.rootfs, + outputs.initrd, + ]; +} + +/** The buildctl `--output` spec for each supported output mode. `oci` writes a + * browsable local layout (what the publish lane scans BEFORE deciding to push); + * `image` names a tagged image for a local dogfood load. */ +export function outputSpec( + mode: "oci" | "image", + tag: string, + ociDir: string, +): string { + // rewrite-timestamp normalises every layer entry's mtime to SOURCE_DATE_EPOCH. + // Without it two builds of a BIT-IDENTICAL staged tree still produce different + // layer digests, because BuildKit carries the context's mtimes into the layer + // tar. Measured: only the `COPY store` layer differed between runs; with this + // set, two builds yield the same manifest digest. + const rewrite = "rewrite-timestamp=true"; + return mode === "oci" + ? `type=oci,dest=${ociDir},tar=false,${rewrite}` + : `type=image,name=${tag},${rewrite}`; +} + +/** + * The full `buildctl` argv (after the binary). Rootless BuildKit is the ruled + * mechanism for a prebuilt-application image; this lane never shells out to + * `docker build` and never mounts a host docker socket, which would hand the + * build the daemon's blast radius. + */ +export function buildctlArgs( + contextDir: string, + outputs: RunnerImageOutputs, + platform: string, + output: string, +): string[] { + const args = [ + "build", + "--frontend", + "dockerfile.v0", + "--local", + `context=${contextDir}`, + "--local", + `dockerfile=${contextDir}`, + "--opt", + "filename=Dockerfile", + "--opt", + `platform=${platform}`, + // Pairs with rewrite-timestamp: this fixes the image config's `created` + // field and is the epoch every layer mtime is rewritten to. 1, not 0, + // matching the mtime nix normalises its store paths to. + "--opt", + `build-arg:SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}`, + ]; + // Sorted, so the argv is deterministic across runs: an unstable arg order + // would make two otherwise-identical builds diff in logs for no reason. + for (const [name, value] of Object.entries(buildArgs(outputs)).sort( + ([a], [b]) => a.localeCompare(b), + )) { + args.push("--opt", `build-arg:${name}=${value}`); + } + args.push("--output", output); + return args; +} diff --git a/tools/runner-image/build.ts b/tools/runner-image/build.ts new file mode 100755 index 000000000..658e5edd3 --- /dev/null +++ b/tools/runner-image/build.ts @@ -0,0 +1,268 @@ +#!/usr/bin/env bun +// Build the compass-runner container image (R1 of the Compass Runner +// containerization record): realise the nix closure, stage it into a build +// context, and hand that plain directory to a rootless BuildKit Dockerfile +// build. +// +// THE MECHANISM IS THE RULED ONE, and it is part of the reviewed surface rather +// than an implementation detail. The fleet's first-party image-builds spec +// splits mechanisms by what the image's RUNTIME is, not by what built the +// artifact: an image that IS a Nix environment earns nix2container; a PREBUILT +// APPLICATION on a minimal base is a Dockerfile built by rootless BuildKit. The +// Runner is the latter — it runs no toolchain, it exec's four binaries — so nix +// is the BUILDER here and never the runtime. The prescribed shape is `nix +// build` then COPY, which is exactly the two halves below. +// +// TypeScript, not bash: this has real logic — four nix builds whose out-paths +// are parsed and mapped to distinct build-args, a transitive closure staged +// path-by-path, and a fail-fast on every missing output — so the parsing and +// mapping core is pure and unit-tested (./build-core.test.ts) while this file is +// the thin I/O shell. +// +// Usage: +// bun tools/runner-image/build.ts [--tag ] [--output oci|image] + +import { spawnSync } from "node:child_process"; +import { + chmodSync, + cpSync, + existsSync, + mkdirSync, + readdirSync, + rmSync, + symlinkSync, +} from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + buildctlArgs, + closureRoots, + outputSpec, + parseOutPaths, + type RunnerImageOutputs, +} from "./build-core.ts"; + +// This file is tools/runner-image/build.ts, so `../..` is the workspace root. +const workspaceRoot = join(dirname(fileURLToPath(import.meta.url)), "..", ".."); +const imageDir = join(workspaceRoot, "runner-image"); +const stageDir = join(imageDir, "store"); +const ociDir = join(imageDir, "out"); + +// Single-arch linux/amd64: the Runner's microVM backend needs KVM on the node, +// and the cluster nodes are amd64. A second leg would need its own KVM-capable +// builder, so it is added when a node arch is, not speculatively. +const IMAGE_PLATFORM = "linux/amd64"; + +function parseArgs(argv: readonly string[]): { + tag: string; + mode: "oci" | "image"; +} { + let tag = "compass-runner:dev"; + let mode: "oci" | "image" = "oci"; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + const value = argv[i + 1]; + if (arg === "--tag" && value !== undefined) { + tag = value; + i += 1; + } else if (arg === "--output" && value !== undefined) { + if (value !== "oci" && value !== "image") { + console.error(`--output must be 'oci' or 'image', got: ${value}`); + process.exit(2); + } + mode = value; + i += 1; + } else { + console.error(`unknown argument: ${arg}`); + process.exit(2); + } + } + return { tag, mode }; +} + +/** Run a nix build and return its realised out-paths in argument order. Exits on + * failure or on an out-path count that does not match what was asked for, so a + * drifted attr set is a named, fail-closed abort rather than a silently + * incomplete image. */ +function nixBuild( + args: readonly string[], + expected: number, + cwd: string, +): string[] { + const result = spawnSync( + "nix", + ["build", "--no-link", "--print-out-paths", ...args], + { + cwd, + encoding: "utf8", + stdio: ["ignore", "pipe", "inherit"], + }, + ); + if (result.status !== 0) { + console.error(`nix build ${args.join(" ")} failed (exit ${result.status})`); + process.exit(1); + } + const paths = parseOutPaths(result.stdout); + if (paths.length !== expected) { + console.error( + `nix build ${args.join(" ")} produced ${paths.length} out-paths, expected ${expected}`, + ); + process.exit(1); + } + return paths; +} + +/** Restore owner-write on a staged tree so it can be removed. Copying from the + * nix store preserves its read-only directory modes, which would otherwise make + * the stage dir undeletable on the next run. A no-op when the path is absent. */ +function makeWritable(dir: string): void { + if (!existsSync(dir)) return; + chmodSync(dir, 0o755); + for (const entry of readdirSync(dir, { withFileTypes: true })) { + // readdirSync withFileTypes uses lstat semantics, so a symlink already + // reports isDirectory() === false and is never followed here. + if (entry.isDirectory()) { + makeWritable(join(dir, entry.name)); + } + } +} + +const { tag, mode } = parseArgs(process.argv.slice(2)); + +// The platform below is a manifest LABEL; BuildKit applies it without checking +// what the COPY'd files actually are. Today only flake.nix's systems list keeps +// the two honest — a separate file this lane never reads — so assert it here +// rather than inherit an unchecked invariant. +if (process.arch !== "x64") { + console.error( + `runner-image targets ${IMAGE_PLATFORM}, but this host is ${process.arch}.\n` + + " Building here would label the image amd64 while staging this host's binaries.", + ); + process.exit(1); +} + +// buildctl is a CLIENT; it needs a reachable buildkitd. This only checks the +// var is SET, so an unreachable daemon still fails at the build step after the +// realise — it catches the common "forgot to start one" case early, not a dead +// socket. There is deliberately no `docker build` fallback. +const buildkitHost = process.env.BUILDKIT_HOST; +if (buildkitHost === undefined || buildkitHost === "") { + console.error( + "BUILDKIT_HOST is unset — start a buildkitd and point it here.\n" + + " This lane never falls back to `docker build`: rootless BuildKit is the\n" + + " ruled mechanism for a prebuilt-application image, not a preference.", + ); + process.exit(1); +} + +// --------------------------------------------------------------------------- +// 1. Realise every carried artifact. +// --------------------------------------------------------------------------- +// The Runner and the VMM env are flake outputs; the three guest assets come from +// guest-image/default.nix, which is a bare nix file and NOT a flake, so it takes +// the `-f` form (no `#attr` flake-ref exists for it) and runs from guest-image/. +console.error("runner-image: realising closure…"); +const [runner] = nixBuild([".#compass-runner"], 1, workspaceRoot) as [string]; +const [stack] = nixBuild([".#compass-stack-env"], 1, workspaceRoot) as [string]; +const guest = nixBuild( + [ + "-f", + "default.nix", + "compass-guest-kernel", + "compass-guest-rootfs", + "compass-guest-initrd", + ], + 3, + join(workspaceRoot, "guest-image"), +); +const [kernelDir, rootfs, initrd] = guest as [string, string, string]; +const outputs: RunnerImageOutputs = { + runner, + stack, + kernelDir, + rootfs, + initrd, +}; + +// --------------------------------------------------------------------------- +// 2. Stage the transitive closure. +// --------------------------------------------------------------------------- +// `nix path-info -r` over all five roots emits the DEDUPED union, so a path +// shared by several roots (glibc, most obviously) is staged once. The transitive +// set — not just the five roots — is what makes the image runnable: see +// closureRoots' note on the measured "missing dynamic library" control. +console.error("runner-image: staging closure…"); +// Nix store paths are read-only, and a recursive copy preserves that — so a +// previously staged tree cannot be removed until its directories are made +// writable again. Without this, the SECOND run of this lane fails EACCES on its +// own leftovers. +makeWritable(stageDir); +rmSync(stageDir, { recursive: true, force: true }); +rmSync(ociDir, { recursive: true, force: true }); +mkdirSync(stageDir, { recursive: true }); + +const closure = spawnSync( + "nix", + ["path-info", "-r", ...closureRoots(outputs)], + { + cwd: workspaceRoot, + encoding: "utf8", + stdio: ["ignore", "pipe", "inherit"], + }, +); +if (closure.status !== 0) { + console.error(`nix path-info -r failed (exit ${closure.status})`); + process.exit(1); +} +const closurePaths = parseOutPaths(closure.stdout); +for (const path of closurePaths) { + // Each basename is unique by construction (it carries a content hash), and + // the store's symlinks are preserved verbatim so the symlinkJoin'd stack env + // still resolves inside the image. + // + // preserveTimestamps is load-bearing for the DIGEST, not just tidiness. Nix + // normalises every store mtime to 1; without this, cpSync stamps wall-clock + // mtimes into the staged tree, they land in the layer tar, and two builds of + // identical inputs produce different digests. The publish lane asserts a + // stable digest, so this is part of what makes that assertion meaningful. + cpSync(path, join(stageDir, path.replace(/^\/nix\/store\//, "")), { + recursive: true, + verbatimSymlinks: true, + preserveTimestamps: true, + }); +} +console.error(`runner-image: staged ${closurePaths.length} store paths`); + +// The Dockerfile's ENTRYPOINT is exec-form and so cannot expand a build ARG, +// while the Runner's own store path carries a hash that moves on every Go +// rebuild. A stable relative symlink INSIDE the staged tree resolves both: the +// entrypoint is always /nix/store/.compass-runner, and what it points at moves +// with the build. +// +// It must live inside store/ and be RELATIVE. An absolute symlink at the +// context root dangles on the host (nothing is mounted at the target yet), and +// BuildKit checksums context entries before any COPY runs, so it fails the +// build with "not found" rather than deferring to runtime. Relative-and-inside +// keeps the link resolvable in both places. +rmSync(join(stageDir, ".compass-runner"), { force: true }); +symlinkSync( + join(runner.replace(/^\/nix\/store\//, ""), "bin", "compass-runner"), + join(stageDir, ".compass-runner"), +); + +// --------------------------------------------------------------------------- +// 3. Build the image from the staged directory. +// --------------------------------------------------------------------------- +mkdirSync(ociDir, { recursive: true }); +console.error(`runner-image: building ${tag}…`); +const build = spawnSync( + "buildctl", + buildctlArgs( + imageDir, + outputs, + IMAGE_PLATFORM, + outputSpec(mode, tag, ociDir), + ), + { cwd: workspaceRoot, stdio: "inherit" }, +); +process.exit(build.status ?? 1); diff --git a/tools/runner-image/moon.yml b/tools/runner-image/moon.yml new file mode 100644 index 000000000..8e6826d14 --- /dev/null +++ b/tools/runner-image/moon.yml @@ -0,0 +1,31 @@ +# yaml-language-server: $schema=https://moonrepo.dev/schemas/project.json +# +# runner-image (R1): the build lane for the compass-runner container image. +# Realises the nix closure, stages it, and drives the rootless-BuildKit build. +# TypeScript rather than a shell script (the no-bash-gate CI task); +# runner-image/ carries the Dockerfile and the staged context. +# +# The BUILD itself needs a reachable buildkitd and realises a multi-GB nix +# closure, so it is not a CI gate task (see runner-image/moon.yml for the build +# task and why it is runInCI:false). The typecheck/test tasks here ARE ordinary +# bun gates that ride CI: they cover the pure core — the out-path parsing, the +# build-arg mapping, and the closure-root selection whose drift would silently +# produce an unrunnable image. +# +# A bun/TypeScript CLI, hoisted root-workspace member (`bun` tag): install is +# inherited via .moon/tasks/tag-bun.yml and lint/format are whole-repo root +# tasks, so this leaf carries no own bun.lock. +layer: 'tool' +language: 'typescript' +tags: ['bun', 'ci-group.bun'] + +tasks: + typecheck: + command: 'bunx tsc --noEmit' + deps: ['install'] + inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock'] + test: + # The pure core (build-core.ts) over fixtures: the out-path parsing, the + # artifact→build-arg mapping (incl. the kernel's bzImage suffix, which the + # file-shaped rootfs/initrd must NOT gain), and the closure-root selection. + inputs: ['*.ts', 'tsconfig.json', '/tsconfig.base.json', 'package.json', '/bun.lock'] diff --git a/tools/runner-image/package.json b/tools/runner-image/package.json new file mode 100644 index 000000000..33cc0cc07 --- /dev/null +++ b/tools/runner-image/package.json @@ -0,0 +1,14 @@ +{ + "name": "@compass/runner-image", + "private": true, + "type": "module", + "description": "Build the compass-runner container image (R1): realise the nix closure, stage it into a build context, and build a Dockerfile on a minimal hardened base with rootless BuildKit. Nix is the builder, never the runtime.", + "module": "build.ts", + "bin": { + "runner-image-build": "./build.ts" + }, + "devDependencies": { + "@types/bun": "catalog:", + "typescript": "catalog:" + } +} diff --git a/tools/runner-image/tsconfig.json b/tools/runner-image/tsconfig.json new file mode 100644 index 000000000..d40cc9e50 --- /dev/null +++ b/tools/runner-image/tsconfig.json @@ -0,0 +1,11 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022"], + "moduleDetection": "force", + "allowJs": true, + "allowImportingTsExtensions": true, + "noUncheckedIndexedAccess": true, + "types": ["bun"] + } +}