From 17185974e840873ae3afc49bb27d43b868cc0545 Mon Sep 17 00:00:00 2001 From: Satya Patel <14907857+saddlepaddle@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:33:11 +0000 Subject: [PATCH] fix(sandbox): drop on-demand swap from memguard A snapshot is a full block copy of the disk with no discard, so swap memguard writes stays in every later snapshot of the workspace. memguard now reaps on MemAvailable below the floor alone, and still removes any /var/swap/superset-* an earlier bundle left. --- docs/cloud-sandbox-considerations.md | 29 ++++---- .../rootfs/usr/local/bin/superset-memguard | 70 +++---------------- packages/sandbox/src/boot.test.ts | 18 +++-- 3 files changed, 38 insertions(+), 79 deletions(-) diff --git a/docs/cloud-sandbox-considerations.md b/docs/cloud-sandbox-considerations.md index d6e12225788..73777baf5c8 100644 --- a/docs/cloud-sandbox-considerations.md +++ b/docs/cloud-sandbox-considerations.md @@ -197,20 +197,21 @@ reach every tool an agent runs; and it reaps the largest least-protected process when `MemAvailable` drops under 8% of the box, so the agent sees an exit 137 from its tool instead of the box dying. Boot also writes `SUPERSET_SANDBOX_MEMORY_MB` and `SUPERSET_SANDBOX_VCPUS` into every login -shell from the cgroup, so a hook or an agent can size its work. **Swap is on demand -(2026-10-22):** memguard adds 2 GB swapfiles under `/var/swap` when free memory -drops below twice its reap floor, up to 1.5x the box (at least 6 GB, at most -16 GB), at `vm.swappiness=10`, and drops the newest one after ten quiet -minutes if RAM can take its pages back; the cgroup permits it -(`memory.swap.max` is unbounded). A box that never needs swap holds none on -disk, so no snapshot carries it. Measured on a 2/4 box: the host-service -typecheck the kernel had killed at 3 GB finished in 22 s with 855 MB swapped; -the `apps/desktop` typecheck, which needs about 8 GB in all, grew swap -0 → 6 GB and finished in 86 s with host-service's health check never above -47 ms, then swap shrank back to nothing. A GC keeps its hot set in RAM, so a -swapped heap is slow, not a stall. Swap is never on without memguard: with a -fixed 2 GB swapfile and no memguard, the desktop typecheck stalled -host-service past 5 s and was killed by the kernel after 233 s. Still open: automatic resume of an agent whose own process was +shell from the cgroup, so a hook or an agent can size its work. **No swap +(2026-11-16):** from 2026-10-22 memguard grew 2 GB swapfiles under `/var/swap` +on demand, up to 1.5x the box (16 GB at most). That let an undersized box +finish a large typecheck slowly instead of losing it, but a snapshot is a +full block copy of the disk with no discard, so swap written once stays in +every later snapshot of the workspace: one box held 8 GB of it, and fleet +snapshots of 14-27 GB exist. memguard now reaps when `MemAvailable` drops +under the floor, with no swap first, and removes any `/var/swap/superset-*` +an earlier bundle left (that frees the space for reuse but does not shrink a +snapshot that already holds it); boot still removes the older fixed +`/swapfile`. Expected, not yet measured: on a 2/4 box the `apps/desktop` +typecheck (about 8 GB) is reaped with a `reap` line in `memguard.log`; the +remedy is a larger machine size. Open: how +many boxes ever grew swap is unknown, as `memguard.log` is box-local. +Still open: automatic resume of an agent whose own process was reaped while its terminal survived. ## Untested behaviour diff --git a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-memguard b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-memguard index fca19a05138..4a4982458b7 100755 --- a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-memguard +++ b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-memguard @@ -12,14 +12,13 @@ # its window manager and stream), gone until the next wake # 500 everything else, the platform's default: what an agent or a # hook ran, and the first thing to go -# swap below twice the floor, add a 2 GB swapfile, up to 1.5x the box -# (at least 6 GB, at most 16 GB); after ten quiet minutes, drop the -# newest one if RAM can take its pages back. Nothing is on disk until -# a box needs it, and swap never exists without this script. -# reap below the floor with swap at its cap, kill the largest process in the least protected -# class. Without swap the kernel drops code pages before it kills -# anything, and host-service's event loop stalls for minutes while -# every process refaults; the kernel's killer is the last resort. +# reap below the floor, kill the largest process in the least protected +# class. The box has no swap, so the kernel drops code pages before it +# kills anything, and host-service's event loop stalls for minutes +# while every process refaults; the kernel's killer is the last resort. +# +# No swap: a snapshot copies every block of the disk, so swap written once +# stays in every later snapshot of the workspace. # # One line per action in memguard.log. Fail-open: a box without this script # behaves as before. @@ -30,9 +29,6 @@ FLOOR_PERCENT="${SUPERSET_MEMGUARD_FLOOR_PERCENT:-8}" INTERVAL_S=1 COOLDOWN_S=3 MIN_VICTIM_KB=32768 -SWAP_DIR=/var/swap -SWAP_STEP_MB=2048 -SWAP_IDLE_S="${SUPERSET_MEMGUARD_SWAP_IDLE_S:-600}" exec 9> /run/superset-memguard.lock flock -n 9 || exit 0 @@ -95,68 +91,24 @@ victim_at_or_above() { [ -n "$best" ] && echo "$best" } -mem_total=0 mem_avail=0 swap_total=0 swap_free=0 +mem_total=0 mem_avail=0 read_meminfo() { local key val _ while read -r key val _; do case "$key" in MemTotal:) mem_total="$val" ;; MemAvailable:) mem_avail="$val" ;; - SwapTotal:) swap_total="$val" ;; SwapFree:) swap_free="$val" ;; esac done < /proc/meminfo } -box_mb() { - local max - max="$(cat /sys/fs/cgroup/memory.max 2>/dev/null || true)" - case "$max" in ''|max) echo $(( $(awk '/MemTotal/ {print $2}' /proc/meminfo) / 1024 )) ;; *) echo $(( max / 1048576 )) ;; esac -} -swap_cap_mb=$(( $(box_mb) * 3 / 2 )) -[ "$swap_cap_mb" -lt 6144 ] && swap_cap_mb=6144 -[ "$swap_cap_mb" -gt 16384 ] && swap_cap_mb=16384 - -swap_files=() -mkdir -p "$SWAP_DIR" -for f in "$SWAP_DIR"/superset-*; do [ -e "$f" ] && { swapoff "$f" 2>/dev/null; rm -f "$f"; }; done - -grow_swap() { - local f="${SWAP_DIR}/superset-${#swap_files[@]}" - [ $(( (${#swap_files[@]} + 1) * SWAP_STEP_MB )) -le "$swap_cap_mb" ] || return 1 - if fallocate -l "${SWAP_STEP_MB}M" "$f" && chmod 600 "$f" && mkswap -q "$f" >/dev/null && swapon "$f"; then - swap_files+=("$f") - sysctl -q -w vm.swappiness=10 - log "swap.grow ${f} total=$(( ${#swap_files[@]} * SWAP_STEP_MB ))MB cap=${swap_cap_mb}MB available=$(( mem_avail / 1024 ))MB" - return 0 - fi - rm -f "$f"; log "swap.grow.failed ${f}"; return 1 -} +# Swapfiles an earlier bundle grew on demand. +for f in /var/swap/superset-*; do [ -e "$f" ] && { swapoff "$f" 2>/dev/null; rm -f "$f"; }; done -shrink_swap() { - local n=${#swap_files[@]} f - [ "$n" -gt 0 ] || return 0 - f="${swap_files[$(( n - 1 ))]}" - [ $(( swap_total - swap_free + floor * 2 )) -lt "$mem_avail" ] || return 0 - if swapoff "$f"; then - rm -f "$f"; unset 'swap_files[$(( n - 1 ))]' - log "swap.shrink ${f} total=$(( ${#swap_files[@]} * SWAP_STEP_MB ))MB" - fi -} - -last_pressure=$EPOCHSECONDS while true; do scan read_meminfo floor=$(( mem_total * FLOOR_PERCENT / 100 )) - if [ "$mem_avail" -lt $(( floor * 2 )) ]; then - last_pressure=$EPOCHSECONDS - if [ "$swap_free" -lt $(( SWAP_STEP_MB * 1024 )) ] && grow_swap; then - sleep "$INTERVAL_S"; continue - fi - elif [ $(( EPOCHSECONDS - last_pressure )) -ge "$SWAP_IDLE_S" ] && [ ${#swap_files[@]} -gt 0 ]; then - shrink_swap - last_pressure=$EPOCHSECONDS - fi - if [ "$mem_avail" -lt "$floor" ] && [ "$swap_free" -lt $(( swap_total / 10 + 1 )) ]; then + if [ "$mem_avail" -lt "$floor" ]; then victim="$(victim_at_or_above 500 || victim_at_or_above 0 || victim_at_or_above -300)" if [ -n "$victim" ]; then log "reap pid=${victim} ${CLASS[$victim]} rss=$(( RSS_KB[$victim] / 1024 ))MB adj=${ADJ[$victim]} available=$(( mem_avail / 1024 ))MB floor=$(( floor / 1024 ))MB" diff --git a/packages/sandbox/src/boot.test.ts b/packages/sandbox/src/boot.test.ts index cf49be58f6a..1e910eeb38c 100644 --- a/packages/sandbox/src/boot.test.ts +++ b/packages/sandbox/src/boot.test.ts @@ -36,13 +36,19 @@ describe("superset-boot", () => { ); }); - test("swap is memguard's, grown on demand up to a cap by box size", () => { + test("nothing turns swap on, and an earlier bundle's swapfiles are removed", () => { expect(boot).not.toMatch(/swapon/); - expect(memguard).toMatch(/swap_cap_mb=\$\(\( \$\(box_mb\) \* 3 \/ 2 \)\)/); - expect(memguard).toMatch(/-lt 6144 \] && swap_cap_mb=6144/); - expect(memguard).toMatch(/-gt 16384 \] && swap_cap_mb=16384/); - expect(memguard).toMatch(/swapon "\$f"/); - expect(memguard).toMatch(/swapoff "\$f"/); + expect(memguard).not.toMatch(/swapon|mkswap|fallocate/); + expect(memguard).toMatch( + /\/var\/swap\/superset-\*.*swapoff "\$f".*rm -f "\$f"/, + ); + expect(boot).toMatch(/rm -f \/swapfile/); + }); + + test("memguard reaps on available memory alone", () => { + expect(memguard).toMatch( + /if \[ "\$mem_avail" -lt "\$floor" \]; then\n\s*victim=/, + ); }); test("writes the cgroup's memory ceiling and CPU count for login shells", () => {