diff --git a/docs/cloud-sandbox-considerations.md b/docs/cloud-sandbox-considerations.md index d6e12225788..73777baf5c8 100644 --- a/docs/cloud-sandbox-considerations.md +++ b/docs/cloud-sandbox-considerations.md @@ -197,20 +197,21 @@ reach every tool an agent runs; and it reaps the largest least-protected process when `MemAvailable` drops under 8% of the box, so the agent sees an exit 137 from its tool instead of the box dying. Boot also writes `SUPERSET_SANDBOX_MEMORY_MB` and `SUPERSET_SANDBOX_VCPUS` into every login -shell from the cgroup, so a hook or an agent can size its work. **Swap is on demand -(2026-10-22):** memguard adds 2 GB swapfiles under `/var/swap` when free memory -drops below twice its reap floor, up to 1.5x the box (at least 6 GB, at most -16 GB), at `vm.swappiness=10`, and drops the newest one after ten quiet -minutes if RAM can take its pages back; the cgroup permits it -(`memory.swap.max` is unbounded). A box that never needs swap holds none on -disk, so no snapshot carries it. Measured on a 2/4 box: the host-service -typecheck the kernel had killed at 3 GB finished in 22 s with 855 MB swapped; -the `apps/desktop` typecheck, which needs about 8 GB in all, grew swap -0 → 6 GB and finished in 86 s with host-service's health check never above -47 ms, then swap shrank back to nothing. A GC keeps its hot set in RAM, so a -swapped heap is slow, not a stall. Swap is never on without memguard: with a -fixed 2 GB swapfile and no memguard, the desktop typecheck stalled -host-service past 5 s and was killed by the kernel after 233 s. Still open: automatic resume of an agent whose own process was +shell from the cgroup, so a hook or an agent can size its work. **No swap +(2026-11-16):** from 2026-10-22 memguard grew 2 GB swapfiles under `/var/swap` +on demand, up to 1.5x the box (16 GB at most). That let an undersized box +finish a large typecheck slowly instead of losing it, but a snapshot is a +full block copy of the disk with no discard, so swap written once stays in +every later snapshot of the workspace: one box held 8 GB of it, and fleet +snapshots of 14-27 GB exist. memguard now reaps when `MemAvailable` drops +under the floor, with no swap first, and removes any `/var/swap/superset-*` +an earlier bundle left (that frees the space for reuse but does not shrink a +snapshot that already holds it); boot still removes the older fixed +`/swapfile`. Expected, not yet measured: on a 2/4 box the `apps/desktop` +typecheck (about 8 GB) is reaped with a `reap` line in `memguard.log`; the +remedy is a larger machine size. Open: how +many boxes ever grew swap is unknown, as `memguard.log` is box-local. +Still open: automatic resume of an agent whose own process was reaped while its terminal survived. ## Untested behaviour diff --git a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-memguard b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-memguard index fca19a05138..4a4982458b7 100755 --- a/packages/sandbox/bundle/rootfs/usr/local/bin/superset-memguard +++ b/packages/sandbox/bundle/rootfs/usr/local/bin/superset-memguard @@ -12,14 +12,13 @@ # its window manager and stream), gone until the next wake # 500 everything else, the platform's default: what an agent or a # hook ran, and the first thing to go -# swap below twice the floor, add a 2 GB swapfile, up to 1.5x the box -# (at least 6 GB, at most 16 GB); after ten quiet minutes, drop the -# newest one if RAM can take its pages back. Nothing is on disk until -# a box needs it, and swap never exists without this script. -# reap below the floor with swap at its cap, kill the largest process in the least protected -# class. Without swap the kernel drops code pages before it kills -# anything, and host-service's event loop stalls for minutes while -# every process refaults; the kernel's killer is the last resort. +# reap below the floor, kill the largest process in the least protected +# class. The box has no swap, so the kernel drops code pages before it +# kills anything, and host-service's event loop stalls for minutes +# while every process refaults; the kernel's killer is the last resort. +# +# No swap: a snapshot copies every block of the disk, so swap written once +# stays in every later snapshot of the workspace. # # One line per action in memguard.log. Fail-open: a box without this script # behaves as before. @@ -30,9 +29,6 @@ FLOOR_PERCENT="${SUPERSET_MEMGUARD_FLOOR_PERCENT:-8}" INTERVAL_S=1 COOLDOWN_S=3 MIN_VICTIM_KB=32768 -SWAP_DIR=/var/swap -SWAP_STEP_MB=2048 -SWAP_IDLE_S="${SUPERSET_MEMGUARD_SWAP_IDLE_S:-600}" exec 9> /run/superset-memguard.lock flock -n 9 || exit 0 @@ -95,68 +91,24 @@ victim_at_or_above() { [ -n "$best" ] && echo "$best" } -mem_total=0 mem_avail=0 swap_total=0 swap_free=0 +mem_total=0 mem_avail=0 read_meminfo() { local key val _ while read -r key val _; do case "$key" in MemTotal:) mem_total="$val" ;; MemAvailable:) mem_avail="$val" ;; - SwapTotal:) swap_total="$val" ;; SwapFree:) swap_free="$val" ;; esac done < /proc/meminfo } -box_mb() { - local max - max="$(cat /sys/fs/cgroup/memory.max 2>/dev/null || true)" - case "$max" in ''|max) echo $(( $(awk '/MemTotal/ {print $2}' /proc/meminfo) / 1024 )) ;; *) echo $(( max / 1048576 )) ;; esac -} -swap_cap_mb=$(( $(box_mb) * 3 / 2 )) -[ "$swap_cap_mb" -lt 6144 ] && swap_cap_mb=6144 -[ "$swap_cap_mb" -gt 16384 ] && swap_cap_mb=16384 - -swap_files=() -mkdir -p "$SWAP_DIR" -for f in "$SWAP_DIR"/superset-*; do [ -e "$f" ] && { swapoff "$f" 2>/dev/null; rm -f "$f"; }; done - -grow_swap() { - local f="${SWAP_DIR}/superset-${#swap_files[@]}" - [ $(( (${#swap_files[@]} + 1) * SWAP_STEP_MB )) -le "$swap_cap_mb" ] || return 1 - if fallocate -l "${SWAP_STEP_MB}M" "$f" && chmod 600 "$f" && mkswap -q "$f" >/dev/null && swapon "$f"; then - swap_files+=("$f") - sysctl -q -w vm.swappiness=10 - log "swap.grow ${f} total=$(( ${#swap_files[@]} * SWAP_STEP_MB ))MB cap=${swap_cap_mb}MB available=$(( mem_avail / 1024 ))MB" - return 0 - fi - rm -f "$f"; log "swap.grow.failed ${f}"; return 1 -} +# Swapfiles an earlier bundle grew on demand. +for f in /var/swap/superset-*; do [ -e "$f" ] && { swapoff "$f" 2>/dev/null; rm -f "$f"; }; done -shrink_swap() { - local n=${#swap_files[@]} f - [ "$n" -gt 0 ] || return 0 - f="${swap_files[$(( n - 1 ))]}" - [ $(( swap_total - swap_free + floor * 2 )) -lt "$mem_avail" ] || return 0 - if swapoff "$f"; then - rm -f "$f"; unset 'swap_files[$(( n - 1 ))]' - log "swap.shrink ${f} total=$(( ${#swap_files[@]} * SWAP_STEP_MB ))MB" - fi -} - -last_pressure=$EPOCHSECONDS while true; do scan read_meminfo floor=$(( mem_total * FLOOR_PERCENT / 100 )) - if [ "$mem_avail" -lt $(( floor * 2 )) ]; then - last_pressure=$EPOCHSECONDS - if [ "$swap_free" -lt $(( SWAP_STEP_MB * 1024 )) ] && grow_swap; then - sleep "$INTERVAL_S"; continue - fi - elif [ $(( EPOCHSECONDS - last_pressure )) -ge "$SWAP_IDLE_S" ] && [ ${#swap_files[@]} -gt 0 ]; then - shrink_swap - last_pressure=$EPOCHSECONDS - fi - if [ "$mem_avail" -lt "$floor" ] && [ "$swap_free" -lt $(( swap_total / 10 + 1 )) ]; then + if [ "$mem_avail" -lt "$floor" ]; then victim="$(victim_at_or_above 500 || victim_at_or_above 0 || victim_at_or_above -300)" if [ -n "$victim" ]; then log "reap pid=${victim} ${CLASS[$victim]} rss=$(( RSS_KB[$victim] / 1024 ))MB adj=${ADJ[$victim]} available=$(( mem_avail / 1024 ))MB floor=$(( floor / 1024 ))MB" diff --git a/packages/sandbox/src/boot.test.ts b/packages/sandbox/src/boot.test.ts index cf49be58f6a..1e910eeb38c 100644 --- a/packages/sandbox/src/boot.test.ts +++ b/packages/sandbox/src/boot.test.ts @@ -36,13 +36,19 @@ describe("superset-boot", () => { ); }); - test("swap is memguard's, grown on demand up to a cap by box size", () => { + test("nothing turns swap on, and an earlier bundle's swapfiles are removed", () => { expect(boot).not.toMatch(/swapon/); - expect(memguard).toMatch(/swap_cap_mb=\$\(\( \$\(box_mb\) \* 3 \/ 2 \)\)/); - expect(memguard).toMatch(/-lt 6144 \] && swap_cap_mb=6144/); - expect(memguard).toMatch(/-gt 16384 \] && swap_cap_mb=16384/); - expect(memguard).toMatch(/swapon "\$f"/); - expect(memguard).toMatch(/swapoff "\$f"/); + expect(memguard).not.toMatch(/swapon|mkswap|fallocate/); + expect(memguard).toMatch( + /\/var\/swap\/superset-\*.*swapoff "\$f".*rm -f "\$f"/, + ); + expect(boot).toMatch(/rm -f \/swapfile/); + }); + + test("memguard reaps on available memory alone", () => { + expect(memguard).toMatch( + /if \[ "\$mem_avail" -lt "\$floor" \]; then\n\s*victim=/, + ); }); test("writes the cgroup's memory ceiling and CPU count for login shells", () => {