From bb035b01755b8c5af8b90870c630347c12b8d02d Mon Sep 17 00:00:00 2001 From: Flegma Date: Fri, 30 Oct 2026 12:20:43 +0100 Subject: [PATCH 01/10] feature: GPU node and Proxmox LXC check scripts gpu-node/gpu-node-check.sh checks a GPU node (driver, firmware, container runtime, CPU governor, disk, k3s version) and gpu-node/proxmox-gpu-lxc.sh checks a Proxmox host and its LXC container. Both only report unless --apply is given. The README walks through dedicated machines, VMs and Proxmox LXC containers for NVIDIA and AMD, one vendor per panel as setup-gpu-nodes.sh sets it up. --- .github/workflows/panel-scripts.yml | 5 +- gpu-node/README.md | 120 +++++ gpu-node/gpu-node-check.sh | 655 ++++++++++++++++++++++++++ gpu-node/proxmox-gpu-lxc.sh | 504 ++++++++++++++++++++ tests/scripts/proxmox_gpu_lxc_test.sh | 106 +++++ 5 files changed, 1389 insertions(+), 1 deletion(-) create mode 100644 gpu-node/README.md create mode 100755 gpu-node/gpu-node-check.sh create mode 100755 gpu-node/proxmox-gpu-lxc.sh create mode 100644 tests/scripts/proxmox_gpu_lxc_test.sh diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml index d3090c6..fe47220 100644 --- a/.github/workflows/panel-scripts.yml +++ b/.github/workflows/panel-scripts.yml @@ -8,6 +8,7 @@ on: - dev.sh - plugin.sh - setup-gpu-nodes.sh + - "gpu-node/**" - update.sh - Tiltfile - "utils/**" @@ -24,6 +25,7 @@ on: - dev.sh - plugin.sh - setup-gpu-nodes.sh + - "gpu-node/**" - update.sh - Tiltfile - "utils/**" @@ -40,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: shellcheck -x utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh tests/scripts/*.sh tests/scripts/stub/kubectl + - run: shellcheck -x utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh gpu-node/*.sh tests/scripts/*.sh tests/scripts/stub/kubectl - run: bash tests/scripts/setup_env_test.sh - run: bash tests/scripts/debug_test.sh - run: bash tests/scripts/startup_probe_test.sh @@ -51,3 +53,4 @@ jobs: - run: bash tests/scripts/write_overlays_test.sh - run: bash tests/scripts/dev_game_server_test.sh - run: bash tests/scripts/gpu_nodes_test.sh + - run: bash tests/scripts/proxmox_gpu_lxc_test.sh diff --git a/gpu-node/README.md b/gpu-node/README.md new file mode 100644 index 0000000..2cfe422 --- /dev/null +++ b/gpu-node/README.md @@ -0,0 +1,120 @@ +# GPU nodes + +A GPU node renders demos, highlights and clips. It can be a dedicated machine, +a VM with the GPU passed through, or a Proxmox LXC container. NVIDIA and AMD +GPUs are both supported, one vendor per panel (`GPU_VENDOR`). + +| Script | Runs on | Does | +| --- | --- | --- | +| `gpu-node-check.sh` | the GPU node | checks the driver, firmware, container runtime, CPU governor, disk and k3s version; `--apply` fixes what it safely can | +| `proxmox-gpu-lxc.sh CTID` | the Proxmox host | checks the host driver and kernel modules and the container's settings; `--apply` writes them | +| `setup-gpu-nodes.sh` | the panel | labels the GPU nodes for the panel's vendor and turns that vendor's support on | + +Every script only reports unless you pass `--apply`, and exits 1 while +something still fails. None of them change the GPU's power or clock settings. + +## Before you start + +- **k3s version.** The node setup command installs the latest k3s, and a + kubelet newer than the panel's API server is unsupported. If the panel runs + an older k3s, read its version with `kubectl version` on the panel + (`Server Version`) and export it on the node before the setup command: + + ```bash + export INSTALL_K3S_VERSION=v1.36.5+k3s1 + ``` + +- **Disk.** The node setup command wants 120 GB free on `/`. +- **CPU governor.** Render pods drop frames while the CPU ramps up, so the + governor is set to `performance`. In an LXC container that happens on the + Proxmox host. + +## Dedicated machine or VM + +1. Install Ubuntu 24.04 or newer, server or desktop. A headless install is + recommended: a desktop session keeps hold of the GPU's display. +2. Run the check, then let it fix what it can and reboot if it asks: + + ```bash + sudo bash gpu-node-check.sh + sudo bash gpu-node-check.sh --apply --k3s-version v1.36.5+k3s1 + ``` + +3. Export `INSTALL_K3S_VERSION`, run the node setup command from the panel + (Settings, Nodes), and run the check again: some checks only apply once k3s + is there. +4. On the panel, run `./setup-gpu-nodes.sh` and pick the vendor and nodes. + All GPU nodes of a panel use the same vendor. + +### NVIDIA + +- `--apply` installs the recommended driver (`ubuntu-drivers install`) and the + NVIDIA container toolkit, and restarts k3s-agent if it started before the + toolkit was there. +- With Secure Boot on, use Ubuntu's signed prebuilt driver modules + (`ubuntu-drivers install` picks them) or enroll the DKMS key with `mokutil`. +- After a driver upgrade, `nvidia-smi` fails until the next reboot. + +### AMD + +- `--apply` installs the AMD firmware package. +- On Ubuntu 24.04, use the HWE kernel (`linux-generic-hwe-24.04`); the original + 6.8 kernel predates recent Radeons. +- Never boot with `nomodeset`, `amdgpu.modeset=0` or `amdgpu.dpm=0`. +- For a discrete card: enable Above 4G decoding and Resizable BAR in the + firmware, and add `amdgpu.runpm=0` to the kernel command line so a headless + card cannot suspend off the bus. +- For an APU (such as Ryzen AI Max), select the node in the integrated list in + `setup-gpu-nodes.sh`. The check says when a GPU is integrated. + +## Proxmox LXC container + +k3s needs a **privileged** container. The host keeps the GPU driver and shares +the device with the container; do not bind the GPU to `vfio-pci`. + +1. On the host, install the GPU driver (NVIDIA: the proxmox headers and + NVIDIA's `.run` installer; AMD: the kernel driver is built in, firmware comes + from `pve-firmware`). +2. Create a privileged Ubuntu container with at least 120 GB of disk and no core + limit (the kubelet pins pods to CPUs by number and sees every host CPU). +3. On the host: + + ```bash + bash proxmox-gpu-lxc.sh 120 + bash proxmox-gpu-lxc.sh 120 --apply + pct reboot 120 + ``` + + `--apply` loads the kernel modules k3s needs (and at every boot), sets the CPU + governor, creates the NVIDIA device nodes at boot, turns on nesting, and adds + these lines to `/etc/pve/lxc/120.conf` (after backing it up to `/root`): + + ``` + lxc.apparmor.profile: unconfined + lxc.cgroup2.devices.allow: a + lxc.cap.drop: + lxc.mount.auto: proc:rw sys:rw + lxc.mount.entry: /dev/kmsg dev/kmsg none bind,optional,create=file + lxc.mount.entry: /dev/net/tun dev/net/tun none bind,optional,create=file + # AMD + lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir + lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file + # NVIDIA: /dev/nvidia0, nvidiactl, nvidia-uvm, nvidia-uvm-tools, + # nvidia-modeset and nvidia-caps, the same way + ``` + +4. Inside the container, run `gpu-node-check.sh --apply`. For NVIDIA it + installs the host driver's exact version of the NVIDIA libraries, without + the kernel module, plus the container toolkit. +5. Continue with step 3 of the dedicated machine steps. + +If NVIDIA pods fail to start in the container with a device cgroup error, set +`no-cgroups = true` under `[nvidia-container-cli]` in +`/etc/nvidia-container-runtime/config.toml` and restart k3s-agent. Leave it at +the default otherwise: the NVIDIA device plugin runs unprivileged and relies +on it. + +## Reporting a problem + +Include the output of `gpu-node-check.sh`, `proxmox-gpu-lxc.sh` for a container, +and `journalctl -k -b | grep -iE "amdgpu|nvidia|nvrm"`. diff --git a/gpu-node/gpu-node-check.sh b/gpu-node/gpu-node-check.sh new file mode 100755 index 0000000..6e450fd --- /dev/null +++ b/gpu-node/gpu-node-check.sh @@ -0,0 +1,655 @@ +#!/bin/bash + +# Checks that a machine can run 5stack GPU render pods, and with --apply fixes +# what it safely can. Run it as root on the GPU node itself: a dedicated +# machine, a VM with the GPU passed through, or a Proxmox LXC container (run +# gpu-node/proxmox-gpu-lxc.sh on the Proxmox host first). +# +# Run it before the panel's node setup command, and again after, since some +# checks (the k3s container runtime) only mean something once k3s is there. +# +# gpu-node-check.sh [--apply] [--vendor nvidia|amd] [--k3s-version v1.36.5+k3s1] +# +# Exit code: 0 when nothing failed, 1 when a check failed, 2 on bad usage. +# +# It never touches the GPU's power or clock settings, the desktop session or +# Tailscale, and it never installs a kernel driver inside a container. + +APPLY=false +VENDOR="" +K3S_VERSION="" + +usage() { + echo "usage: gpu-node-check.sh [--apply] [--vendor nvidia|amd] [--k3s-version v1.36.5+k3s1]" >&2 + exit 2 +} + +while [ $# -gt 0 ]; do + case "$1" in + --apply) APPLY=true ;; + --vendor) + VENDOR="$2" + shift + ;; + --k3s-version) + K3S_VERSION="$2" + shift + ;; + -h | --help) usage ;; + *) + echo "unknown option: $1" >&2 + usage + ;; + esac + shift +done + +case "$VENDOR" in + "" | nvidia | amd) ;; + *) + echo "--vendor must be nvidia or amd" >&2 + exit 2 + ;; +esac + +if [ -t 1 ]; then + C_RESET=$'\033[0m' C_STEP=$'\033[1;36m' C_OK=$'\033[0;32m' C_WARN=$'\033[1;33m' C_ERR=$'\033[0;31m' +else + C_RESET='' C_STEP='' C_OK='' C_WARN='' C_ERR='' +fi + +FAILS=0 +WARNS=0 +step() { echo; echo "${C_STEP}==> $1${C_RESET}"; } +pass() { echo "${C_OK} [PASS] $1${C_RESET}"; } +info() { echo " [INFO] $1"; } +warn() { WARNS=$((WARNS + 1)); echo "${C_WARN} [WARN] $1${C_RESET}"; } +fail() { FAILS=$((FAILS + 1)); echo "${C_ERR} [FAIL] $1${C_RESET}"; } +fixed() { echo "${C_OK} [FIX ] $1${C_RESET}"; } +hint() { echo " $1"; } + +if [ "$APPLY" = true ] && [ "$(id -u)" -ne 0 ]; then + echo "--apply must run as root" >&2 + exit 2 +fi + +# shellcheck source=/dev/null +. /etc/os-release 2>/dev/null +OS_ID="${ID:-unknown}" +OS_VERSION="${VERSION_ID:-}" + +apt_has() { + command -v apt-cache >/dev/null 2>&1 && + apt-cache policy "$1" 2>/dev/null | grep -q 'Candidate: [^(]' +} + +pkg_installed() { + dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q 'install ok installed' +} + +# Package installs never prompt: under "curl ... | bash" a prompt would read +# the rest of this script as its answer. +APT_UPDATED=false +apt_install() { + if [ "$APT_UPDATED" = false ]; then + DEBIAN_FRONTEND=noninteractive apt-get update /dev/null || return 1 + APT_UPDATED=true + fi + DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold "$@" /dev/null +} + +# apply_step DESCRIPTION COMMAND... +# Runs an --apply step and reports it as fixed or failed. +apply_step() { + local description="$1" + shift + if "$@"; then + fixed "$description" + else + fail "could not apply: $description" + fi +} + +# The kernel log, when this machine may read it. A container usually may not. +kernel_log() { + dmesg 2>/dev/null || journalctl -k -b 0 --no-pager 2>/dev/null +} + +step "Machine" + +CONTAINER=$(systemd-detect-virt -c 2>/dev/null || true) +VM=$(systemd-detect-virt -v 2>/dev/null || true) +IN_LXC=false +ON_HOST="" +case "$CONTAINER" in + lxc*) + IN_LXC=true + ON_HOST=" on the host" + ;; + none | "") ;; + *) + fail "running in a $CONTAINER container; only bare metal, VMs and LXC are supported" + ;; +esac + +if [ "$IN_LXC" = true ]; then + info "LXC container on kernel $(uname -r); the host owns the GPU driver, firmware and CPU governor" +elif [ -n "$VM" ] && [ "$VM" != none ]; then + info "$VM VM on kernel $(uname -r); the GPU must be passed through to it" +else + info "bare metal, kernel $(uname -r)" +fi +info "$OS_ID $OS_VERSION" + +# GPUs, from sysfs, which a container sees too. +NVIDIA_GPUS=() +AMD_GPUS=() +for dev in /sys/bus/pci/devices/*; do + class=$(cat "$dev/class" 2>/dev/null) + case "$class" in + 0x0300* | 0x0302* | 0x0380*) ;; + *) continue ;; + esac + case "$(cat "$dev/vendor" 2>/dev/null)" in + 0x10de) NVIDIA_GPUS+=("$dev") ;; + 0x1002) AMD_GPUS+=("$dev") ;; + esac +done + +if [ -z "$VENDOR" ]; then + if [ ${#NVIDIA_GPUS[@]} -gt 0 ] && [ ${#AMD_GPUS[@]} -gt 0 ]; then + fail "found both NVIDIA and AMD GPUs; pass --vendor nvidia or --vendor amd" + VENDOR=none + elif [ ${#NVIDIA_GPUS[@]} -gt 0 ]; then + VENDOR=nvidia + elif [ ${#AMD_GPUS[@]} -gt 0 ]; then + VENDOR=amd + else + fail "no NVIDIA or AMD GPU found on the PCI bus" + VENDOR=none + fi +fi + +driver_of() { + basename "$(readlink "$1/driver" 2>/dev/null)" 2>/dev/null +} + +cmdline_has() { + tr ' ' '\n' /dev/null)" = vddnb ] && return 0 + done + rev=$(od -An -tu1 -j2 -N1 "$dev/gpu_metrics" 2>/dev/null | tr -d ' ') + [ -n "$rev" ] && [ "$rev" -ge 2 ] +} + +check_amd() { + local dev pci driver node render integrated bar0 size level + + step "AMD GPU" + + if [ -d /sys/module/amdgpu ]; then + pass "amdgpu driver loaded" + else + fail "amdgpu driver not loaded" + [ "$IN_LXC" = true ] && hint "Load it on the host; a container cannot." + fi + + for dev in "${AMD_GPUS[@]}"; do + pci=$(basename "$dev") + driver=$(driver_of "$dev") + integrated=false + amd_integrated "$dev" && integrated=true + + if [ "$driver" != amdgpu ]; then + fail "$pci is bound to '${driver:-nothing}', not amdgpu" + [ "$driver" = vfio-pci ] && hint "It is reserved for VM passthrough; an LXC or this host cannot use it." + continue + fi + + render="" + for node in "$dev"/drm/renderD*; do + [ -e "$node" ] && render=$(basename "$node") && break + done + if [ -z "$render" ]; then + fail "$pci has no render node" + continue + fi + if [ -c "/dev/dri/$render" ]; then + pass "$pci: /dev/dri/$render ($([ "$integrated" = true ] && echo integrated || echo discrete))" + else + fail "$pci: /dev/dri/$render is missing" + [ "$IN_LXC" = true ] && hint "Bind /dev/dri into the container (proxmox-gpu-lxc.sh does)." + fi + + if [ "$integrated" = true ]; then + info "$pci is integrated: select this node in setup-gpu-nodes.sh's integrated list (5stack-gpu-integrated=true)" + else + # BAR0 is the CPU's window into VRAM. 256 MiB means Above 4G + # decoding (and Resizable BAR) is off in the firmware. + bar0=$(head -1 "$dev/resource" 2>/dev/null) + size=$(( $(echo "$bar0" | awk '{print $2}') - $(echo "$bar0" | awk '{print $1}') + 1 )) + if [ "$size" -gt 0 ] && [ "$size" -le $((256 * 1024 * 1024)) ]; then + warn "$pci: VRAM window is $((size / 1024 / 1024)) MiB; enable Above 4G decoding and Resizable BAR in the firmware" + fi + if ! cmdline_has amdgpu.runpm=0; then + warn "$pci: a headless discrete GPU can drop off the bus in runtime suspend; add amdgpu.runpm=0 to the kernel command line${ON_HOST:+ of the host}" + fi + fi + + level=$(cat "$dev/power_dpm_force_performance_level" 2>/dev/null) + if [ -n "$level" ] && [ "$level" != auto ]; then + warn "$pci: power_dpm_force_performance_level is '$level'; 5stack expects 'auto' and never changes it" + fi + done + + for arg in nomodeset amdgpu.modeset=0 amdgpu.dpm=0; do + if cmdline_has "$arg"; then + fail "kernel command line has $arg, which disables what the streamer needs; remove it" + fi + done + + if [ -e /dev/kfd ]; then + info "/dev/kfd present (not needed for rendering)" + fi + + local log + log=$(kernel_log) + if [ ! -d /sys/module/amdgpu ]; then + : + elif [ -z "$log" ]; then + info "kernel log not readable here; check firmware errors with 'dmesg | grep amdgpu' on the host" + elif echo "$log" | grep -q "Direct firmware load for amdgpu/.* failed"; then + fail "amdgpu firmware is missing: $(echo "$log" | grep -o "amdgpu/[^ ]*" | head -1)" + hint "Install the AMD firmware package$ON_HOST, then reboot it." + elif echo "$log" | grep -q "amdgpu.*Fatal error during GPU init"; then + fail "amdgpu failed to start the GPU this boot; see 'dmesg | grep amdgpu'" + else + pass "no amdgpu firmware or init errors this boot" + if echo "$log" | grep -qE "amdgpu.*(GPU reset begin|ring .* timeout)"; then + warn "the GPU has reset this boot; see 'dmesg | grep -E \"reset|timeout\"'" + fi + fi + + if [ "$IN_LXC" = false ] && command -v apt-cache >/dev/null 2>&1; then + local firmware="" + for candidate in linux-firmware-amd-graphics firmware-amd-graphics pve-firmware linux-firmware; do + if pkg_installed "$candidate"; then + firmware="$candidate" + break + fi + done + if [ -n "$firmware" ]; then + pass "firmware package $firmware installed" + else + for candidate in linux-firmware-amd-graphics firmware-amd-graphics; do + if apt_has "$candidate"; then + if [ "$APPLY" = true ]; then + apply_step "installed $candidate; reboot to load it" apt_install "$candidate" + else + warn "$candidate is not installed (--apply installs it)" + fi + break + fi + done + fi + fi + + if [ "$IN_LXC" = false ] && [ "$OS_ID" = ubuntu ] && [ "$OS_VERSION" = "24.04" ]; then + case "$(uname -r)" in + 6.8.*) warn "Ubuntu 24.04's original 6.8 kernel predates recent Radeons; install the HWE kernel (linux-generic-hwe-24.04)" ;; + esac + fi + + if [ "$IN_LXC" = false ]; then + local dm + for dm in gdm gdm3 sddm lightdm display-manager; do + if systemctl is-active --quiet "$dm" 2>/dev/null; then + warn "$dm is running; a desktop session holds the GPU's display, so a headless install is recommended" + break + fi + done + fi +} + +# --- NVIDIA ---------------------------------------------------------------- + +# 595.99.02 from "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 595.99.02 Release Build ..." +nvidia_kernel_version() { + head -1 /proc/driver/nvidia/version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1 +} + +install_nvidia_userspace() { + # The container shares the host's kernel module, so it needs the same + # version's libraries and nothing else. The .run installer is the one + # source that has every version, matching the host whatever installed it. + local version="$1" run opt + run="/root/NVIDIA-Linux-x86_64-$version.run" + if [ ! -f "$run" ]; then + command -v curl >/dev/null 2>&1 || apt_install curl || { + fail "could not install curl to download the NVIDIA $version installer" + return 1 + } + # Download beside the target, so an interrupted download is never + # mistaken for the installer on the next run. + if ! curl -fsSL -o "$run.part" "https://us.download.nvidia.com/XFree86/Linux-x86_64/$version/NVIDIA-Linux-x86_64-$version.run" /dev/null | grep -oE -- '--no-kernel-modules?' | head -1) + if sh "$run" --silent "${opt:---no-kernel-module}" /dev/null 2>&1; then + fixed "installed the NVIDIA $version libraries" + else + fail "the NVIDIA $version installer failed; see /var/log/nvidia-installer.log" + return 1 + fi +} + +install_nvidia_container_toolkit() { + if [ "$OS_ID" != ubuntu ] && [ "$OS_ID" != debian ]; then + fail "install nvidia-container-toolkit by hand on $OS_ID: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html" + return 1 + fi + apt_install curl gnupg ca-certificates && + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey /etc/apt/sources.list.d/nvidia-container-toolkit.list || return 1 + # The new repository needs its own package list. + APT_UPDATED=false + apt_install nvidia-container-toolkit +} + +check_nvidia() { + local dev pci driver kernel_version smi_version missing node + + step "NVIDIA GPU" + + for dev in "${NVIDIA_GPUS[@]}"; do + pci=$(basename "$dev") + driver=$(driver_of "$dev") + case "$driver" in + nvidia) pass "$pci bound to nvidia" ;; + nouveau) + fail "$pci is driven by nouveau; install the NVIDIA driver$ON_HOST" + ;; + vfio-pci) + fail "$pci is reserved for VM passthrough (vfio-pci); an LXC or this host cannot use it" + ;; + *) + fail "$pci has no driver" + if [ "$IN_LXC" = false ] && mokutil --sb-state 2>/dev/null | grep -q enabled; then + hint "Secure Boot is on: use the signed prebuilt modules ('ubuntu-drivers install') or enroll the DKMS key with mokutil." + fi + ;; + esac + done + + kernel_version=$(nvidia_kernel_version) + if [ -z "$kernel_version" ]; then + fail "the NVIDIA kernel module is not loaded" + if [ "$IN_LXC" = false ] && [ "$APPLY" = true ] && [ "$OS_ID" = ubuntu ] && command -v ubuntu-drivers >/dev/null 2>&1; then + apply_step "installed the recommended NVIDIA driver; reboot, then run this again" \ + env DEBIAN_FRONTEND=noninteractive ubuntu-drivers install /dev/null + elif [ "$IN_LXC" = false ] && [ "$OS_ID" = ubuntu ]; then + hint "Install it with 'ubuntu-drivers install' (--apply does), then reboot." + fi + else + pass "kernel module $kernel_version" + fi + + missing="" + for node in /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-modeset; do + [ -c "$node" ] || missing+=" $node" + done + ls /dev/nvidia[0-9]* >/dev/null 2>&1 || missing+=" /dev/nvidia0" + if [ -z "$missing" ]; then + pass "device nodes present" + else + fail "missing device nodes:$missing" + if [ "$IN_LXC" = true ]; then + hint "Bind them into the container (proxmox-gpu-lxc.sh does), and create them on the host first." + else + hint "'nvidia-modprobe -u -c0' creates them; nvidia-persistenced keeps them across boots." + fi + fi + + if [ -n "$kernel_version" ]; then + smi_version=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1) + if [ "$smi_version" = "$kernel_version" ]; then + pass "nvidia-smi sees $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1), driver $smi_version" + elif [ "$IN_LXC" = true ]; then + if [ "$APPLY" = true ]; then + install_nvidia_userspace "$kernel_version" + else + fail "the container's NVIDIA libraries ($( [ -n "$smi_version" ] && echo "$smi_version" || echo none)) do not match the host's module $kernel_version (--apply installs $kernel_version)" + fi + else + fail "nvidia-smi does not work: $(nvidia-smi 2>&1 | head -1)" + hint "A library/module version mismatch clears after a reboot." + fi + fi + + if command -v nvidia-container-runtime >/dev/null 2>&1; then + pass "nvidia-container-toolkit installed" + elif [ "$APPLY" = true ]; then + apply_step "installed nvidia-container-toolkit" install_nvidia_container_toolkit + else + fail "nvidia-container-toolkit is not installed (--apply installs it)" + fi + + if command -v nvidia-container-cli >/dev/null 2>&1; then + if nvidia-container-cli info >/dev/null 2>&1; then + pass "nvidia-container-cli reaches the GPU" + else + fail "nvidia-container-cli cannot reach the GPU: $(nvidia-container-cli info 2>&1 | tail -1)" + fi + fi + + # k3s writes an nvidia runtime into its containerd config only when it + # finds nvidia-container-runtime at start, so a toolkit installed after + # the join needs a k3s restart. + local containerd=/var/lib/rancher/k3s/agent/etc/containerd/config.toml service + service=$(k3s_service) + if [ -f "$containerd" ]; then + if grep -q nvidia-container-runtime "$containerd"; then + pass "k3s containerd has the nvidia runtime" + elif [ "$APPLY" = true ] && command -v nvidia-container-runtime >/dev/null 2>&1; then + apply_step "restarted $service to pick up the nvidia runtime" systemctl restart "$service" + else + fail "k3s containerd has no nvidia runtime; restart $service after installing the toolkit" + fi + fi +} + +# --- Common ---------------------------------------------------------------- + +# The k3s unit: k3s-agent on a node that joined the panel, k3s when the GPU +# node is the panel's server itself. +k3s_service() { + if systemctl is-enabled --quiet k3s 2>/dev/null; then + echo k3s + else + echo k3s-agent + fi +} + +GOVERNOR_UNIT=/etc/systemd/system/cpu-governor-performance.service + +install_governor_unit() { + cat >"$GOVERNOR_UNIT" <<'UNIT' || return 1 +[Unit] +Description=Set the CPU frequency governor to performance for game streaming +After=sysinit.target power-profiles-daemon.service + +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'for g in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "$g"; done' +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +UNIT + systemctl daemon-reload && systemctl enable --now cpu-governor-performance.service >/dev/null 2>&1 +} + +check_common() { + step "Node" + + local tool + for tool in curl jq; do + if command -v "$tool" >/dev/null 2>&1; then + pass "$tool installed" + elif [ "$APPLY" = true ] && command -v apt-get >/dev/null 2>&1; then + apply_step "installed $tool" apt_install "$tool" + else + fail "$tool is not installed (--apply installs it); the node setup command needs it" + fi + done + + local free_gb + free_gb=$(df -BG / | awk 'NR==2 {print $4}' | tr -d 'G') + if [ "${free_gb:-0}" -ge 120 ]; then + pass "${free_gb} GB free on /" + else + warn "${free_gb} GB free on /; the node setup asks for 120 GB on a fresh node" + fi + + local governor + governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null) + if [ -z "$governor" ]; then + info "no CPU frequency scaling exposed" + elif [ "$governor" = performance ]; then + pass "CPU governor is performance" + elif [ "$IN_LXC" = true ]; then + warn "CPU governor is '$governor'; set it to performance on the host (proxmox-gpu-lxc.sh --apply does)" + elif [ "$APPLY" = true ]; then + apply_step "CPU governor set to performance, at every boot ($GOVERNOR_UNIT)" install_governor_unit + else + warn "CPU governor is '$governor'; render pods drop frames on a ramping CPU (--apply sets performance at every boot)" + fi + + if command -v k3s >/dev/null 2>&1; then + local installed service + installed=$(k3s --version 2>/dev/null | awk 'NR==1 {print $3}') + service=$(k3s_service) + if [ -n "$K3S_VERSION" ] && [ "$installed" != "$K3S_VERSION" ] && + [ "$(printf '%s\n' "$installed" "$K3S_VERSION" | sort -V | tail -1)" = "$installed" ]; then + warn "k3s $installed is newer than the panel's $K3S_VERSION; a kubelet newer than its API server is unsupported" + hint "Re-join with: export INSTALL_K3S_VERSION=$K3S_VERSION, then the panel's node setup command." + else + pass "k3s $installed installed" + fi + if systemctl is-active --quiet "$service" 2>/dev/null; then + pass "$service running" + else + fail "$service is not running; see 'journalctl -u $service -n 50'" + fi + else + info "k3s not installed yet. The node setup command installs the latest k3s; to match the panel, first run:" + if [ -n "$K3S_VERSION" ]; then + hint "export INSTALL_K3S_VERSION=$K3S_VERSION" + else + hint "export INSTALL_K3S_VERSION=" + fi + fi +} + +# k3s in an LXC needs a privileged container with the host's kernel features +# opened up; proxmox-gpu-lxc.sh sets all of this on the host. +check_lxc() { + step "LXC container" + + if awk '$1 == 0 && $2 == 0 { found = 1 } END { exit !found }' /proc/self/uid_map; then + pass "privileged container" + else + fail "unprivileged container; k3s needs a privileged one" + fi + + local profile + profile=$(cat /proc/self/attr/current 2>/dev/null) + case "$profile" in + unconfined | "") pass "AppArmor unconfined" ;; + *) fail "AppArmor profile '$profile' blocks containerd; set lxc.apparmor.profile: unconfined" ;; + esac + + if [ -e /dev/kmsg ]; then + pass "/dev/kmsg present" + else + fail "/dev/kmsg missing; the kubelet needs it" + fi + if [ -c /dev/net/tun ]; then + pass "/dev/net/tun present" + else + fail "/dev/net/tun missing; Tailscale needs it" + fi + + if [ -w /proc/sys/net/ipv4/ip_forward ]; then + pass "/proc/sys writable" + else + fail "/proc/sys is read-only; set lxc.mount.auto: proc:rw sys:rw" + fi + + local controllers controller + controllers=$(cat /sys/fs/cgroup/cgroup.controllers 2>/dev/null) + for controller in cpuset cpu memory pids; do + if [[ " $controllers " != *" $controller "* ]]; then + fail "cgroup controller '$controller' is not delegated to the container" + fi + done + + # Check what each module provides, not /sys/module: a module built into + # the host kernel (br_netfilter on Proxmox) has no /sys/module entry, and + # the container has no /lib/modules for the host kernel to ask modinfo. + if grep -qw overlay /proc/filesystems 2>/dev/null; then + pass "kernel module overlay loaded" + else + fail "kernel module overlay is not loaded; load it on the host" + fi + if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + pass "kernel module br_netfilter loaded" + else + fail "kernel module br_netfilter is not loaded; load it on the host" + fi + + # The kubelet's static CPU manager finds CPUs through sysfs, which shows + # every host CPU even where lxcfs narrows /proc/cpuinfo to the container's + # cores, so a container held to fewer cores gets pods pinned to CPUs it + # cannot use. + local allowed host_cpus + allowed=$(nproc 2>/dev/null) + host_cpus=$(find /sys/devices/system/cpu -maxdepth 1 -name 'cpu[0-9]*' | wc -l) + if [ -n "$allowed" ] && [ "$host_cpus" -gt "$allowed" ]; then + warn "the container may use $allowed of the host's $host_cpus CPUs; remove its core limit (pct set --delete cores), or the kubelet can pin pods to CPUs it cannot use" + else + pass "the container may use every CPU ($allowed)" + fi +} + +[ "$IN_LXC" = true ] && check_lxc +case "$VENDOR" in + amd) check_amd ;; + nvidia) check_nvidia ;; +esac +check_common + +echo +if [ "$FAILS" -gt 0 ]; then + echo "${C_ERR}$FAILS failed, $WARNS warnings${C_RESET}" + [ "$APPLY" = false ] && echo "Run with --apply to fix what this script can, then run it again." + exit 1 +fi +echo "${C_OK}ready for the ${VENDOR} GPU setup ($WARNS warnings)${C_RESET}" +echo "Next: run the panel's node setup command here, then setup-gpu-nodes.sh on the panel." diff --git a/gpu-node/proxmox-gpu-lxc.sh b/gpu-node/proxmox-gpu-lxc.sh new file mode 100755 index 0000000..0a8a340 --- /dev/null +++ b/gpu-node/proxmox-gpu-lxc.sh @@ -0,0 +1,504 @@ +#!/bin/bash + +# Prepares a Proxmox host and one of its LXC containers to run a 5stack GPU +# node: k3s inside the container, rendering on the host's NVIDIA or AMD GPU. +# Run it as root on the Proxmox host. With --apply it also fixes what it can; +# it backs up the container's config first and never restarts the container. +# +# proxmox-gpu-lxc.sh CTID [--apply] [--vendor nvidia|amd] +# +# Then, inside the container, run gpu-node/gpu-node-check.sh. +# +# Exit code: 0 when nothing failed, 1 when a check failed, 2 on bad usage. + +CTID="" +APPLY=false +VENDOR="" +PVE_LXC_DIR="${PVE_LXC_DIR:-/etc/pve/lxc}" + +# The kernel modules k3s, flannel and kube-proxy load. A container cannot load +# modules, so the host loads them, now and at every boot. +K3S_MODULES="overlay br_netfilter nf_conntrack vxlan ip_tables iptable_nat iptable_filter iptable_mangle xt_conntrack xt_mark xt_comment nf_tables tun" +MODULES_FILE=/etc/modules-load.d/5stack-k3s.conf + +if [ -t 1 ]; then + C_RESET=$'\033[0m' C_STEP=$'\033[1;36m' C_OK=$'\033[0;32m' C_WARN=$'\033[1;33m' C_ERR=$'\033[0;31m' +else + C_RESET='' C_STEP='' C_OK='' C_WARN='' C_ERR='' +fi + +FAILS=0 +WARNS=0 +step() { echo; echo "${C_STEP}==> $1${C_RESET}"; } +pass() { echo "${C_OK} [PASS] $1${C_RESET}"; } +info() { echo " [INFO] $1"; } +warn() { WARNS=$((WARNS + 1)); echo "${C_WARN} [WARN] $1${C_RESET}"; } +fail() { FAILS=$((FAILS + 1)); echo "${C_ERR} [FAIL] $1${C_RESET}"; } +fixed() { echo "${C_OK} [FIX ] $1${C_RESET}"; } +hint() { echo " $1"; } + +usage() { + echo "usage: proxmox-gpu-lxc.sh CTID [--apply] [--vendor nvidia|amd]" >&2 + exit 2 +} + +# apply_step DESCRIPTION COMMAND... +# Runs an --apply step and reports it as fixed or failed. +apply_step() { + local description="$1" + shift + if "$@"; then + fixed "$description" + else + fail "could not apply: $description" + fi +} + +# conf_value FILE KEY +# The KEY's value from the container's own section (snapshots follow it). +conf_value() { + awk -v key="$2" '/^\[/ { exit } index($0, key ": ") == 1 { print substr($0, length(key) + 3); exit }' "$1" +} + +# size_gb SIZE +# 128G -> 128, 2T -> 2048, 512M -> 0. +size_gb() { + local size="$1" number unit + number="${size%[KMGT]}" + unit="${size#"$number"}" + case "$number" in '' | *[!0-9]*) return ;; esac + case "$unit" in + T) echo $((number * 1024)) ;; + G | "") echo "$number" ;; + *) echo 0 ;; + esac +} + +# features_with_nesting FEATURES +# keyctl=1,fuse=1 -> keyctl=1,fuse=1,nesting=1, keeping every other feature. +features_with_nesting() { + local feature out="" + local IFS=, + for feature in $1; do + case "$feature" in nesting=*) continue ;; esac + out+="${out:+,}$feature" + done + echo "${out:+$out,}nesting=1" +} + +# lxc_lines VENDOR +# The raw LXC lines that let k3s run in the container and reach the GPU. +lxc_lines() { + echo "lxc.apparmor.profile: unconfined" + echo "lxc.cgroup2.devices.allow: a" + echo "lxc.cap.drop: " + echo "lxc.mount.auto: proc:rw sys:rw" + echo "lxc.mount.entry: /dev/kmsg dev/kmsg none bind,optional,create=file" + echo "lxc.mount.entry: /dev/net/tun dev/net/tun none bind,optional,create=file" + case "$1" in + amd) + echo "lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir" + echo "lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file" + ;; + nvidia) + local node + for node in /dev/nvidia[0-9]* /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-uvm-tools /dev/nvidia-modeset; do + case "$node" in '/dev/nvidia[0-9]*') node=/dev/nvidia0 ;; esac + echo "lxc.mount.entry: $node ${node#/} none bind,optional,create=file" + done + echo "lxc.mount.entry: /dev/nvidia-caps dev/nvidia-caps none bind,optional,create=dir" + ;; + esac +} + +# conf_has_line FILE LINE +# True when the container's own section has LINE. Proxmox rewrites the file on +# every change, so trailing spaces do not count. +conf_has_line() { + LINE="$2" awk ' + function trim(s) { sub(/[ \t]+$/, "", s); return s } + /^\[/ { exit } + trim($0) == trim(ENVIRON["LINE"]) { found = 1; exit } + END { exit !found } + ' "$1" +} + +# missing_lines FILE LINE... +# Prints the LINEs the container's own section lacks. +missing_lines() { + local file="$1" line + shift + for line in "$@"; do + conf_has_line "$file" "$line" || echo "$line" + done +} + +# add_lxc_lines FILE LINE... +# Adds the LINEs FILE lacks to the container's own section, which ends where +# the first snapshot section starts. Prints how many it added. +add_lxc_lines() { + local file="$1" missing tmp + shift + missing=$(missing_lines "$file" "$@") + if [ -n "$missing" ]; then + tmp=$(mktemp) + # Blank lines are held back until the next line shows whether they + # end the container's section. + LINES="$missing" awk ' + !done && /^$/ { blanks = blanks "\n"; next } + !done && /^\[/ { print ENVIRON["LINES"]; done = 1 } + { printf "%s", blanks; blanks = ""; print } + END { if (!done) print ENVIRON["LINES"]; printf "%s", blanks } + ' "$file" >"$tmp" + # Rewrite in place: /etc/pve is a cluster file system that does not + # take renames over an existing file, and is read-only without quorum. + if ! cat "$tmp" >"$file"; then + rm -f "$tmp" + return 1 + fi + rm -f "$tmp" + fi +} + +# locked_add_lxc_lines FILE LINE... +# add_lxc_lines under the lock pct and backups take on the container's config. +locked_add_lxc_lines() { + local lock="/run/lock/lxc/pve-config-$CTID.lock" + mkdir -p "${lock%/*}" + ( + flock -w 30 9 || exit 1 + add_lxc_lines "$@" + ) 9>"$lock" +} + +check_host_gpu() { + local dev class driver pci + local nvidia=() amd=() + + step "Host GPU" + + for dev in /sys/bus/pci/devices/*; do + class=$(cat "$dev/class" 2>/dev/null) + case "$class" in 0x0300* | 0x0302* | 0x0380*) ;; *) continue ;; esac + case "$(cat "$dev/vendor" 2>/dev/null)" in + 0x10de) nvidia+=("$dev") ;; + 0x1002) amd+=("$dev") ;; + esac + done + + if [ -z "$VENDOR" ]; then + if [ ${#nvidia[@]} -gt 0 ] && [ ${#amd[@]} -gt 0 ]; then + fail "found both NVIDIA and AMD GPUs; pass --vendor nvidia or --vendor amd" + return + elif [ ${#nvidia[@]} -gt 0 ]; then + VENDOR=nvidia + elif [ ${#amd[@]} -gt 0 ]; then + VENDOR=amd + else + fail "no NVIDIA or AMD GPU found on the PCI bus" + return + fi + fi + + local devices=("${amd[@]}") expected=amdgpu + [ "$VENDOR" = nvidia ] && devices=("${nvidia[@]}") expected=nvidia + + for dev in "${devices[@]}"; do + pci=$(basename "$dev") + driver=$(basename "$(readlink "$dev/driver" 2>/dev/null)" 2>/dev/null) + if [ "$driver" = "$expected" ]; then + pass "$pci bound to $driver" + elif [ "$driver" = vfio-pci ]; then + fail "$pci is bound to vfio-pci for VM passthrough; a container shares the host's driver instead" + hint "Remove it from the vfio-pci ids (/etc/modprobe.d) and any VM's hostpci, then reboot." + else + fail "$pci is bound to '${driver:-nothing}', not $expected" + fi + done + + local arg + for arg in nomodeset amdgpu.modeset=0 amdgpu.dpm=0; do + if tr ' ' '\n' /dev/null 2>&1; then + pass "render nodes: $(cd /dev/dri && echo renderD*)" + else + fail "no /dev/dri/renderD* on the host" + fi + + local log + log=$(journalctl -k -b 0 --no-pager 2>/dev/null) + if echo "$log" | grep -q "Direct firmware load for amdgpu/.* failed"; then + fail "amdgpu firmware is missing: $(echo "$log" | grep -o "amdgpu/[^ ]*" | head -1)" + hint "Proxmox ships GPU firmware in pve-firmware; update it ('apt install pve-firmware') and the kernel, then reboot." + elif echo "$log" | grep -q "amdgpu.*Fatal error during GPU init"; then + fail "amdgpu failed to start the GPU this boot; see 'journalctl -k -b | grep amdgpu'" + else + pass "no amdgpu firmware or init errors this boot" + fi +} + +NVIDIA_UNIT=/etc/systemd/system/5stack-nvidia-devices.service + +check_host_nvidia() { + local version + version=$(head -1 /proc/driver/nvidia/version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1) + if [ -z "$version" ]; then + fail "the NVIDIA driver is not loaded on the host" + hint "Install it on the host with the proxmox-headers package and NVIDIA's .run installer, then reboot." + return + fi + pass "NVIDIA driver $version; the container needs the same version's libraries (gpu-node-check.sh --apply installs them)" + + # The NVIDIA module creates no device nodes itself; they appear once + # something opens the GPU. The container cannot create them, so the host + # makes them at boot: nvidia-smi the per-GPU nodes and nvidiactl, + # nvidia-modprobe the uvm and modeset ones. + local missing="" node + for node in /dev/nvidia0 /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-uvm-tools /dev/nvidia-modeset; do + [ -c "$node" ] || missing+=" $node" + done + if [ -z "$missing" ] && [ -f "$NVIDIA_UNIT" ]; then + pass "device nodes present, and created at every boot" + elif [ "$APPLY" = true ] && command -v nvidia-modprobe >/dev/null 2>&1; then + apply_step "device nodes created, and at every boot ($NVIDIA_UNIT)" install_nvidia_unit + elif [ -n "$missing" ]; then + fail "missing device nodes:$missing (--apply creates them at every boot with nvidia-modprobe)" + else + warn "nothing creates the NVIDIA device nodes at boot, so a container that starts first misses them (--apply adds a unit)" + fi +} + +install_nvidia_unit() { + cat >"$NVIDIA_UNIT" <<'UNIT' || return 1 +[Unit] +Description=Create the NVIDIA device nodes for 5stack GPU containers +After=systemd-modules-load.service +Before=pve-guests.service + +[Service] +Type=oneshot +ExecStart=-/usr/bin/nvidia-smi -L +ExecStart=-/usr/bin/nvidia-modprobe -c0 +ExecStart=-/usr/bin/nvidia-modprobe -c0 -u +ExecStart=-/usr/bin/nvidia-modprobe -m +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +UNIT + systemctl daemon-reload && systemctl enable --now 5stack-nvidia-devices.service >/dev/null 2>&1 +} + +write_modules_file() { + # shellcheck disable=SC2086 # one module per line + printf '%s\n' $K3S_MODULES >"$MODULES_FILE" +} + +check_host_kernel() { + step "Host kernel modules" + + local module missing=() + for module in $K3S_MODULES; do + [ -d "/sys/module/$module" ] || modinfo "$module" 2>/dev/null | grep -q '^filename:.*(builtin)' || missing+=("$module") + done + local persisted=true + [ -f "$MODULES_FILE" ] || persisted=false + + if [ ${#missing[@]} -eq 0 ] && [ "$persisted" = true ]; then + pass "k3s modules loaded, and at every boot" + elif [ "$APPLY" = true ]; then + apply_step "k3s modules load at every boot ($MODULES_FILE)" write_modules_file + for module in "${missing[@]}"; do + apply_step "loaded $module" modprobe "$module" + done + elif [ ${#missing[@]} -gt 0 ]; then + fail "not loaded: ${missing[*]} (--apply loads them, and at every boot)" + else + warn "the k3s modules are loaded now but not at every boot (--apply writes $MODULES_FILE)" + fi + + local governor + governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null) + if [ -z "$governor" ] || [ "$governor" = performance ]; then + pass "CPU governor ${governor:-not exposed}" + elif [ "$APPLY" = true ]; then + apply_step "CPU governor set to performance, at every boot" install_governor_unit + else + warn "CPU governor is '$governor'; render pods drop frames on a ramping CPU (--apply sets performance at every boot)" + fi +} + +install_governor_unit() { + cat >/etc/systemd/system/cpu-governor-performance.service <<'UNIT' || return 1 +[Unit] +Description=Set the CPU frequency governor to performance for game streaming +After=sysinit.target + +[Service] +Type=oneshot +ExecStart=/bin/sh -c 'for g in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "$g"; done' +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +UNIT + systemctl daemon-reload && systemctl enable --now cpu-governor-performance.service >/dev/null 2>&1 +} + +check_container() { + local conf="$PVE_LXC_DIR/$CTID.conf" + + step "Container $CTID" + + if [ ! -f "$conf" ]; then + fail "no container $CTID ($conf)" + return + fi + + if [ "$(conf_value "$conf" unprivileged)" = 1 ]; then + fail "the container is unprivileged; k3s needs a privileged container" + hint "Back it up and restore it as privileged: vzdump $CTID, then pct restore with --unprivileged 0." + else + pass "privileged container" + fi + + local rootfs size gb storage + rootfs=$(conf_value "$conf" rootfs) + size=$(echo "$rootfs" | grep -oE 'size=[0-9]+[KMGT]?' | cut -d= -f2) + gb=$(size_gb "$size") + if [ -z "$gb" ]; then + warn "could not read the root disk size from '$rootfs'" + elif [ "$gb" -ge 120 ]; then + pass "root disk ${size}" + else + fail "root disk is ${size}; the node setup needs 120 GB free (pct resize $CTID rootfs 128G)" + fi + + storage=${rootfs%%:*} + if command -v pvesm >/dev/null 2>&1 && pvesm status 2>/dev/null | awk -v s="$storage" '$1 == s { print $2 }' | grep -qx zfspool; then + local zfs + zfs=$(zfs version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+' | head -1) + if [ -n "$zfs" ] && [ "$(printf '%s\n' 2.2 "$zfs" | sort -V | head -1)" != 2.2 ]; then + fail "the root disk is on ZFS $zfs; containerd's overlayfs needs ZFS 2.2 or newer" + fi + fi + + local cores host_cpus + cores=$(conf_value "$conf" cores) + host_cpus=$(nproc --all) + if [ -n "$cores" ] && [ "$cores" -lt "$host_cpus" ]; then + warn "the container is limited to $cores of $host_cpus cores; the kubelet's static CPU manager can pin pods to cores it cannot use" + hint "Remove the limit: pct set $CTID --delete cores" + fi + + # A change to a running container waits in its pending section until the + # next restart; pct config shows it, conf_value does not. + local features pending + features=$(conf_value "$conf" features) + pending=$(pct config "$CTID" 2>/dev/null | sed -n 's/^features: //p') + if [[ ",$features," == *",nesting=1,"* ]]; then + pass "nesting enabled" + elif [[ ",$pending," == *",nesting=1,"* ]]; then + warn "nesting turns on at the next container restart" + RESTART_NEEDED=true + elif [ "$APPLY" = true ]; then + backup_conf "$conf" + apply_step "enabled nesting" pct set "$CTID" --features "$(features_with_nesting "$features")" + RESTART_NEEDED=true + else + fail "nesting is off (--apply turns it on)" + fi + + [ -z "$VENDOR" ] && return + + local lines=() line missing=() + mapfile -t lines < <(lxc_lines "$VENDOR") + mapfile -t missing < <(missing_lines "$conf" "${lines[@]}") + if [ ${#missing[@]} -eq 0 ]; then + pass "LXC settings for k3s and the $VENDOR GPU in place" + elif [ "$APPLY" = true ]; then + backup_conf "$conf" + apply_step "added ${#missing[@]} LXC lines for k3s and the $VENDOR GPU" locked_add_lxc_lines "$conf" "${lines[@]}" + RESTART_NEEDED=true + else + fail "${#missing[@]} LXC lines for k3s and the $VENDOR GPU are missing (--apply adds them):" + for line in "${missing[@]}"; do + hint "$line" + done + fi +} + +BACKED_UP=false +backup_conf() { + [ "$BACKED_UP" = true ] && return + local backup + backup="/root/5stack-lxc-$CTID.conf.$(date +%Y%m%d-%H%M%S)" + cp "$1" "$backup" && BACKED_UP=true && info "backed up the container config to $backup" +} + +main() { + while [ $# -gt 0 ]; do + case "$1" in + --apply) APPLY=true ;; + --vendor) + VENDOR="$2" + shift + ;; + -h | --help) usage ;; + -*) + echo "unknown option: $1" >&2 + usage + ;; + *) CTID="$1" ;; + esac + shift + done + + case "$CTID" in '' | *[!0-9]*) usage ;; esac + case "$VENDOR" in + "" | nvidia | amd) ;; + *) + echo "--vendor must be nvidia or amd" >&2 + exit 2 + ;; + esac + if [ "$(id -u)" -ne 0 ] || ! command -v pct >/dev/null 2>&1; then + echo "run this as root on the Proxmox host" >&2 + exit 2 + fi + + RESTART_NEEDED=false + check_host_gpu + check_host_kernel + check_container + + echo + if [ "$RESTART_NEEDED" = true ]; then + echo "${C_WARN}Restart the container to apply its new settings: pct reboot $CTID${C_RESET}" + fi + if [ "$FAILS" -gt 0 ]; then + echo "${C_ERR}$FAILS failed, $WARNS warnings${C_RESET}" + [ "$APPLY" = false ] && echo "Run with --apply to fix what this script can, then run it again." + exit 1 + fi + echo "${C_OK}host and container $CTID ready ($WARNS warnings)${C_RESET}" + echo "Next, inside the container: gpu-node-check.sh --apply, then the panel's node setup command." +} + +# The tests source this file for its functions. Comparing BASH_SOURCE with $0 +# cannot tell that apart from "curl ... | bash", where both differ. +if [ -z "${PROXMOX_GPU_LXC_LIB:-}" ]; then + main "$@" +fi diff --git a/tests/scripts/proxmox_gpu_lxc_test.sh b/tests/scripts/proxmox_gpu_lxc_test.sh new file mode 100644 index 0000000..537616c --- /dev/null +++ b/tests/scripts/proxmox_gpu_lxc_test.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +load() { + # shellcheck source=gpu-node/proxmox-gpu-lxc.sh + PROXMOX_GPU_LXC_LIB=1 source "$TEST_ROOT/gpu-node/proxmox-gpu-lxc.sh" + # The script has its own fail(); the asserts need the one that exits. + # shellcheck source=tests/scripts/lib.sh + source "$TEST_ROOT/tests/scripts/lib.sh" +} + +write_conf() { + cat >"$WORK_DIR/100.conf" <<'CONF' +arch: amd64 +cores: 8 +features: keyctl=1 +hostname: gpu-node +rootfs: local-lvm:vm-100-disk-0,size=128G +unprivileged: 0 + +[before-k3s] +arch: amd64 +rootfs: local-lvm:vm-100-disk-0,size=32G +unprivileged: 1 +CONF +} + +test_conf_value_reads_the_container_not_its_snapshots() { + load + write_conf + assert_eq "$(conf_value "$WORK_DIR/100.conf" rootfs)" "local-lvm:vm-100-disk-0,size=128G" + assert_eq "$(conf_value "$WORK_DIR/100.conf" unprivileged)" "0" + assert_eq "$(conf_value "$WORK_DIR/100.conf" nesting)" "" +} + +test_size_gb_reads_proxmox_disk_sizes() { + load + assert_eq "$(size_gb 128G)" "128" + assert_eq "$(size_gb 2T)" "2048" + assert_eq "$(size_gb 512M)" "0" + assert_eq "$(size_gb "")" "" +} + +test_nesting_is_added_without_dropping_other_features() { + load + assert_eq "$(features_with_nesting "keyctl=1,fuse=1")" "keyctl=1,fuse=1,nesting=1" + assert_eq "$(features_with_nesting "nesting=0,keyctl=1")" "keyctl=1,nesting=1" + assert_eq "$(features_with_nesting "")" "nesting=1" +} + +test_lxc_lines_go_before_the_first_snapshot() { + load + write_conf + add_lxc_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined" "lxc.cap.drop: " || fail "add_lxc_lines failed" + assert_eq "$(sed -n '6,10p' "$WORK_DIR/100.conf" | sed 's/ *$//')" "unprivileged: 0 +lxc.apparmor.profile: unconfined +lxc.cap.drop: + +[before-k3s]" + assert_eq "$(tail -1 "$WORK_DIR/100.conf")" "unprivileged: 1" +} + +# Proxmox drops the trailing space of an empty value when it rewrites the file. +test_lxc_lines_are_added_once() { + load + write_conf + add_lxc_lines "$WORK_DIR/100.conf" "lxc.cap.drop: " + sed -i 's/^lxc.cap.drop: $/lxc.cap.drop:/' "$WORK_DIR/100.conf" + add_lxc_lines "$WORK_DIR/100.conf" "lxc.cap.drop: " || fail "add_lxc_lines failed" + assert_eq "$(grep -c '^lxc.cap.drop' "$WORK_DIR/100.conf")" "1" +} + +test_a_line_in_a_snapshot_does_not_count() { + load + write_conf + echo "lxc.apparmor.profile: unconfined" >>"$WORK_DIR/100.conf" + assert_eq "$(missing_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined")" "lxc.apparmor.profile: unconfined" +} + +test_a_config_without_snapshots_gets_the_lines_at_the_end() { + load + printf 'arch: amd64\nunprivileged: 0\n' >"$WORK_DIR/101.conf" + add_lxc_lines "$WORK_DIR/101.conf" "lxc.mount.auto: proc:rw sys:rw" + assert_eq "$(cat "$WORK_DIR/101.conf")" "arch: amd64 +unprivileged: 0 +lxc.mount.auto: proc:rw sys:rw" +} + +test_amd_lines_bind_dri_and_kfd() { + load + assert_contains "$(lxc_lines amd)" "lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir" + assert_contains "$(lxc_lines amd)" "lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file" + assert_not_contains "$(lxc_lines amd)" "nvidia" +} + +test_nvidia_lines_bind_the_nvidia_devices() { + load + assert_contains "$(lxc_lines nvidia)" "lxc.mount.entry: /dev/nvidiactl dev/nvidiactl none bind,optional,create=file" + assert_contains "$(lxc_lines nvidia)" "lxc.mount.entry: /dev/nvidia-uvm dev/nvidia-uvm none bind,optional,create=file" + assert_contains "$(lxc_lines nvidia)" "/dev/nvidia0 dev/nvidia0" + assert_not_contains "$(lxc_lines nvidia)" "/dev/kfd" +} + +run_tests From 321e8bff083bec57a501a9b5051eb41903706afd Mon Sep 17 00:00:00 2001 From: Flegma Date: Mon, 2 Nov 2026 23:36:18 +0100 Subject: [PATCH 02/10] fix: proxmox-gpu-lxc.sh --apply never leaves a cut-off container config - The config is rewritten like pve-common's file_set_contents: a temp file beside it, checked (awk result, not empty, every line kept), then renamed over it. Any failure leaves the config as it was and reports a FAIL. - The backup is checked; when it fails, --apply changes nothing. - --apply stops at an unprivileged container instead of adding the lines. - proc:rw now comes with the warning that the kubelet then sets vm.overcommit_memory, kernel.panic and kernel.panic_on_oops on the whole host. - The disk check asks for 120 GB free: measured inside a running container, from the size (140 GB with the system) in a stopped one. Sizes in M, K or bytes are read right (131072M is 128 GB). - nomodeset and the amdgpu kernel arguments only fail an AMD host. - The CPU governor uses 5stack-cpu-governor.service, the panel tuning's unit, is restarted rather than enabled with --now, and only reports a fix once every CPU runs performance. The NVIDIA device unit is checked the same way. --- gpu-node/proxmox-gpu-lxc.sh | 251 ++++++++++++++++++------ tests/scripts/proxmox_gpu_lxc_test.sh | 268 ++++++++++++++++++++++++++ 2 files changed, 458 insertions(+), 61 deletions(-) diff --git a/gpu-node/proxmox-gpu-lxc.sh b/gpu-node/proxmox-gpu-lxc.sh index 0a8a340..7a3f49c 100755 --- a/gpu-node/proxmox-gpu-lxc.sh +++ b/gpu-node/proxmox-gpu-lxc.sh @@ -15,6 +15,16 @@ CTID="" APPLY=false VENDOR="" PVE_LXC_DIR="${PVE_LXC_DIR:-/etc/pve/lxc}" +BACKUP_DIR="${BACKUP_DIR:-/root}" +SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}" +GOVERNOR_SCRIPT="${GOVERNOR_SCRIPT:-/usr/local/bin/5stack-cpu-governor.sh}" +CPU_SYSFS="${CPU_SYSFS:-/sys/devices/system/cpu}" +PROC_CMDLINE="${PROC_CMDLINE:-/proc/cmdline}" + +# The node setup command stops unless / has this much free (df -BG), and the +# system, k3s and the render images need room on top of it. +REQUIRED_FREE_GB=120 +SYSTEM_HEADROOM_GB=20 # The kernel modules k3s, flannel and kube-proxy load. A container cannot load # modules, so the host loads them, now and at every boot. @@ -61,17 +71,21 @@ conf_value() { } # size_gb SIZE -# 128G -> 128, 2T -> 2048, 512M -> 0. +# A Proxmox disk size in whole GiB, rounded down: 128G -> 128, 2T -> 2048, +# 131072M -> 128, 512M -> 0. Without a unit the size is in bytes, as +# pve-common's parse_size reads it. size_gb() { local size="$1" number unit number="${size%[KMGT]}" unit="${size#"$number"}" - case "$number" in '' | *[!0-9]*) return ;; esac - case "$unit" in - T) echo $((number * 1024)) ;; - G | "") echo "$number" ;; - *) echo 0 ;; + case "$number" in + '' | .* | *. | *.*.* | *[!0-9.]*) return ;; esac + awk -v n="$number" -v unit="$unit" 'BEGIN { + divisor["T"] = 1 / 1024; divisor["G"] = 1; divisor["M"] = 1024 + divisor["K"] = 1024 * 1024; divisor[""] = 1024 * 1024 * 1024 + printf "%d\n", n / divisor[unit] + }' } # features_with_nesting FEATURES @@ -135,28 +149,45 @@ missing_lines() { # add_lxc_lines FILE LINE... # Adds the LINEs FILE lacks to the container's own section, which ends where -# the first snapshot section starts. Prints how many it added. +# the first snapshot section starts. FILE is either replaced whole or left +# as it was. add_lxc_lines() { - local file="$1" missing tmp + local file="$1" missing tmp before after added shift - missing=$(missing_lines "$file" "$@") - if [ -n "$missing" ]; then - tmp=$(mktemp) - # Blank lines are held back until the next line shows whether they - # end the container's section. - LINES="$missing" awk ' - !done && /^$/ { blanks = blanks "\n"; next } - !done && /^\[/ { print ENVIRON["LINES"]; done = 1 } - { printf "%s", blanks; blanks = ""; print } - END { if (!done) print ENVIRON["LINES"]; printf "%s", blanks } - ' "$file" >"$tmp" - # Rewrite in place: /etc/pve is a cluster file system that does not - # take renames over an existing file, and is read-only without quorum. - if ! cat "$tmp" >"$file"; then - rm -f "$tmp" - return 1 - fi + missing=$(missing_lines "$file" "$@") || return 1 + [ -n "$missing" ] || return 0 + + # Written the way pve-common's file_set_contents writes a guest config: + # a temporary file in the same directory, renamed over the old one, so a + # failed write never leaves a cut-off config behind. + tmp="$file.tmp.$$" + rm -f "$tmp" + # Blank lines are held back until the next line shows whether they end + # the container's section. + if ! ADD_LINES="$missing" awk ' + !done && /^$/ { blanks = blanks "\n"; next } + !done && /^\[/ { print ENVIRON["ADD_LINES"]; done = 1 } + { printf "%s", blanks; blanks = ""; print } + END { if (!done) print ENVIRON["ADD_LINES"]; printf "%s", blanks } + ' "$file" >"$tmp"; then + hint "could not write $tmp; $file is unchanged" rm -f "$tmp" + return 1 + fi + + # Every old line plus the new ones, or the result is not used. + before=$(grep -c '' "$file") + after=$(grep -c '' "$tmp") + added=$(printf '%s\n' "$missing" | wc -l) + if [ ! -s "$tmp" ] || [ "$after" -ne $((before + added)) ]; then + hint "the new config came out wrong ($after lines, expected $((before + added))); $file is unchanged" + rm -f "$tmp" + return 1 + fi + if ! mv -f "$tmp" "$file"; then + hint "could not rename $tmp over $file; $file is unchanged" + rm -f "$tmp" + return 1 fi } @@ -216,13 +247,6 @@ check_host_gpu() { fi done - local arg - for arg in nomodeset amdgpu.modeset=0 amdgpu.dpm=0; do - if tr ' ' '\n' /dev/null 2>&1; then pass "render nodes: $(cd /dev/dri && echo renderD*)" else @@ -249,7 +282,8 @@ check_host_amd() { fi } -NVIDIA_UNIT=/etc/systemd/system/5stack-nvidia-devices.service +NVIDIA_UNIT="$SYSTEMD_DIR/5stack-nvidia-devices.service" +NVIDIA_NODES="/dev/nvidia0 /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-uvm-tools /dev/nvidia-modeset" check_host_nvidia() { local version @@ -265,10 +299,8 @@ check_host_nvidia() { # something opens the GPU. The container cannot create them, so the host # makes them at boot: nvidia-smi the per-GPU nodes and nvidiactl, # nvidia-modprobe the uvm and modeset ones. - local missing="" node - for node in /dev/nvidia0 /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-uvm-tools /dev/nvidia-modeset; do - [ -c "$node" ] || missing+=" $node" - done + local missing + missing=$(missing_nvidia_nodes) if [ -z "$missing" ] && [ -f "$NVIDIA_UNIT" ]; then pass "device nodes present, and created at every boot" elif [ "$APPLY" = true ] && command -v nvidia-modprobe >/dev/null 2>&1; then @@ -280,7 +312,15 @@ check_host_nvidia() { fi } +missing_nvidia_nodes() { + local node + for node in $NVIDIA_NODES; do + [ -c "$node" ] || printf ' %s' "$node" + done +} + install_nvidia_unit() { + local missing cat >"$NVIDIA_UNIT" <<'UNIT' || return 1 [Unit] Description=Create the NVIDIA device nodes for 5stack GPU containers @@ -298,7 +338,15 @@ RemainAfterExit=yes [Install] WantedBy=multi-user.target UNIT - systemctl daemon-reload && systemctl enable --now 5stack-nvidia-devices.service >/dev/null 2>&1 + # restart, not enable --now, which leaves an active unit alone. + systemctl daemon-reload && + systemctl enable 5stack-nvidia-devices.service >/dev/null 2>&1 && + systemctl restart 5stack-nvidia-devices.service >/dev/null 2>&1 || return 1 + missing=$(missing_nvidia_nodes) + if [ -n "$missing" ]; then + hint "still missing:$missing" + return 1 + fi } write_modules_file() { @@ -334,27 +382,98 @@ check_host_kernel() { if [ -z "$governor" ] || [ "$governor" = performance ]; then pass "CPU governor ${governor:-not exposed}" elif [ "$APPLY" = true ]; then - apply_step "CPU governor set to performance, at every boot" install_governor_unit + apply_step "CPU governor set to performance, at every boot (5stack-cpu-governor.service)" install_governor_unit else warn "CPU governor is '$governor'; render pods drop frames on a ramping CPU (--apply sets performance at every boot)" fi } +# The same script and unit the panel's CPU governor tuning installs on a +# node, so a machine never ends up with two of them. install_governor_unit() { - cat >/etc/systemd/system/cpu-governor-performance.service <<'UNIT' || return 1 + local unit="$SYSTEMD_DIR/5stack-cpu-governor.service" governor + mkdir -p "${GOVERNOR_SCRIPT%/*}" || return 1 + cat >"$GOVERNOR_SCRIPT.5stack-new" <<'SCRIPT' || return 1 +#!/bin/sh +for governor in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do + echo performance > "$governor" +done +SCRIPT + chmod 0755 "$GOVERNOR_SCRIPT.5stack-new" && mv -f "$GOVERNOR_SCRIPT.5stack-new" "$GOVERNOR_SCRIPT" || return 1 + cat >"$unit.5stack-new" < "$g"; done' RemainAfterExit=yes +ExecStart=$GOVERNOR_SCRIPT [Install] WantedBy=multi-user.target UNIT - systemctl daemon-reload && systemctl enable --now cpu-governor-performance.service >/dev/null 2>&1 + mv -f "$unit.5stack-new" "$unit" || return 1 + # restart, not enable --now: --now leaves an active unit alone, and the + # governor may have been changed since it ran. + systemctl daemon-reload && + systemctl enable 5stack-cpu-governor.service >/dev/null 2>&1 && + systemctl restart 5stack-cpu-governor.service >/dev/null 2>&1 || return 1 + for governor in "$CPU_SYSFS"/cpu*/cpufreq/scaling_governor; do + [ -f "$governor" ] || continue + if [ "$(cat "$governor")" != performance ]; then + hint "$governor is still $(cat "$governor")" + return 1 + fi + done +} + +# container_free_gb +# GB free on / inside the container (df -BG, as the node setup reads it), or +# nothing while the container is not running. +container_free_gb() { + pct status "$CTID" 2>/dev/null | grep -q 'status: running' || return 0 + pct exec "$CTID" -- df -BG / 2>/dev/null | awk 'NR == 2 { sub(/G$/, "", $4); print $4 }' +} + +# check_disk ROOTFS +# The node setup needs REQUIRED_FREE_GB free on /, so a running container is +# judged by its free space, and a stopped one by its size. +check_disk() { + local rootfs="$1" size gb free want + size=$(echo "$rootfs" | grep -oE 'size=[0-9.]+[KMGT]?' | cut -d= -f2) + gb=$(size_gb "$size") + if [ -z "$gb" ]; then + warn "could not read the root disk size from '$rootfs'" + return + fi + want=$((REQUIRED_FREE_GB + SYSTEM_HEADROOM_GB)) + free=$(container_free_gb) + case "$free" in *[!0-9]*) free="" ;; esac + + if [ -n "$free" ]; then + if [ "$free" -ge "$REQUIRED_FREE_GB" ]; then + pass "root disk $size, ${free} GB free" + else + fail "root disk $size has ${free} GB free; the node setup needs $REQUIRED_FREE_GB GB free" + hint "Grow it: pct resize $CTID rootfs +$((REQUIRED_FREE_GB - free + SYSTEM_HEADROOM_GB))G" + fi + elif [ "$gb" -lt "$REQUIRED_FREE_GB" ]; then + fail "root disk is $size; the node setup needs $REQUIRED_FREE_GB GB free on it" + hint "Grow it: pct resize $CTID rootfs ${want}G" + elif [ "$gb" -lt "$want" ]; then + warn "root disk is $size; the node setup needs $REQUIRED_FREE_GB GB free, which leaves little for the system (${want}G is safer)" + hint "Start the container and run this again to check its free space." + else + pass "root disk $size (start the container to check its free space)" + fi +} + +# proc:rw is what lets the kubelet change kernel values, and the ones it sets +# are not namespaced, so they change the Proxmox host itself. +proc_rw_warning() { + warn "lxc.mount.auto: proc:rw lets the kubelet set vm.overcommit_memory=1, kernel.panic=10 and kernel.panic_on_oops=1, which apply to the whole host" + hint "The host then reboots 10 seconds after a kernel oops, taking every guest on it down." + hint "On a host shared with other guests, decide whether that is acceptable before restarting the container." } check_container() { @@ -370,21 +489,17 @@ check_container() { if [ "$(conf_value "$conf" unprivileged)" = 1 ]; then fail "the container is unprivileged; k3s needs a privileged container" hint "Back it up and restore it as privileged: vzdump $CTID, then pct restore with --unprivileged 0." + if [ "$APPLY" = true ]; then + hint "--apply changes nothing in this container until it is privileged." + return + fi else pass "privileged container" fi - local rootfs size gb storage + local rootfs storage rootfs=$(conf_value "$conf" rootfs) - size=$(echo "$rootfs" | grep -oE 'size=[0-9]+[KMGT]?' | cut -d= -f2) - gb=$(size_gb "$size") - if [ -z "$gb" ]; then - warn "could not read the root disk size from '$rootfs'" - elif [ "$gb" -ge 120 ]; then - pass "root disk ${size}" - else - fail "root disk is ${size}; the node setup needs 120 GB free (pct resize $CTID rootfs 128G)" - fi + check_disk "$rootfs" storage=${rootfs%%:*} if command -v pvesm >/dev/null 2>&1 && pvesm status 2>/dev/null | awk -v s="$storage" '$1 == s { print $2 }' | grep -qx zfspool; then @@ -414,7 +529,7 @@ check_container() { warn "nesting turns on at the next container restart" RESTART_NEEDED=true elif [ "$APPLY" = true ]; then - backup_conf "$conf" + backup_conf "$conf" || return apply_step "enabled nesting" pct set "$CTID" --features "$(features_with_nesting "$features")" RESTART_NEEDED=true else @@ -428,8 +543,13 @@ check_container() { mapfile -t missing < <(missing_lines "$conf" "${lines[@]}") if [ ${#missing[@]} -eq 0 ]; then pass "LXC settings for k3s and the $VENDOR GPU in place" - elif [ "$APPLY" = true ]; then - backup_conf "$conf" + return + fi + for line in "${missing[@]}"; do + [ "$line" = "lxc.mount.auto: proc:rw sys:rw" ] && proc_rw_warning + done + if [ "$APPLY" = true ]; then + backup_conf "$conf" || return apply_step "added ${#missing[@]} LXC lines for k3s and the $VENDOR GPU" locked_add_lxc_lines "$conf" "${lines[@]}" RESTART_NEEDED=true else @@ -440,12 +560,21 @@ check_container() { fi } +# backup_conf FILE +# Copies FILE to BACKUP_DIR once per run. --apply changes nothing in the +# container without a backup, so a failed copy fails the run. BACKED_UP=false backup_conf() { - [ "$BACKED_UP" = true ] && return + [ "$BACKED_UP" = true ] && return 0 local backup - backup="/root/5stack-lxc-$CTID.conf.$(date +%Y%m%d-%H%M%S)" - cp "$1" "$backup" && BACKED_UP=true && info "backed up the container config to $backup" + backup="$BACKUP_DIR/5stack-lxc-$CTID.conf.$(date +%Y%m%d-%H%M%S)" + if cp "$1" "$backup" 2>/dev/null && cmp -s "$1" "$backup"; then + BACKED_UP=true + info "backed up the container config to $backup" + return 0 + fi + fail "could not back up $1 to $backup, so --apply changes nothing in the container" + return 1 } main() { @@ -494,7 +623,7 @@ main() { exit 1 fi echo "${C_OK}host and container $CTID ready ($WARNS warnings)${C_RESET}" - echo "Next, inside the container: gpu-node-check.sh --apply, then the panel's node setup command." + echo "Next, inside the container: gpu-node-check.sh --apply, then join it from the panel's Game Server Nodes page." } # The tests source this file for its functions. Comparing BASH_SOURCE with $0 diff --git a/tests/scripts/proxmox_gpu_lxc_test.sh b/tests/scripts/proxmox_gpu_lxc_test.sh index 537616c..e2e2051 100644 --- a/tests/scripts/proxmox_gpu_lxc_test.sh +++ b/tests/scripts/proxmox_gpu_lxc_test.sh @@ -7,10 +7,21 @@ load() { # shellcheck source=gpu-node/proxmox-gpu-lxc.sh PROXMOX_GPU_LXC_LIB=1 source "$TEST_ROOT/gpu-node/proxmox-gpu-lxc.sh" # The script has its own fail(); the asserts need the one that exits. + SCRIPT_FAIL=$(declare -f fail) # shellcheck source=tests/scripts/lib.sh source "$TEST_ROOT/tests/scripts/lib.sh" } +# script FUNCTION ARGS... +# Runs one of the script's functions with the script's own fail(), which +# records a [FAIL] line instead of ending the test. +script() { + ( + eval "$SCRIPT_FAIL" + "$@" + ) +} + write_conf() { cat >"$WORK_DIR/100.conf" <<'CONF' arch: amd64 @@ -39,8 +50,14 @@ test_size_gb_reads_proxmox_disk_sizes() { load assert_eq "$(size_gb 128G)" "128" assert_eq "$(size_gb 2T)" "2048" + assert_eq "$(size_gb 131072M)" "128" assert_eq "$(size_gb 512M)" "0" + assert_eq "$(size_gb 134217728K)" "128" + assert_eq "$(size_gb 137438953472)" "128" + assert_eq "$(size_gb 8.5G)" "8" assert_eq "$(size_gb "")" "" + assert_eq "$(size_gb abc)" "" + assert_eq "$(size_gb 1.2.3G)" "" } test_nesting_is_added_without_dropping_other_features() { @@ -103,4 +120,255 @@ test_nvidia_lines_bind_the_nvidia_devices() { assert_not_contains "$(lxc_lines nvidia)" "/dev/kfd" } +# An awk that fails only for the rewrite in add_lxc_lines, the one call that +# passes ADD_LINES. MODE is fail (writes half the file, exits 1, as on a full +# disk), empty (writes nothing, exits 0) or short (writes 2 lines, exits 0). +shim_awk() { + local real + real=$(command -v awk) + mkdir -p "$WORK_DIR/bin" + cat >"$WORK_DIR/bin/awk" <"$WORK_DIR/out"; then + fail "add_lxc_lines succeeded although awk failed" + fi + assert_conf_untouched + assert_contains "$(cat "$WORK_DIR/out")" "is unchanged" +} + +test_an_empty_rewrite_is_refused() { + load + write_conf + cp "$WORK_DIR/100.conf" "$WORK_DIR/100.conf.orig" + shim_awk empty + if add_lxc_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined" >/dev/null; then + fail "add_lxc_lines took an empty result" + fi + assert_conf_untouched +} + +test_a_cut_off_rewrite_is_refused() { + load + write_conf + cp "$WORK_DIR/100.conf" "$WORK_DIR/100.conf.orig" + shim_awk short + if add_lxc_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined" >/dev/null; then + fail "add_lxc_lines took a cut-off result" + fi + assert_conf_untouched +} + +# A directory that takes no new file, like a full disk or /etc/pve without +# quorum. Root ignores the permission, so this only runs unprivileged. +test_a_directory_that_takes_no_temp_file_leaves_the_config_as_it_was() { + load + [ "$(id -u)" -eq 0 ] && return 0 + mkdir "$WORK_DIR/pve" + write_conf + mv "$WORK_DIR/100.conf" "$WORK_DIR/pve/100.conf" + cp "$WORK_DIR/pve/100.conf" "$WORK_DIR/100.conf.orig" + chmod 0555 "$WORK_DIR/pve" + if add_lxc_lines "$WORK_DIR/pve/100.conf" "lxc.apparmor.profile: unconfined" >/dev/null 2>&1; then + chmod 0755 "$WORK_DIR/pve" + fail "add_lxc_lines succeeded in a read-only directory" + fi + chmod 0755 "$WORK_DIR/pve" + assert_eq "$(cat "$WORK_DIR/pve/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" +} + +test_the_rewrite_replaces_the_file_by_rename() { + load + write_conf + local inode + inode=$(stat -c %i "$WORK_DIR/100.conf") + add_lxc_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined" || fail "add_lxc_lines failed" + [ "$(stat -c %i "$WORK_DIR/100.conf")" != "$inode" ] || fail "the config was rewritten in place" + assert_eq "$(find "$WORK_DIR" -maxdepth 1 -name '100.conf.tmp.*' | wc -l)" "0" +} + +# pct as the container tests see it: STATUS is what pct status says, FREE the +# free GB df reports inside, and every other call is logged. +stub_pct() { + PCT_STATUS="$1" + PCT_FREE="${2:-}" + # shellcheck disable=SC2317 # called by the function under test + pct() { + echo "pct $*" >>"$WORK_DIR/pct.log" + case "$1" in + status) echo "status: $PCT_STATUS" ;; + exec) + printf 'Filesystem 1G-blocks Used Available Use%% Mounted on\n' + printf '/dev/loop0 200G 10G %sG 10%% /\n' "$PCT_FREE" + ;; + esac + } + touch "$WORK_DIR/pct.log" +} + +test_a_running_container_is_judged_by_its_free_space() { + load + CTID=100 + stub_pct running 130 + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=128G")" "[PASS] root disk 128G, 130 GB free" + stub_pct running 100 + local out + out=$(script check_disk "local-lvm:vm-100-disk-0,size=128G") + assert_contains "$out" "[FAIL] root disk 128G has 100 GB free; the node setup needs 120 GB free" + assert_contains "$out" "pct resize 100 rootfs +40G" +} + +test_a_stopped_container_is_judged_by_its_size() { + load + CTID=100 + stub_pct stopped + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=100G")" "[FAIL] root disk is 100G" + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=120G")" "[WARN] root disk is 120G" + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=131072M")" "[WARN] root disk is 131072M" + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=150G")" "[PASS] root disk 150G" + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0")" "[WARN] could not read the root disk size" +} + +write_container() { + write_conf + mkdir -p "$WORK_DIR/lxc" "$WORK_DIR/backup" + sed "s/^unprivileged: 0$/unprivileged: $1/" "$WORK_DIR/100.conf" >"$WORK_DIR/lxc/100.conf" + cp "$WORK_DIR/lxc/100.conf" "$WORK_DIR/100.conf.orig" + PVE_LXC_DIR="$WORK_DIR/lxc" + BACKUP_DIR="$WORK_DIR/backup" + CTID=100 + VENDOR=amd + APPLY=true + # The lock lives under /run/lock; the tests take none. + # shellcheck disable=SC2317 # called by the function under test + locked_add_lxc_lines() { add_lxc_lines "$@"; } +} + +test_apply_stops_at_an_unprivileged_container() { + load + write_container 1 + stub_pct stopped + local out + out=$(script check_container) + assert_contains "$out" "[FAIL] the container is unprivileged" + assert_contains "$out" "--apply changes nothing in this container until it is privileged" + assert_eq "$(cat "$WORK_DIR/lxc/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" + assert_not_contains "$(cat "$WORK_DIR/pct.log")" "pct set" + assert_eq "$(find "$WORK_DIR/backup" -type f | wc -l)" "0" +} + +test_apply_backs_up_then_adds_the_lines_and_warns_about_proc_rw() { + load + write_container 0 + stub_pct stopped + local out + out=$(script check_container) + assert_contains "$out" "[INFO] backed up the container config to $WORK_DIR/backup/5stack-lxc-100.conf." + assert_contains "$out" "[WARN] lxc.mount.auto: proc:rw lets the kubelet set vm.overcommit_memory=1, kernel.panic=10 and kernel.panic_on_oops=1" + assert_contains "$out" "reboots 10 seconds after a kernel oops" + assert_contains "$out" "[FIX ] added 8 LXC lines for k3s and the amd GPU" + assert_contains "$(cat "$WORK_DIR/pct.log")" "pct set 100 --features keyctl=1,nesting=1" + assert_eq "$(cat "$WORK_DIR/backup"/5stack-lxc-100.conf.*)" "$(cat "$WORK_DIR/100.conf.orig")" + local lines + mapfile -t lines < <(lxc_lines amd) + assert_eq "$(missing_lines "$WORK_DIR/lxc/100.conf" "${lines[@]}")" "" +} + +test_apply_changes_nothing_when_the_backup_fails() { + load + write_container 0 + stub_pct stopped + BACKUP_DIR="$WORK_DIR/no-such-dir" + local out + out=$(script check_container) + assert_contains "$out" "[FAIL] could not back up $WORK_DIR/lxc/100.conf" + assert_not_contains "$out" "[FIX ]" + assert_not_contains "$(cat "$WORK_DIR/pct.log")" "pct set" + assert_eq "$(cat "$WORK_DIR/lxc/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" +} + +test_report_mode_lists_the_lines_and_warns_about_proc_rw() { + load + write_container 0 + stub_pct stopped + APPLY=false + local out + out=$(script check_container) + assert_contains "$out" "[FAIL] 8 LXC lines for k3s and the amd GPU are missing" + assert_contains "$out" "[WARN] lxc.mount.auto: proc:rw lets the kubelet set" + assert_eq "$(cat "$WORK_DIR/lxc/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" +} + +test_nomodeset_only_fails_an_amd_host() { + load + echo "BOOT_IMAGE=/vmlinuz root=/dev/sda1 nomodeset quiet" >"$WORK_DIR/cmdline" + PROC_CMDLINE="$WORK_DIR/cmdline" + assert_contains "$(script check_host_amd)" "[FAIL] the kernel command line has nomodeset" + VENDOR=nvidia + assert_not_contains "$(script check_host_gpu)" "nomodeset" +} + +# systemctl for the governor unit: a restart either sets every governor or, +# like an already-active unit that enable --now skips, changes nothing. +stub_governor() { + mkdir -p "$WORK_DIR/cpu/cpu0/cpufreq" "$WORK_DIR/cpu/cpu1/cpufreq" "$WORK_DIR/systemd" + echo powersave >"$WORK_DIR/cpu/cpu0/cpufreq/scaling_governor" + echo powersave >"$WORK_DIR/cpu/cpu1/cpufreq/scaling_governor" + CPU_SYSFS="$WORK_DIR/cpu" + SYSTEMD_DIR="$WORK_DIR/systemd" + GOVERNOR_SCRIPT="$WORK_DIR/bin/5stack-cpu-governor.sh" + RESTART_SETS_GOVERNOR="$1" + systemctl() { + echo "systemctl $*" >>"$WORK_DIR/systemctl.log" + if [ "$1" = restart ] && [ "$RESTART_SETS_GOVERNOR" = true ]; then + local g + for g in "$CPU_SYSFS"/cpu*/cpufreq/scaling_governor; do echo performance >"$g"; done + fi + return 0 + } +} + +test_the_governor_unit_is_the_panels_and_restarted() { + load + stub_governor true + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + assert_contains "$(cat "$WORK_DIR/systemd/5stack-cpu-governor.service")" "ExecStart=$GOVERNOR_SCRIPT" + # shellcheck disable=SC2016 # the literal line the unit's script holds + assert_contains "$(cat "$GOVERNOR_SCRIPT")" 'echo performance > "$governor"' + assert_contains "$(cat "$WORK_DIR/systemctl.log")" "systemctl restart 5stack-cpu-governor.service" + assert_not_contains "$(cat "$WORK_DIR/systemctl.log")" "--now" +} + +test_the_governor_fix_fails_when_the_governor_did_not_change() { + load + stub_governor false + local out + out=$(script apply_step "CPU governor set to performance" install_governor_unit) + assert_contains "$out" "[FAIL] could not apply: CPU governor set to performance" + assert_contains "$out" "is still powersave" + assert_not_contains "$out" "[FIX ]" +} + run_tests From 711db8185b7a2021e5b1de79319b016694562f3c Mon Sep 17 00:00:00 2001 From: Flegma Date: Mon, 2 Nov 2026 23:36:18 +0100 Subject: [PATCH 03/10] fix: gpu-node-check.sh gives the right AMD, NVIDIA and k3s advice - An integrated AMD GPU is only ticked in setup-gpu-nodes.sh's integrated list when it is the node's only AMD GPU; beside a discrete card it stays unticked so the card renders. - The amdgpu.runpm=0 advice is gone: there is no kernel parameter to add. - The NVIDIA driver install keeps its own output quiet, so its FIX or FAIL line and the reboot note print again. - The missing device node hint names nvidia-modprobe -m for /dev/nvidia-modeset. - The k3s pin goes after sudo in the join command (sudo INSTALL_K3S_VERSION=... sh -s -- join), since sudo drops an exported variable. - The LXC check tests /proc/sys/vm/overcommit_memory, which proc:mixed leaves read-only, instead of /proc/sys/net. - The CPU governor uses 5stack-cpu-governor.service like the panel tuning, restarted and verified. - The script can be sourced for its functions; gpu_node_check_test.sh covers them and runs in the panel scripts workflow. --- .github/workflows/panel-scripts.yml | 1 + gpu-node/gpu-node-check.sh | 327 +++++++++++++++++---------- tests/scripts/gpu_node_check_test.sh | 250 ++++++++++++++++++++ 3 files changed, 455 insertions(+), 123 deletions(-) create mode 100644 tests/scripts/gpu_node_check_test.sh diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml index 7d83ad4..b7f97e4 100644 --- a/.github/workflows/panel-scripts.yml +++ b/.github/workflows/panel-scripts.yml @@ -64,3 +64,4 @@ jobs: - run: bash tests/scripts/postgres_backup_test.sh - run: bash tests/scripts/choose_node_test.sh - run: bash tests/scripts/proxmox_gpu_lxc_test.sh + - run: bash tests/scripts/gpu_node_check_test.sh diff --git a/gpu-node/gpu-node-check.sh b/gpu-node/gpu-node-check.sh index 6e450fd..7fbe572 100755 --- a/gpu-node/gpu-node-check.sh +++ b/gpu-node/gpu-node-check.sh @@ -18,39 +18,51 @@ APPLY=false VENDOR="" K3S_VERSION="" +IN_LXC=false +ON_HOST="" +NVIDIA_GPUS=() +AMD_GPUS=() +SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}" +GOVERNOR_SCRIPT="${GOVERNOR_SCRIPT:-/usr/local/bin/5stack-cpu-governor.sh}" +CPU_SYSFS="${CPU_SYSFS:-/sys/devices/system/cpu}" +# LXC's proc:mixed leaves /proc/sys/net writable but not /proc/sys/vm, and +# the kubelet writes to /proc/sys/vm, so that is what is tested. +PROC_SYS_PROBE="${PROC_SYS_PROBE:-/proc/sys/vm/overcommit_memory}" usage() { echo "usage: gpu-node-check.sh [--apply] [--vendor nvidia|amd] [--k3s-version v1.36.5+k3s1]" >&2 exit 2 } -while [ $# -gt 0 ]; do - case "$1" in - --apply) APPLY=true ;; - --vendor) - VENDOR="$2" - shift - ;; - --k3s-version) - K3S_VERSION="$2" - shift - ;; - -h | --help) usage ;; +parse_args() { + while [ $# -gt 0 ]; do + case "$1" in + --apply) APPLY=true ;; + --vendor) + VENDOR="$2" + shift + ;; + --k3s-version) + K3S_VERSION="$2" + shift + ;; + -h | --help) usage ;; + *) + echo "unknown option: $1" >&2 + usage + ;; + esac + shift + done + + case "$VENDOR" in + "" | nvidia | amd) ;; *) - echo "unknown option: $1" >&2 - usage + echo "--vendor must be nvidia or amd" >&2 + exit 2 ;; esac - shift -done - -case "$VENDOR" in - "" | nvidia | amd) ;; - *) - echo "--vendor must be nvidia or amd" >&2 - exit 2 - ;; -esac +} if [ -t 1 ]; then C_RESET=$'\033[0m' C_STEP=$'\033[1;36m' C_OK=$'\033[0;32m' C_WARN=$'\033[1;33m' C_ERR=$'\033[0;31m' @@ -68,16 +80,6 @@ fail() { FAILS=$((FAILS + 1)); echo "${C_ERR} [FAIL] $1${C_RESET}"; } fixed() { echo "${C_OK} [FIX ] $1${C_RESET}"; } hint() { echo " $1"; } -if [ "$APPLY" = true ] && [ "$(id -u)" -ne 0 ]; then - echo "--apply must run as root" >&2 - exit 2 -fi - -# shellcheck source=/dev/null -. /etc/os-release 2>/dev/null -OS_ID="${ID:-unknown}" -OS_VERSION="${VERSION_ID:-}" - apt_has() { command -v apt-cache >/dev/null 2>&1 && apt-cache policy "$1" 2>/dev/null | grep -q 'Candidate: [^(]' @@ -115,60 +117,60 @@ kernel_log() { dmesg 2>/dev/null || journalctl -k -b 0 --no-pager 2>/dev/null } -step "Machine" +check_machine() { + local container vm dev class -CONTAINER=$(systemd-detect-virt -c 2>/dev/null || true) -VM=$(systemd-detect-virt -v 2>/dev/null || true) -IN_LXC=false -ON_HOST="" -case "$CONTAINER" in - lxc*) - IN_LXC=true - ON_HOST=" on the host" - ;; - none | "") ;; - *) - fail "running in a $CONTAINER container; only bare metal, VMs and LXC are supported" - ;; -esac - -if [ "$IN_LXC" = true ]; then - info "LXC container on kernel $(uname -r); the host owns the GPU driver, firmware and CPU governor" -elif [ -n "$VM" ] && [ "$VM" != none ]; then - info "$VM VM on kernel $(uname -r); the GPU must be passed through to it" -else - info "bare metal, kernel $(uname -r)" -fi -info "$OS_ID $OS_VERSION" + step "Machine" -# GPUs, from sysfs, which a container sees too. -NVIDIA_GPUS=() -AMD_GPUS=() -for dev in /sys/bus/pci/devices/*; do - class=$(cat "$dev/class" 2>/dev/null) - case "$class" in - 0x0300* | 0x0302* | 0x0380*) ;; - *) continue ;; - esac - case "$(cat "$dev/vendor" 2>/dev/null)" in - 0x10de) NVIDIA_GPUS+=("$dev") ;; - 0x1002) AMD_GPUS+=("$dev") ;; + container=$(systemd-detect-virt -c 2>/dev/null || true) + vm=$(systemd-detect-virt -v 2>/dev/null || true) + case "$container" in + lxc*) + IN_LXC=true + ON_HOST=" on the host" + ;; + none | "") ;; + *) + fail "running in a $container container; only bare metal, VMs and LXC are supported" + ;; esac -done -if [ -z "$VENDOR" ]; then - if [ ${#NVIDIA_GPUS[@]} -gt 0 ] && [ ${#AMD_GPUS[@]} -gt 0 ]; then - fail "found both NVIDIA and AMD GPUs; pass --vendor nvidia or --vendor amd" - VENDOR=none - elif [ ${#NVIDIA_GPUS[@]} -gt 0 ]; then - VENDOR=nvidia - elif [ ${#AMD_GPUS[@]} -gt 0 ]; then - VENDOR=amd + if [ "$IN_LXC" = true ]; then + info "LXC container on kernel $(uname -r); the host owns the GPU driver, firmware and CPU governor" + elif [ -n "$vm" ] && [ "$vm" != none ]; then + info "$vm VM on kernel $(uname -r); the GPU must be passed through to it" else - fail "no NVIDIA or AMD GPU found on the PCI bus" - VENDOR=none + info "bare metal, kernel $(uname -r)" fi -fi + info "$OS_ID $OS_VERSION" + + # GPUs, from sysfs, which a container sees too. + for dev in /sys/bus/pci/devices/*; do + class=$(cat "$dev/class" 2>/dev/null) + case "$class" in + 0x0300* | 0x0302* | 0x0380*) ;; + *) continue ;; + esac + case "$(cat "$dev/vendor" 2>/dev/null)" in + 0x10de) NVIDIA_GPUS+=("$dev") ;; + 0x1002) AMD_GPUS+=("$dev") ;; + esac + done + + if [ -z "$VENDOR" ]; then + if [ ${#NVIDIA_GPUS[@]} -gt 0 ] && [ ${#AMD_GPUS[@]} -gt 0 ]; then + fail "found both NVIDIA and AMD GPUs; pass --vendor nvidia or --vendor amd" + VENDOR=none + elif [ ${#NVIDIA_GPUS[@]} -gt 0 ]; then + VENDOR=nvidia + elif [ ${#AMD_GPUS[@]} -gt 0 ]; then + VENDOR=amd + else + fail "no NVIDIA or AMD GPU found on the PCI bus" + VENDOR=none + fi + fi +} driver_of() { basename "$(readlink "$1/driver" 2>/dev/null)" 2>/dev/null @@ -191,11 +193,27 @@ amd_integrated() { [ -n "$rev" ] && [ "$rev" -ge 2 ] } +# amd_integrated_note PCI HAS_DISCRETE +# What to do in setup-gpu-nodes.sh's integrated list for an integrated GPU. +# A node with a discrete card stays unticked: ticked, it renders on the +# integrated GPU and leaves the card unused. +amd_integrated_note() { + if [ "$2" = true ]; then + echo "$1 is integrated, and this node has a discrete AMD GPU: leave it unticked in setup-gpu-nodes.sh's integrated list so it renders on the discrete card" + else + echo "$1 is integrated and the only AMD GPU: tick this node in setup-gpu-nodes.sh's integrated list (5stack-gpu-integrated=true)" + fi +} + check_amd() { - local dev pci driver node render integrated bar0 size level + local dev pci driver node render integrated bar0 size level has_discrete=false step "AMD GPU" + for dev in "${AMD_GPUS[@]}"; do + amd_integrated "$dev" || has_discrete=true + done + if [ -d /sys/module/amdgpu ]; then pass "amdgpu driver loaded" else @@ -231,7 +249,7 @@ check_amd() { fi if [ "$integrated" = true ]; then - info "$pci is integrated: select this node in setup-gpu-nodes.sh's integrated list (5stack-gpu-integrated=true)" + info "$(amd_integrated_note "$pci" "$has_discrete")" else # BAR0 is the CPU's window into VRAM. 256 MiB means Above 4G # decoding (and Resizable BAR) is off in the firmware. @@ -240,9 +258,6 @@ check_amd() { if [ "$size" -gt 0 ] && [ "$size" -le $((256 * 1024 * 1024)) ]; then warn "$pci: VRAM window is $((size / 1024 / 1024)) MiB; enable Above 4G decoding and Resizable BAR in the firmware" fi - if ! cmdline_has amdgpu.runpm=0; then - warn "$pci: a headless discrete GPU can drop off the bus in runtime suspend; add amdgpu.runpm=0 to the kernel command line${ON_HOST:+ of the host}" - fi fi level=$(cat "$dev/power_dpm_force_performance_level" 2>/dev/null) @@ -356,6 +371,29 @@ install_nvidia_userspace() { fi } +# The installer's output goes to /dev/null here, not on the apply_step call, +# which would swallow apply_step's own result line with it. +install_nvidia_driver() { + DEBIAN_FRONTEND=noninteractive ubuntu-drivers install /dev/null +} + +# nvidia_nodes_hint MISSING +# The nvidia-modprobe commands that create the MISSING device nodes on a +# machine that owns the GPU: -c0 makes nvidia0 and nvidiactl, -c0 -u the uvm +# nodes instead, and only -m makes nvidia-modeset, which a headless node +# without a display often lacks. +nvidia_nodes_hint() { + local missing="$1" commands=() + case "$missing" in *"/dev/nvidia0"* | *"/dev/nvidiactl"*) commands+=("nvidia-modprobe -c0") ;; esac + case "$missing" in *"/dev/nvidia-uvm"*) commands+=("nvidia-modprobe -c0 -u") ;; esac + case "$missing" in *"/dev/nvidia-modeset"*) commands+=("nvidia-modprobe -m") ;; esac + local joined="" command + for command in "${commands[@]}"; do + joined+="${joined:+; }$command" + done + echo "Create them with '$joined'. They are gone after a reboot, so run that at boot too (a oneshot systemd unit)." +} + install_nvidia_container_toolkit() { if [ "$OS_ID" != ubuntu ] && [ "$OS_ID" != debian ]; then fail "install nvidia-container-toolkit by hand on $OS_ID: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html" @@ -401,8 +439,7 @@ check_nvidia() { if [ -z "$kernel_version" ]; then fail "the NVIDIA kernel module is not loaded" if [ "$IN_LXC" = false ] && [ "$APPLY" = true ] && [ "$OS_ID" = ubuntu ] && command -v ubuntu-drivers >/dev/null 2>&1; then - apply_step "installed the recommended NVIDIA driver; reboot, then run this again" \ - env DEBIAN_FRONTEND=noninteractive ubuntu-drivers install /dev/null + apply_step "installed the recommended NVIDIA driver; reboot, then run this again" install_nvidia_driver elif [ "$IN_LXC" = false ] && [ "$OS_ID" = ubuntu ]; then hint "Install it with 'ubuntu-drivers install' (--apply does), then reboot." fi @@ -422,7 +459,7 @@ check_nvidia() { if [ "$IN_LXC" = true ]; then hint "Bind them into the container (proxmox-gpu-lxc.sh does), and create them on the host first." else - hint "'nvidia-modprobe -u -c0' creates them; nvidia-persistenced keeps them across boots." + hint "$(nvidia_nodes_hint "$missing")" fi fi @@ -486,23 +523,50 @@ k3s_service() { fi } -GOVERNOR_UNIT=/etc/systemd/system/cpu-governor-performance.service - +# The same script and unit the panel's CPU governor tuning installs, so a +# node that later gets that tuning keeps one unit, not two. install_governor_unit() { - cat >"$GOVERNOR_UNIT" <<'UNIT' || return 1 + local unit="$SYSTEMD_DIR/5stack-cpu-governor.service" governor + mkdir -p "${GOVERNOR_SCRIPT%/*}" || return 1 + cat >"$GOVERNOR_SCRIPT.5stack-new" <<'SCRIPT' || return 1 +#!/bin/sh +for governor in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do + echo performance > "$governor" +done +SCRIPT + chmod 0755 "$GOVERNOR_SCRIPT.5stack-new" && mv -f "$GOVERNOR_SCRIPT.5stack-new" "$GOVERNOR_SCRIPT" || return 1 + cat >"$unit.5stack-new" < "$g"; done' RemainAfterExit=yes +ExecStart=$GOVERNOR_SCRIPT [Install] WantedBy=multi-user.target UNIT - systemctl daemon-reload && systemctl enable --now cpu-governor-performance.service >/dev/null 2>&1 + mv -f "$unit.5stack-new" "$unit" || return 1 + # restart, not enable --now: --now leaves an active unit alone, and the + # governor may have been changed since it ran. + systemctl daemon-reload && + systemctl enable 5stack-cpu-governor.service >/dev/null 2>&1 && + systemctl restart 5stack-cpu-governor.service >/dev/null 2>&1 || return 1 + for governor in "$CPU_SYSFS"/cpu*/cpufreq/scaling_governor; do + [ -f "$governor" ] || continue + if [ "$(cat "$governor")" != performance ]; then + hint "$governor is still $(cat "$governor")" + return 1 + fi + done +} + +# k3s_join_hint VERSION +# The panel's join command with the k3s version pinned. sudo drops exported +# variables, so the pin goes after sudo, where it reaches the k3s installer. +k3s_join_hint() { + echo "curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=$1 sh -s -- join --panel-key " } check_common() { @@ -528,7 +592,7 @@ check_common() { fi local governor - governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null) + governor=$(cat "$CPU_SYSFS/cpu0/cpufreq/scaling_governor" 2>/dev/null) if [ -z "$governor" ]; then info "no CPU frequency scaling exposed" elif [ "$governor" = performance ]; then @@ -536,7 +600,7 @@ check_common() { elif [ "$IN_LXC" = true ]; then warn "CPU governor is '$governor'; set it to performance on the host (proxmox-gpu-lxc.sh --apply does)" elif [ "$APPLY" = true ]; then - apply_step "CPU governor set to performance, at every boot ($GOVERNOR_UNIT)" install_governor_unit + apply_step "CPU governor set to performance, at every boot (5stack-cpu-governor.service)" install_governor_unit else warn "CPU governor is '$governor'; render pods drop frames on a ramping CPU (--apply sets performance at every boot)" fi @@ -548,7 +612,7 @@ check_common() { if [ -n "$K3S_VERSION" ] && [ "$installed" != "$K3S_VERSION" ] && [ "$(printf '%s\n' "$installed" "$K3S_VERSION" | sort -V | tail -1)" = "$installed" ]; then warn "k3s $installed is newer than the panel's $K3S_VERSION; a kubelet newer than its API server is unsupported" - hint "Re-join with: export INSTALL_K3S_VERSION=$K3S_VERSION, then the panel's node setup command." + hint "Re-join with the version pinned: $(k3s_join_hint "$K3S_VERSION")" else pass "k3s $installed installed" fi @@ -558,12 +622,8 @@ check_common() { fail "$service is not running; see 'journalctl -u $service -n 50'" fi else - info "k3s not installed yet. The node setup command installs the latest k3s; to match the panel, first run:" - if [ -n "$K3S_VERSION" ]; then - hint "export INSTALL_K3S_VERSION=$K3S_VERSION" - else - hint "export INSTALL_K3S_VERSION=" - fi + info "k3s not installed yet. The join command installs the latest k3s; to match the panel, join with the version pinned:" + hint "$(k3s_join_hint "${K3S_VERSION:-}")" fi } @@ -596,10 +656,10 @@ check_lxc() { fail "/dev/net/tun missing; Tailscale needs it" fi - if [ -w /proc/sys/net/ipv4/ip_forward ]; then - pass "/proc/sys writable" + if [ -w "$PROC_SYS_PROBE" ]; then + pass "/proc/sys/vm writable" else - fail "/proc/sys is read-only; set lxc.mount.auto: proc:rw sys:rw" + fail "/proc/sys/vm is read-only, so the kubelet cannot set its kernel values; set lxc.mount.auto: proc:rw sys:rw on the host" fi local controllers controller @@ -638,18 +698,39 @@ check_lxc() { fi } -[ "$IN_LXC" = true ] && check_lxc -case "$VENDOR" in - amd) check_amd ;; - nvidia) check_nvidia ;; -esac -check_common - -echo -if [ "$FAILS" -gt 0 ]; then - echo "${C_ERR}$FAILS failed, $WARNS warnings${C_RESET}" - [ "$APPLY" = false ] && echo "Run with --apply to fix what this script can, then run it again." - exit 1 +main() { + parse_args "$@" + + if [ "$APPLY" = true ] && [ "$(id -u)" -ne 0 ]; then + echo "--apply must run as root" >&2 + exit 2 + fi + + # shellcheck source=/dev/null + . /etc/os-release 2>/dev/null + OS_ID="${ID:-unknown}" + OS_VERSION="${VERSION_ID:-}" + + check_machine + [ "$IN_LXC" = true ] && check_lxc + case "$VENDOR" in + amd) check_amd ;; + nvidia) check_nvidia ;; + esac + check_common + + echo + if [ "$FAILS" -gt 0 ]; then + echo "${C_ERR}$FAILS failed, $WARNS warnings${C_RESET}" + [ "$APPLY" = false ] && echo "Run with --apply to fix what this script can, then run it again." + exit 1 + fi + echo "${C_OK}ready for the ${VENDOR} GPU setup ($WARNS warnings)${C_RESET}" + echo "Next: join this node from the panel's Game Server Nodes page, then run setup-gpu-nodes.sh on the panel." +} + +# The tests source this file for its functions. Comparing BASH_SOURCE with $0 +# cannot tell that apart from "curl ... | bash", where both differ. +if [ -z "${GPU_NODE_CHECK_LIB:-}" ]; then + main "$@" fi -echo "${C_OK}ready for the ${VENDOR} GPU setup ($WARNS warnings)${C_RESET}" -echo "Next: run the panel's node setup command here, then setup-gpu-nodes.sh on the panel." diff --git a/tests/scripts/gpu_node_check_test.sh b/tests/scripts/gpu_node_check_test.sh new file mode 100644 index 0000000..fb66f01 --- /dev/null +++ b/tests/scripts/gpu_node_check_test.sh @@ -0,0 +1,250 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +load() { + # shellcheck source=gpu-node/gpu-node-check.sh + GPU_NODE_CHECK_LIB=1 source "$TEST_ROOT/gpu-node/gpu-node-check.sh" + # The script has its own fail(); the asserts need the one that exits. + SCRIPT_FAIL=$(declare -f fail) + # shellcheck source=tests/scripts/lib.sh + source "$TEST_ROOT/tests/scripts/lib.sh" +} + +# script FUNCTION ARGS... +# Runs one of the script's functions with the script's own fail(), which +# records a [FAIL] line instead of ending the test. +script() { + ( + eval "$SCRIPT_FAIL" + "$@" + ) +} + +# fake_amd_gpu NAME integrated|discrete +# A PCI device directory as sysfs shows an amdgpu device, with a render node. +fake_amd_gpu() { + local dev="$WORK_DIR/pci/$1" + mkdir -p "$dev/drm/renderD128" "$dev/hwmon/hwmon0" + ln -s "$WORK_DIR/drivers/amdgpu" "$dev/driver" + mkdir -p "$WORK_DIR/drivers/amdgpu" + if [ "$2" = integrated ]; then + echo vddnb >"$dev/hwmon/hwmon0/in1_label" + else + echo vddgfx >"$dev/hwmon/hwmon0/in1_label" + fi + echo "$dev" +} + +test_the_script_runs_nothing_when_sourced() { + local out + out=$(GPU_NODE_CHECK_LIB=1 bash -c "source '$TEST_ROOT/gpu-node/gpu-node-check.sh'; echo sourced" 2>&1) + assert_eq "$out" "sourced" +} + +test_a_bad_vendor_is_a_usage_error() { + load + local status + (parse_args --vendor intel) 2>/dev/null + status=$? + assert_eq "$status" "2" + (parse_args --bogus) 2>/dev/null + status=$? + assert_eq "$status" "2" +} + +test_an_apu_is_recognised_by_its_vddnb_rail() { + load + local apu card + apu=$(fake_amd_gpu 0000:c5:00.0 integrated) + card=$(fake_amd_gpu 0000:03:00.0 discrete) + amd_integrated "$apu" || fail "the APU was not seen as integrated" + if amd_integrated "$card"; then + fail "the discrete card was seen as integrated" + fi +} + +test_an_apu_is_recognised_by_its_gpu_metrics_revision() { + load + local dev="$WORK_DIR/pci/0000:c5:00.0" + mkdir -p "$dev" + printf '\x00\x00\x02\x00' >"$dev/gpu_metrics" + amd_integrated "$dev" || fail "gpu_metrics revision 2 was not seen as integrated" + printf '\x00\x00\x01\x00' >"$dev/gpu_metrics" + if amd_integrated "$dev"; then + fail "gpu_metrics revision 1 was seen as integrated" + fi +} + +test_the_integrated_note_depends_on_a_discrete_card() { + load + local alone beside + alone=$(amd_integrated_note 0000:c5:00.0 false) + beside=$(amd_integrated_note 0000:c5:00.0 true) + assert_contains "$alone" "tick this node" + assert_contains "$beside" "leave it unticked" + assert_contains "$beside" "renders on the discrete card" + assert_not_contains "$beside" "tick this node" +} + +# check_amd reads the kernel log and /proc/cmdline; these keep it on the +# fake devices. +quiet_amd_host() { + IN_LXC=true + # shellcheck disable=SC2317 # called by the function under test + kernel_log() { :; } + # shellcheck disable=SC2317 # called by the function under test + cmdline_has() { return 1; } +} + +test_an_apu_beside_a_discrete_card_stays_unticked() { + load + quiet_amd_host + AMD_GPUS=("$(fake_amd_gpu 0000:c5:00.0 integrated)" "$(fake_amd_gpu 0000:03:00.0 discrete)") + local out + out=$(script check_amd) + assert_contains "$out" "0000:c5:00.0 is integrated, and this node has a discrete AMD GPU: leave it unticked" + assert_not_contains "$out" "tick this node" + assert_not_contains "$out" "runpm" +} + +test_an_apu_on_its_own_is_ticked() { + load + quiet_amd_host + AMD_GPUS=("$(fake_amd_gpu 0000:c5:00.0 integrated)") + local out + out=$(script check_amd) + assert_contains "$out" "0000:c5:00.0 is integrated and the only AMD GPU: tick this node" + assert_not_contains "$out" "runpm" +} + +test_the_nvidia_node_hint_uses_m_for_modeset() { + load + local out + out=$(nvidia_nodes_hint " /dev/nvidia-modeset") + assert_contains "$out" "'nvidia-modprobe -m'" + assert_not_contains "$out" "-c0" + out=$(nvidia_nodes_hint " /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-modeset /dev/nvidia0") + assert_contains "$out" "'nvidia-modprobe -c0; nvidia-modprobe -c0 -u; nvidia-modprobe -m'" + out=$(nvidia_nodes_hint " /dev/nvidia-uvm") + assert_contains "$out" "'nvidia-modprobe -c0 -u'" + assert_not_contains "$out" "modprobe -m" +} + +test_the_k3s_pin_goes_after_sudo() { + load + assert_eq "$(k3s_join_hint v1.36.5+k3s1)" \ + "curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join --panel-key " +} + +test_without_k3s_the_join_hint_has_the_pin() { + load + # shellcheck disable=SC2317 # called by the function under test + command() { + if [ "$1" = -v ] && [ "$2" = k3s ]; then return 1; fi + builtin command "$@" + } + # shellcheck disable=SC2317 # called by the function under test + df() { printf 'Filesystem 1G-blocks Used Available Use%% Mounted on\n/dev/root 500G 10G 490G 2%% /\n'; } + CPU_SYSFS="$WORK_DIR/cpu" + K3S_VERSION=v1.36.5+k3s1 + local out + out=$(script check_common) + assert_contains "$out" "sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join" + assert_not_contains "$out" "INSTALL_K3S_VERSION=v1.36.5+k3s1 curl" +} + +# ubuntu-drivers on PATH, writing to stdout as the real one does. +shim_ubuntu_drivers() { + mkdir -p "$WORK_DIR/bin" + cat >"$WORK_DIR/bin/ubuntu-drivers" <"$WORK_DIR/overcommit_memory" + chmod 0444 "$WORK_DIR/overcommit_memory" + PROC_SYS_PROBE="$WORK_DIR/overcommit_memory" + local out + out=$(script check_lxc) + assert_contains "$out" "[FAIL] /proc/sys/vm is read-only" + assert_not_contains "$out" "[PASS] /proc/sys/vm writable" + + chmod 0644 "$WORK_DIR/overcommit_memory" + out=$(script check_lxc) + assert_contains "$out" "[PASS] /proc/sys/vm writable" +} + +# systemctl for the governor unit: a restart either sets every governor or, +# like an already-active unit that enable --now skips, changes nothing. +stub_governor() { + mkdir -p "$WORK_DIR/cpu/cpu0/cpufreq" "$WORK_DIR/cpu/cpu1/cpufreq" "$WORK_DIR/systemd" + echo powersave >"$WORK_DIR/cpu/cpu0/cpufreq/scaling_governor" + echo powersave >"$WORK_DIR/cpu/cpu1/cpufreq/scaling_governor" + CPU_SYSFS="$WORK_DIR/cpu" + SYSTEMD_DIR="$WORK_DIR/systemd" + GOVERNOR_SCRIPT="$WORK_DIR/bin/5stack-cpu-governor.sh" + RESTART_SETS_GOVERNOR="$1" + systemctl() { + echo "systemctl $*" >>"$WORK_DIR/systemctl.log" + if [ "$1" = restart ] && [ "$RESTART_SETS_GOVERNOR" = true ]; then + local g + for g in "$CPU_SYSFS"/cpu*/cpufreq/scaling_governor; do echo performance >"$g"; done + fi + return 0 + } +} + +test_the_governor_unit_is_the_panels_and_restarted() { + load + stub_governor true + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + assert_contains "$(cat "$WORK_DIR/systemd/5stack-cpu-governor.service")" "ExecStart=$GOVERNOR_SCRIPT" + # shellcheck disable=SC2016 # the literal line the unit's script holds + assert_contains "$(cat "$GOVERNOR_SCRIPT")" 'echo performance > "$governor"' + assert_contains "$(cat "$WORK_DIR/systemctl.log")" "systemctl restart 5stack-cpu-governor.service" + assert_not_contains "$(cat "$WORK_DIR/systemctl.log")" "--now" + [ -e "$WORK_DIR/systemd/5stack-cpu-governor.service.5stack-new" ] && fail "the temp unit was left behind" + return 0 +} + +test_the_governor_fix_fails_when_the_governor_did_not_change() { + load + stub_governor false + local out + out=$(script apply_step "CPU governor set to performance" install_governor_unit) + assert_contains "$out" "[FAIL] could not apply: CPU governor set to performance" + assert_contains "$out" "is still powersave" + assert_not_contains "$out" "[FIX ]" +} + +run_tests From a4b38dea9b15b799d1c1c4add4e4db63277ed59d Mon Sep 17 00:00:00 2001 From: Flegma Date: Mon, 2 Nov 2026 23:36:18 +0100 Subject: [PATCH 04/10] docs: the GPU node README follows the current panel flows - How to get the scripts onto a GPU node or Proxmox host. - The k3s pin goes after sudo in the get.5stack.gg join command. - Nodes are created on the Game Server Nodes page; setup-gpu-nodes.sh runs as sudo 5stack run setup-gpu-nodes.sh on a service install. - Only the two check scripts take --apply. - The proc:rw host-wide warning, the 120 GB free disk rule, the discrete-card rule for the integrated list, nvidia-modprobe -m, and no amdgpu.runpm=0. --- gpu-node/README.md | 80 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 24 deletions(-) diff --git a/gpu-node/README.md b/gpu-node/README.md index 2cfe422..fc18408 100644 --- a/gpu-node/README.md +++ b/gpu-node/README.md @@ -10,24 +10,37 @@ GPUs are both supported, one vendor per panel (`GPU_VENDOR`). | `proxmox-gpu-lxc.sh CTID` | the Proxmox host | checks the host driver and kernel modules and the container's settings; `--apply` writes them | | `setup-gpu-nodes.sh` | the panel | labels the GPU nodes for the panel's vendor and turns that vendor's support on | -Every script only reports unless you pass `--apply`, and exits 1 while -something still fails. None of them change the GPU's power or clock settings. +The two check scripts only report unless you pass `--apply`, and exit 1 while +something still fails. None of these scripts change the GPU's power or clock +settings. + +## Getting the scripts + +A GPU node or a Proxmox host has no panel checkout, so download the script you +need there: + +```bash +curl -fsSLO https://raw.githubusercontent.com/5stackgg/5stack-panel/main/gpu-node/gpu-node-check.sh +curl -fsSLO https://raw.githubusercontent.com/5stackgg/5stack-panel/main/gpu-node/proxmox-gpu-lxc.sh +``` ## Before you start -- **k3s version.** The node setup command installs the latest k3s, and a - kubelet newer than the panel's API server is unsupported. If the panel runs - an older k3s, read its version with `kubectl version` on the panel - (`Server Version`) and export it on the node before the setup command: +- **k3s version.** The join command installs the latest k3s, and a kubelet + newer than the panel's API server is unsupported. If the panel runs an older + k3s, read its version with `kubectl version` on the panel (`Server Version`) + and put it after `sudo` in the join command, since `sudo` drops a variable + exported before it: ```bash - export INSTALL_K3S_VERSION=v1.36.5+k3s1 + curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join --panel-key ``` -- **Disk.** The node setup command wants 120 GB free on `/`. +- **Disk.** The join wants 120 GB free on `/`, on top of what the system uses. - **CPU governor.** Render pods drop frames while the CPU ramps up, so the - governor is set to `performance`. In an LXC container that happens on the - Proxmox host. + governor is set to `performance` with `5stack-cpu-governor.service`, the + unit the panel's CPU governor tuning installs. In an LXC container that + happens on the Proxmox host. ## Dedicated machine or VM @@ -40,11 +53,14 @@ something still fails. None of them change the GPU's power or clock settings. sudo bash gpu-node-check.sh --apply --k3s-version v1.36.5+k3s1 ``` -3. Export `INSTALL_K3S_VERSION`, run the node setup command from the panel - (Settings, Nodes), and run the check again: some checks only apply once k3s - is there. -4. On the panel, run `./setup-gpu-nodes.sh` and pick the vendor and nodes. - All GPU nodes of a panel use the same vendor. +3. Join the node: on the panel's **Game Server Nodes** page + (`/game-server-nodes`), create a node and run the command it shows on this + machine, with `INSTALL_K3S_VERSION` after `sudo` as above. Then run the check + again: some checks only apply once k3s is there. +4. On the panel, run `setup-gpu-nodes.sh` and pick the vendor and nodes: from + the panel checkout as `./setup-gpu-nodes.sh`, or on a panel that runs as the + 5Stack service, `sudo 5stack run setup-gpu-nodes.sh`. All GPU nodes of a + panel use the same vendor. ### NVIDIA @@ -54,18 +70,23 @@ something still fails. None of them change the GPU's power or clock settings. - With Secure Boot on, use Ubuntu's signed prebuilt driver modules (`ubuntu-drivers install` picks them) or enroll the DKMS key with `mokutil`. - After a driver upgrade, `nvidia-smi` fails until the next reboot. +- A headless node may have no `/dev/nvidia-modeset`; `nvidia-modprobe -m` + creates it. ### AMD - `--apply` installs the AMD firmware package. - On Ubuntu 24.04, use the HWE kernel (`linux-generic-hwe-24.04`); the original 6.8 kernel predates recent Radeons. -- Never boot with `nomodeset`, `amdgpu.modeset=0` or `amdgpu.dpm=0`. -- For a discrete card: enable Above 4G decoding and Resizable BAR in the - firmware, and add `amdgpu.runpm=0` to the kernel command line so a headless - card cannot suspend off the bus. -- For an APU (such as Ryzen AI Max), select the node in the integrated list in - `setup-gpu-nodes.sh`. The check says when a GPU is integrated. +- Never boot with `nomodeset`, `amdgpu.modeset=0` or `amdgpu.dpm=0`. There is + no kernel parameter to add. +- For a discrete card, enable Above 4G decoding and Resizable BAR in the + firmware. +- Tick a node in the integrated list of `setup-gpu-nodes.sh` only when its + integrated GPU (an APU, such as Ryzen AI Max) is its only AMD GPU. Leave a + node with a discrete Radeon card unticked, even when its CPU also has an + integrated GPU: ticked, it renders on the integrated GPU and leaves the card + unused. The check says which case a node is. ## Proxmox LXC container @@ -75,8 +96,9 @@ the device with the container; do not bind the GPU to `vfio-pci`. 1. On the host, install the GPU driver (NVIDIA: the proxmox headers and NVIDIA's `.run` installer; AMD: the kernel driver is built in, firmware comes from `pve-firmware`). -2. Create a privileged Ubuntu container with at least 120 GB of disk and no core - limit (the kubelet pins pods to CPUs by number and sees every host CPU). +2. Create a privileged Ubuntu container with at least 140 GB of disk (120 GB + free for the join, plus the system) and no core limit (the kubelet pins pods + to CPUs by number and sees every host CPU). 3. On the host: ```bash @@ -87,7 +109,10 @@ the device with the container; do not bind the GPU to `vfio-pci`. `--apply` loads the kernel modules k3s needs (and at every boot), sets the CPU governor, creates the NVIDIA device nodes at boot, turns on nesting, and adds - these lines to `/etc/pve/lxc/120.conf` (after backing it up to `/root`): + these lines to `/etc/pve/lxc/120.conf`. It backs the config up to `/root` + first and changes nothing in the container if that fails, writes the new + config beside the old one and renames it over, and stops at an unprivileged + container: ``` lxc.apparmor.profile: unconfined @@ -103,6 +128,13 @@ the device with the container; do not bind the GPU to `vfio-pci`. # nvidia-modeset and nvidia-caps, the same way ``` + > **`proc:rw` changes the whole host.** It lets the kubelet set + > `vm.overcommit_memory=1`, `kernel.panic=10` and `kernel.panic_on_oops=1`, + > which are not per container: the Proxmox host then reboots 10 seconds + > after a kernel oops, and every guest on it goes down. On a host shared + > with other guests, decide whether that is acceptable before you restart + > the container. `--apply` prints this warning too. + 4. Inside the container, run `gpu-node-check.sh --apply`. For NVIDIA it installs the host driver's exact version of the NVIDIA libraries, without the kernel module, plus the container toolkit. From f7f0f67b50d47e50a7d91c7d7c75d673d6973e52 Mon Sep 17 00:00:00 2001 From: Flegma Date: Mon, 2 Nov 2026 23:53:11 +0100 Subject: [PATCH 05/10] fix: proxmox-gpu-lxc.sh review fixes - A running container with CS2 installed needs the existing install minimum (60 GB) free, not the fresh one; the disk messages name both panel settings. - The governor unit also sets GOVERNOR="performance" in /etc/default/cpufrequtils when that file exists, as the panel's CPU governor tuning does. - --apply with a missing or unprivileged container turns itself off before any host step, so a run that cannot finish changes nothing. - A failed pct set or LXC line change no longer asks for a restart. - Without nvidia-modprobe, --apply says it cannot add the device node unit instead of promising one. - The host governor check reads CPU_SYSFS like the rest of the script. --- gpu-node/proxmox-gpu-lxc.sh | 92 ++++++++++--- tests/scripts/proxmox_gpu_lxc_test.sh | 185 +++++++++++++++++++++++++- 2 files changed, 254 insertions(+), 23 deletions(-) diff --git a/gpu-node/proxmox-gpu-lxc.sh b/gpu-node/proxmox-gpu-lxc.sh index 7a3f49c..b43f3a3 100755 --- a/gpu-node/proxmox-gpu-lxc.sh +++ b/gpu-node/proxmox-gpu-lxc.sh @@ -19,12 +19,18 @@ BACKUP_DIR="${BACKUP_DIR:-/root}" SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}" GOVERNOR_SCRIPT="${GOVERNOR_SCRIPT:-/usr/local/bin/5stack-cpu-governor.sh}" CPU_SYSFS="${CPU_SYSFS:-/sys/devices/system/cpu}" +CPUFREQUTILS_DEFAULT="${CPUFREQUTILS_DEFAULT:-/etc/default/cpufrequtils}" PROC_CMDLINE="${PROC_CMDLINE:-/proc/cmdline}" +NVIDIA_PROC_VERSION="${NVIDIA_PROC_VERSION:-/proc/driver/nvidia/version}" -# The node setup command stops unless / has this much free (df -BG), and the -# system, k3s and the render images need room on top of it. +# The node setup command stops unless / has this much free (df -BG): the +# panel's "Required disk space - fresh install" setting before CS2 is +# installed, "Required disk space - existing install" once it is. These are +# their defaults. The system, k3s and the render images need room on top. REQUIRED_FREE_GB=120 +REQUIRED_FREE_EXISTING_GB=60 SYSTEM_HEADROOM_GB=20 +CS2_DIR=/opt/5stack/serverfiles/game/csgo # The kernel modules k3s, flannel and kube-proxy load. A container cannot load # modules, so the host loads them, now and at every boot. @@ -53,7 +59,8 @@ usage() { } # apply_step DESCRIPTION COMMAND... -# Runs an --apply step and reports it as fixed or failed. +# Runs an --apply step and reports it as fixed or failed, returning 1 when it +# failed. apply_step() { local description="$1" shift @@ -61,6 +68,7 @@ apply_step() { fixed "$description" else fail "could not apply: $description" + return 1 fi } @@ -287,7 +295,7 @@ NVIDIA_NODES="/dev/nvidia0 /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-uvm-tools check_host_nvidia() { local version - version=$(head -1 /proc/driver/nvidia/version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1) + version=$(head -1 "$NVIDIA_PROC_VERSION" 2>/dev/null | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1) if [ -z "$version" ]; then fail "the NVIDIA driver is not loaded on the host" hint "Install it on the host with the proxmox-headers package and NVIDIA's .run installer, then reboot." @@ -305,6 +313,13 @@ check_host_nvidia() { pass "device nodes present, and created at every boot" elif [ "$APPLY" = true ] && command -v nvidia-modprobe >/dev/null 2>&1; then apply_step "device nodes created, and at every boot ($NVIDIA_UNIT)" install_nvidia_unit + elif [ "$APPLY" = true ]; then + if [ -n "$missing" ]; then + fail "missing device nodes:$missing, and --apply cannot create them without nvidia-modprobe" + else + warn "nothing creates the NVIDIA device nodes at boot, and --apply cannot add the unit that does without nvidia-modprobe" + fi + hint "NVIDIA's .run installer installs nvidia-modprobe; install the driver with it, then run this again." elif [ -n "$missing" ]; then fail "missing device nodes:$missing (--apply creates them at every boot with nvidia-modprobe)" else @@ -378,7 +393,7 @@ check_host_kernel() { fi local governor - governor=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null) + governor=$(cat "$CPU_SYSFS/cpu0/cpufreq/scaling_governor" 2>/dev/null) if [ -z "$governor" ] || [ "$governor" = performance ]; then pass "CPU governor ${governor:-not exposed}" elif [ "$APPLY" = true ]; then @@ -389,7 +404,8 @@ check_host_kernel() { } # The same script and unit the panel's CPU governor tuning installs on a -# node, so a machine never ends up with two of them. +# node, so a machine never ends up with two of them, and the same change to +# cpufrequtils's settings, whose boot script would set its own governor. install_governor_unit() { local unit="$SYSTEMD_DIR/5stack-cpu-governor.service" governor mkdir -p "${GOVERNOR_SCRIPT%/*}" || return 1 @@ -413,6 +429,10 @@ ExecStart=$GOVERNOR_SCRIPT WantedBy=multi-user.target UNIT mv -f "$unit.5stack-new" "$unit" || return 1 + if [ -f "$CPUFREQUTILS_DEFAULT" ]; then + { grep -v '^GOVERNOR=' "$CPUFREQUTILS_DEFAULT" || true; echo 'GOVERNOR="performance"'; } >"$CPUFREQUTILS_DEFAULT.5stack-new" && + mv -f "$CPUFREQUTILS_DEFAULT.5stack-new" "$CPUFREQUTILS_DEFAULT" || return 1 + fi # restart, not enable --now: --now leaves an active unit alone, and the # governor may have been changed since it ran. systemctl daemon-reload && @@ -435,9 +455,21 @@ container_free_gb() { pct exec "$CTID" -- df -BG / 2>/dev/null | awk 'NR == 2 { sub(/G$/, "", $4); print $4 }' } +# container_has_cs2 +# True when CS2 is installed in the running container, where the node setup +# looks for it. +container_has_cs2() { + pct exec "$CTID" -- test -d "$CS2_DIR" 2>/dev/null +} + +disk_settings_hint() { + hint "The panel sets both minimums: \"Required disk space - fresh install\" ($REQUIRED_FREE_GB GB by default) and \"Required disk space - existing install\" ($REQUIRED_FREE_EXISTING_GB GB), under Settings, Application, Servers." +} + # check_disk ROOTFS -# The node setup needs REQUIRED_FREE_GB free on /, so a running container is -# judged by its free space, and a stopped one by its size. +# The node setup needs REQUIRED_FREE_GB free on /, or REQUIRED_FREE_EXISTING_GB +# once CS2 is installed, so a running container is judged by its free space, +# and a stopped one by its size. check_disk() { local rootfs="$1" size gb free want size=$(echo "$rootfs" | grep -oE 'size=[0-9.]+[KMGT]?' | cut -d= -f2) @@ -451,15 +483,23 @@ check_disk() { case "$free" in *[!0-9]*) free="" ;; esac if [ -n "$free" ]; then - if [ "$free" -ge "$REQUIRED_FREE_GB" ]; then - pass "root disk $size, ${free} GB free" + local need=$REQUIRED_FREE_GB when="before CS2 is installed" + if container_has_cs2; then + need=$REQUIRED_FREE_EXISTING_GB + when="with CS2 already installed" + fi + if [ "$free" -ge "$need" ]; then + pass "root disk $size, ${free} GB free ($when the node setup needs $need GB)" else - fail "root disk $size has ${free} GB free; the node setup needs $REQUIRED_FREE_GB GB free" - hint "Grow it: pct resize $CTID rootfs +$((REQUIRED_FREE_GB - free + SYSTEM_HEADROOM_GB))G" + fail "root disk $size has ${free} GB free; $when the node setup needs $need GB free" + hint "Grow it: pct resize $CTID rootfs +$((need - free + SYSTEM_HEADROOM_GB))G" + disk_settings_hint fi elif [ "$gb" -lt "$REQUIRED_FREE_GB" ]; then - fail "root disk is $size; the node setup needs $REQUIRED_FREE_GB GB free on it" + fail "root disk is $size; before CS2 is installed the node setup needs $REQUIRED_FREE_GB GB free on it" hint "Grow it: pct resize $CTID rootfs ${want}G" + hint "If CS2 is already installed in it, start the container and run this again: $REQUIRED_FREE_EXISTING_GB GB free is then enough." + disk_settings_hint elif [ "$gb" -lt "$want" ]; then warn "root disk is $size; the node setup needs $REQUIRED_FREE_GB GB free, which leaves little for the system (${want}G is safer)" hint "Start the container and run this again to check its free space." @@ -530,8 +570,8 @@ check_container() { RESTART_NEEDED=true elif [ "$APPLY" = true ]; then backup_conf "$conf" || return - apply_step "enabled nesting" pct set "$CTID" --features "$(features_with_nesting "$features")" - RESTART_NEEDED=true + apply_step "enabled nesting" pct set "$CTID" --features "$(features_with_nesting "$features")" && + RESTART_NEEDED=true else fail "nesting is off (--apply turns it on)" fi @@ -550,8 +590,8 @@ check_container() { done if [ "$APPLY" = true ]; then backup_conf "$conf" || return - apply_step "added ${#missing[@]} LXC lines for k3s and the $VENDOR GPU" locked_add_lxc_lines "$conf" "${lines[@]}" - RESTART_NEEDED=true + apply_step "added ${#missing[@]} LXC lines for k3s and the $VENDOR GPU" locked_add_lxc_lines "$conf" "${lines[@]}" && + RESTART_NEEDED=true else fail "${#missing[@]} LXC lines for k3s and the $VENDOR GPU are missing (--apply adds them):" for line in "${missing[@]}"; do @@ -577,6 +617,21 @@ backup_conf() { return 1 } +# preflight_apply +# --apply changes the host only for a container it can then finish, so a +# missing or unprivileged container turns it off before any host step runs. +APPLY_REFUSED=false +preflight_apply() { + local conf="$PVE_LXC_DIR/$CTID.conf" + [ "$APPLY" = true ] || return 0 + if [ -f "$conf" ] && [ "$(conf_value "$conf" unprivileged)" != 1 ]; then + return 0 + fi + APPLY=false + APPLY_REFUSED=true + echo "${C_WARN}--apply changes nothing on the host or in container $CTID until the container exists and is privileged; only checking${C_RESET}" +} + main() { while [ $# -gt 0 ]; do case "$1" in @@ -609,6 +664,7 @@ main() { fi RESTART_NEEDED=false + preflight_apply check_host_gpu check_host_kernel check_container @@ -619,7 +675,7 @@ main() { fi if [ "$FAILS" -gt 0 ]; then echo "${C_ERR}$FAILS failed, $WARNS warnings${C_RESET}" - [ "$APPLY" = false ] && echo "Run with --apply to fix what this script can, then run it again." + [ "$APPLY" = false ] && [ "$APPLY_REFUSED" = false ] && echo "Run with --apply to fix what this script can, then run it again." exit 1 fi echo "${C_OK}host and container $CTID ready ($WARNS warnings)${C_RESET}" diff --git a/tests/scripts/proxmox_gpu_lxc_test.sh b/tests/scripts/proxmox_gpu_lxc_test.sh index e2e2051..7ec03e3 100644 --- a/tests/scripts/proxmox_gpu_lxc_test.sh +++ b/tests/scripts/proxmox_gpu_lxc_test.sh @@ -210,18 +210,27 @@ test_the_rewrite_replaces_the_file_by_rename() { } # pct as the container tests see it: STATUS is what pct status says, FREE the -# free GB df reports inside, and every other call is logged. +# free GB df reports inside, CS2 whether CS2 is installed there (true or +# false), and every call is logged. PCT_SET_FAILS=true makes pct set fail. stub_pct() { PCT_STATUS="$1" PCT_FREE="${2:-}" + PCT_CS2="${3:-false}" + PCT_SET_FAILS=false # shellcheck disable=SC2317 # called by the function under test pct() { echo "pct $*" >>"$WORK_DIR/pct.log" case "$1" in status) echo "status: $PCT_STATUS" ;; + set) [ "$PCT_SET_FAILS" != true ] ;; exec) - printf 'Filesystem 1G-blocks Used Available Use%% Mounted on\n' - printf '/dev/loop0 200G 10G %sG 10%% /\n' "$PCT_FREE" + case "$4" in + test) [ "$PCT_CS2" = true ] ;; + *) + printf 'Filesystem 1G-blocks Used Available Use%% Mounted on\n' + printf '/dev/loop0 200G 10G %sG 10%% /\n' "$PCT_FREE" + ;; + esac ;; esac } @@ -236,15 +245,34 @@ test_a_running_container_is_judged_by_its_free_space() { stub_pct running 100 local out out=$(script check_disk "local-lvm:vm-100-disk-0,size=128G") - assert_contains "$out" "[FAIL] root disk 128G has 100 GB free; the node setup needs 120 GB free" + assert_contains "$out" "[FAIL] root disk 128G has 100 GB free; before CS2 is installed the node setup needs 120 GB free" assert_contains "$out" "pct resize 100 rootfs +40G" + assert_contains "$out" '"Required disk space - fresh install" (120 GB by default) and "Required disk space - existing install" (60 GB)' + assert_contains "$(cat "$WORK_DIR/pct.log")" "pct exec 100 -- test -d /opt/5stack/serverfiles/game/csgo" +} + +# With CS2 installed the node setup takes the existing install minimum. +test_a_container_with_cs2_installed_needs_60_gb_free() { + load + CTID=100 + stub_pct running 100 true + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=128G")" "[PASS] root disk 128G, 100 GB free (with CS2 already installed the node setup needs 60 GB)" + stub_pct running 50 true + local out + out=$(script check_disk "local-lvm:vm-100-disk-0,size=128G") + assert_contains "$out" "[FAIL] root disk 128G has 50 GB free; with CS2 already installed the node setup needs 60 GB free" + assert_contains "$out" "pct resize 100 rootfs +30G" + assert_contains "$out" '"Required disk space - existing install" (60 GB)' } test_a_stopped_container_is_judged_by_its_size() { load CTID=100 stub_pct stopped - assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=100G")" "[FAIL] root disk is 100G" + local out + out=$(script check_disk "local-lvm:vm-100-disk-0,size=100G") + assert_contains "$out" "[FAIL] root disk is 100G" + assert_contains "$out" "If CS2 is already installed in it, start the container and run this again: 60 GB free is then enough." assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=120G")" "[WARN] root disk is 120G" assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=131072M")" "[WARN] root disk is 131072M" assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=150G")" "[PASS] root disk 150G" @@ -279,6 +307,84 @@ test_apply_stops_at_an_unprivileged_container() { assert_eq "$(find "$WORK_DIR/backup" -type f | wc -l)" "0" } +# The host steps run before the container's, so --apply has to stop before +# them, not only in check_container. +test_apply_turns_off_before_the_host_steps_for_an_unprivileged_container() { + load + write_container 1 + local out + out=$(preflight_apply) + assert_contains "$out" "--apply changes nothing on the host or in container 100 until the container exists and is privileged" + preflight_apply >/dev/null + assert_eq "$APPLY" "false" + assert_eq "$APPLY_REFUSED" "true" + + MODULES_FILE="$WORK_DIR/5stack-k3s.conf" + # shellcheck disable=SC2317 # called by the function under test + modprobe() { echo "modprobe $*" >>"$WORK_DIR/modprobe.log"; } + stub_governor true + out=$(script check_host_kernel) + assert_not_contains "$out" "[FIX ]" + [ -e "$MODULES_FILE" ] && fail "the modules file was written" + [ -e "$WORK_DIR/modprobe.log" ] && fail "a module was loaded" + [ -e "$WORK_DIR/systemd/5stack-cpu-governor.service" ] && fail "the governor unit was installed" + return 0 +} + +# main as root on a Proxmox host, with an unprivileged container: the refusal +# comes before the first host step, and nothing on the host changes. +test_main_refuses_apply_before_the_host_steps() { + load + write_container 1 + stub_pct stopped + stub_governor true + MODULES_FILE="$WORK_DIR/5stack-k3s.conf" + # shellcheck disable=SC2317 # called by the function under test + id() { echo 0; } + # shellcheck disable=SC2317 # called by the function under test + modprobe() { echo "modprobe $*" >>"$WORK_DIR/modprobe.log"; } + local out + out=$( (main 100 --apply --vendor amd) 2>&1) + assert_contains "$out" "--apply changes nothing on the host or in container 100" + assert_contains "$(echo "$out" | sed -n '1,/==> Host GPU/p')" "--apply changes nothing on the host" + assert_not_contains "$out" "[FIX ]" + assert_not_contains "$out" "Run with --apply" + [ -e "$MODULES_FILE" ] && fail "the modules file was written" + [ -e "$WORK_DIR/modprobe.log" ] && fail "a module was loaded" + [ -e "$WORK_DIR/systemd/5stack-cpu-governor.service" ] && fail "the governor unit was installed" + assert_eq "$(cat "$WORK_DIR/lxc/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" +} + +test_apply_goes_ahead_for_a_privileged_container() { + load + write_container 0 + preflight_apply >"$WORK_DIR/out" + assert_eq "$(cat "$WORK_DIR/out")" "" + assert_eq "$APPLY" "true" + CTID=101 + preflight_apply >/dev/null + assert_eq "$APPLY" "false" +} + +test_a_failed_change_asks_for_no_restart() { + load + write_container 0 + stub_pct stopped + PCT_SET_FAILS=true + # shellcheck disable=SC2317 # called by the function under test + locked_add_lxc_lines() { return 1; } + local out + out=$( + eval "$SCRIPT_FAIL" + RESTART_NEEDED=false + check_container + echo "RESTART_NEEDED=$RESTART_NEEDED" + ) + assert_contains "$out" "[FAIL] could not apply: enabled nesting" + assert_contains "$out" "[FAIL] could not apply: added 8 LXC lines" + assert_contains "$out" "RESTART_NEEDED=false" +} + test_apply_backs_up_then_adds_the_lines_and_warns_about_proc_rw() { load write_container 0 @@ -321,6 +427,43 @@ test_report_mode_lists_the_lines_and_warns_about_proc_rw() { assert_eq "$(cat "$WORK_DIR/lxc/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" } +# nvidia_host DRIVER_VERSION +# check_host_nvidia's view of a host with the driver loaded, no device nodes +# and no nvidia-modprobe. +nvidia_host() { + echo "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 $1 Release Build" >"$WORK_DIR/nvidia-version" + NVIDIA_PROC_VERSION="$WORK_DIR/nvidia-version" + NVIDIA_NODES="$WORK_DIR/dev/nvidia0 $WORK_DIR/dev/nvidiactl" + SYSTEMD_DIR="$WORK_DIR/systemd" + NVIDIA_UNIT="$SYSTEMD_DIR/5stack-nvidia-devices.service" + # shellcheck disable=SC2317 # called by the function under test + command() { + if [ "$1" = -v ] && [ "$2" = nvidia-modprobe ]; then return 1; fi + builtin command "$@" + } +} + +test_apply_without_nvidia_modprobe_says_it_cannot_create_the_nodes() { + load + nvidia_host 595.99.02 + APPLY=true + local out + out=$(script check_host_nvidia) + assert_contains "$out" "[PASS] NVIDIA driver 595.99.02" + assert_contains "$out" "[FAIL] missing device nodes: $WORK_DIR/dev/nvidia0 $WORK_DIR/dev/nvidiactl, and --apply cannot create them without nvidia-modprobe" + assert_contains "$out" "NVIDIA's .run installer installs nvidia-modprobe" + assert_not_contains "$out" "--apply creates them" + [ -e "$NVIDIA_UNIT" ] && fail "the unit was written" + return 0 +} + +test_report_mode_without_device_nodes_offers_apply() { + load + nvidia_host 595.99.02 + APPLY=false + assert_contains "$(script check_host_nvidia)" "(--apply creates them at every boot with nvidia-modprobe)" +} + test_nomodeset_only_fails_an_amd_host() { load echo "BOOT_IMAGE=/vmlinuz root=/dev/sda1 nomodeset quiet" >"$WORK_DIR/cmdline" @@ -339,6 +482,7 @@ stub_governor() { CPU_SYSFS="$WORK_DIR/cpu" SYSTEMD_DIR="$WORK_DIR/systemd" GOVERNOR_SCRIPT="$WORK_DIR/bin/5stack-cpu-governor.sh" + CPUFREQUTILS_DEFAULT="$WORK_DIR/cpufrequtils" RESTART_SETS_GOVERNOR="$1" systemctl() { echo "systemctl $*" >>"$WORK_DIR/systemctl.log" @@ -361,6 +505,37 @@ test_the_governor_unit_is_the_panels_and_restarted() { assert_not_contains "$(cat "$WORK_DIR/systemctl.log")" "--now" } +test_the_governor_check_reads_the_cpu_sysfs_it_is_given() { + load + stub_governor true + MODULES_FILE="$WORK_DIR/5stack-k3s.conf" + APPLY=false + assert_contains "$(script check_host_kernel)" "[WARN] CPU governor is 'powersave'" + echo performance >"$WORK_DIR/cpu/cpu0/cpufreq/scaling_governor" + assert_contains "$(script check_host_kernel)" "[PASS] CPU governor performance" +} + +# The panel's tuning points cpufrequtils's boot script at performance too. +test_the_governor_fix_sets_cpufrequtils_to_performance() { + load + stub_governor true + printf 'ENABLE="true"\nGOVERNOR="ondemand"\nMAX_SPEED="0"\n' >"$CPUFREQUTILS_DEFAULT" + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + assert_eq "$(cat "$CPUFREQUTILS_DEFAULT")" 'ENABLE="true" +MAX_SPEED="0" +GOVERNOR="performance"' + [ -e "$CPUFREQUTILS_DEFAULT.5stack-new" ] && fail "the temp file was left behind" + return 0 +} + +test_the_governor_fix_adds_no_cpufrequtils_settings() { + load + stub_governor true + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + [ -e "$CPUFREQUTILS_DEFAULT" ] && fail "cpufrequtils settings were created" + return 0 +} + test_the_governor_fix_fails_when_the_governor_did_not_change() { load stub_governor false From 615fc10196f3f5e74f6f77a43bc31f87216f178a Mon Sep 17 00:00:00 2001 From: Flegma Date: Mon, 2 Nov 2026 23:53:11 +0100 Subject: [PATCH 06/10] fix: gpu-node-check.sh review fixes - Only an AMD card bound to amdgpu counts as a discrete GPU for the integrated list advice; one on vfio-pci or without a driver does not. - The governor unit also sets GOVERNOR="performance" in /etc/default/cpufrequtils when that file exists, as the panel's CPU governor tuning does. - The k3s pin hint says it is untested: INSTALL_K3S_VERSION only takes effect if the join passes it through to the k3s installer. - A test drives install_nvidia_driver through check_nvidia. --- gpu-node/gpu-node-check.sh | 34 +++++++++---- tests/scripts/gpu_node_check_test.sh | 71 +++++++++++++++++++++++++--- 2 files changed, 90 insertions(+), 15 deletions(-) diff --git a/gpu-node/gpu-node-check.sh b/gpu-node/gpu-node-check.sh index 7fbe572..59de8da 100755 --- a/gpu-node/gpu-node-check.sh +++ b/gpu-node/gpu-node-check.sh @@ -25,6 +25,7 @@ AMD_GPUS=() SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}" GOVERNOR_SCRIPT="${GOVERNOR_SCRIPT:-/usr/local/bin/5stack-cpu-governor.sh}" CPU_SYSFS="${CPU_SYSFS:-/sys/devices/system/cpu}" +CPUFREQUTILS_DEFAULT="${CPUFREQUTILS_DEFAULT:-/etc/default/cpufrequtils}" # LXC's proc:mixed leaves /proc/sys/net writable but not /proc/sys/vm, and # the kubelet writes to /proc/sys/vm, so that is what is tested. PROC_SYS_PROBE="${PROC_SYS_PROBE:-/proc/sys/vm/overcommit_memory}" @@ -195,13 +196,13 @@ amd_integrated() { # amd_integrated_note PCI HAS_DISCRETE # What to do in setup-gpu-nodes.sh's integrated list for an integrated GPU. -# A node with a discrete card stays unticked: ticked, it renders on the -# integrated GPU and leaves the card unused. +# A node with a discrete card on amdgpu stays unticked: ticked, it renders on +# the integrated GPU and leaves the card unused. amd_integrated_note() { if [ "$2" = true ]; then - echo "$1 is integrated, and this node has a discrete AMD GPU: leave it unticked in setup-gpu-nodes.sh's integrated list so it renders on the discrete card" + echo "$1 is integrated, and this node has a discrete AMD GPU on amdgpu: leave it unticked in setup-gpu-nodes.sh's integrated list so it renders on the discrete card" else - echo "$1 is integrated and the only AMD GPU: tick this node in setup-gpu-nodes.sh's integrated list (5stack-gpu-integrated=true)" + echo "$1 is integrated and the only AMD GPU on amdgpu: tick this node in setup-gpu-nodes.sh's integrated list (5stack-gpu-integrated=true)" fi } @@ -210,7 +211,10 @@ check_amd() { step "AMD GPU" + # A card on vfio-pci or without a driver renders nothing here, so it does + # not count as a discrete GPU. for dev in "${AMD_GPUS[@]}"; do + [ "$(driver_of "$dev")" = amdgpu ] || continue amd_integrated "$dev" || has_discrete=true done @@ -524,7 +528,9 @@ k3s_service() { } # The same script and unit the panel's CPU governor tuning installs, so a -# node that later gets that tuning keeps one unit, not two. +# node that later gets that tuning keeps one unit, not two, and the same +# change to cpufrequtils's settings, whose boot script would set its own +# governor. install_governor_unit() { local unit="$SYSTEMD_DIR/5stack-cpu-governor.service" governor mkdir -p "${GOVERNOR_SCRIPT%/*}" || return 1 @@ -548,6 +554,10 @@ ExecStart=$GOVERNOR_SCRIPT WantedBy=multi-user.target UNIT mv -f "$unit.5stack-new" "$unit" || return 1 + if [ -f "$CPUFREQUTILS_DEFAULT" ]; then + { grep -v '^GOVERNOR=' "$CPUFREQUTILS_DEFAULT" || true; echo 'GOVERNOR="performance"'; } >"$CPUFREQUTILS_DEFAULT.5stack-new" && + mv -f "$CPUFREQUTILS_DEFAULT.5stack-new" "$CPUFREQUTILS_DEFAULT" || return 1 + fi # restart, not enable --now: --now leaves an active unit alone, and the # governor may have been changed since it ran. systemctl daemon-reload && @@ -563,12 +573,18 @@ UNIT } # k3s_join_hint VERSION -# The panel's join command with the k3s version pinned. sudo drops exported -# variables, so the pin goes after sudo, where it reaches the k3s installer. +# The panel's join command with a k3s version pin. sudo drops exported +# variables, so the pin goes after sudo. The k3s installer reads it, but the +# join reaches that installer through the 5Stack agent and the panel's node +# script, and whether they pass it on has not been tested. k3s_join_hint() { echo "curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=$1 sh -s -- join --panel-key " } +k3s_pin_caveat() { + hint "Untested: the pin only takes effect if the join passes INSTALL_K3S_VERSION on to the k3s installer. Check 'k3s --version' afterwards." +} + check_common() { step "Node" @@ -613,6 +629,7 @@ check_common() { [ "$(printf '%s\n' "$installed" "$K3S_VERSION" | sort -V | tail -1)" = "$installed" ]; then warn "k3s $installed is newer than the panel's $K3S_VERSION; a kubelet newer than its API server is unsupported" hint "Re-join with the version pinned: $(k3s_join_hint "$K3S_VERSION")" + k3s_pin_caveat else pass "k3s $installed installed" fi @@ -622,8 +639,9 @@ check_common() { fail "$service is not running; see 'journalctl -u $service -n 50'" fi else - info "k3s not installed yet. The join command installs the latest k3s; to match the panel, join with the version pinned:" + info "k3s not installed yet. The join command installs the latest k3s; to try to match the panel, join with the version pinned:" hint "$(k3s_join_hint "${K3S_VERSION:-}")" + k3s_pin_caveat fi } diff --git a/tests/scripts/gpu_node_check_test.sh b/tests/scripts/gpu_node_check_test.sh index fb66f01..fbd423b 100644 --- a/tests/scripts/gpu_node_check_test.sh +++ b/tests/scripts/gpu_node_check_test.sh @@ -22,13 +22,14 @@ script() { ) } -# fake_amd_gpu NAME integrated|discrete -# A PCI device directory as sysfs shows an amdgpu device, with a render node. +# fake_amd_gpu NAME integrated|discrete [DRIVER] +# A PCI device directory as sysfs shows an AMD GPU, with a render node, bound +# to DRIVER (amdgpu unless given). fake_amd_gpu() { - local dev="$WORK_DIR/pci/$1" + local dev="$WORK_DIR/pci/$1" driver="${3:-amdgpu}" mkdir -p "$dev/drm/renderD128" "$dev/hwmon/hwmon0" - ln -s "$WORK_DIR/drivers/amdgpu" "$dev/driver" - mkdir -p "$WORK_DIR/drivers/amdgpu" + ln -s "$WORK_DIR/drivers/$driver" "$dev/driver" + mkdir -p "$WORK_DIR/drivers/$driver" if [ "$2" = integrated ]; then echo vddnb >"$dev/hwmon/hwmon0/in1_label" else @@ -104,7 +105,7 @@ test_an_apu_beside_a_discrete_card_stays_unticked() { AMD_GPUS=("$(fake_amd_gpu 0000:c5:00.0 integrated)" "$(fake_amd_gpu 0000:03:00.0 discrete)") local out out=$(script check_amd) - assert_contains "$out" "0000:c5:00.0 is integrated, and this node has a discrete AMD GPU: leave it unticked" + assert_contains "$out" "0000:c5:00.0 is integrated, and this node has a discrete AMD GPU on amdgpu: leave it unticked" assert_not_contains "$out" "tick this node" assert_not_contains "$out" "runpm" } @@ -115,10 +116,23 @@ test_an_apu_on_its_own_is_ticked() { AMD_GPUS=("$(fake_amd_gpu 0000:c5:00.0 integrated)") local out out=$(script check_amd) - assert_contains "$out" "0000:c5:00.0 is integrated and the only AMD GPU: tick this node" + assert_contains "$out" "0000:c5:00.0 is integrated and the only AMD GPU on amdgpu: tick this node" assert_not_contains "$out" "runpm" } +# A card held for VM passthrough renders nothing on this node, so the APU is +# still the GPU to tick. +test_a_discrete_card_on_vfio_pci_does_not_count() { + load + quiet_amd_host + AMD_GPUS=("$(fake_amd_gpu 0000:c5:00.0 integrated)" "$(fake_amd_gpu 0000:03:00.0 discrete vfio-pci)") + local out + out=$(script check_amd) + assert_contains "$out" "[FAIL] 0000:03:00.0 is bound to 'vfio-pci', not amdgpu" + assert_contains "$out" "0000:c5:00.0 is integrated and the only AMD GPU on amdgpu: tick this node" + assert_not_contains "$out" "leave it unticked" +} + test_the_nvidia_node_hint_uses_m_for_modeset() { load local out @@ -153,6 +167,7 @@ test_without_k3s_the_join_hint_has_the_pin() { out=$(script check_common) assert_contains "$out" "sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join" assert_not_contains "$out" "INSTALL_K3S_VERSION=v1.36.5+k3s1 curl" + assert_contains "$out" "Untested: the pin only takes effect if the join passes INSTALL_K3S_VERSION on to the k3s installer" } # ubuntu-drivers on PATH, writing to stdout as the real one does. @@ -183,6 +198,26 @@ test_a_failed_driver_install_is_reported() { assert_eq "$out" " [FAIL] could not apply: installed the recommended NVIDIA driver; reboot, then run this again" } +# check_nvidia on an Ubuntu machine without the NVIDIA module, as --apply +# reaches install_nvidia_driver there. +test_check_nvidia_installs_the_driver_and_reports_it() { + load + shim_ubuntu_drivers 0 + IN_LXC=false + APPLY=true + OS_ID=ubuntu + NVIDIA_GPUS=() + # shellcheck disable=SC2317 # called by the function under test + nvidia_kernel_version() { :; } + # shellcheck disable=SC2317 # called by the function under test + install_nvidia_container_toolkit() { return 0; } + local out + out=$(script check_nvidia) + assert_contains "$out" "[FAIL] the NVIDIA kernel module is not loaded" + assert_contains "$out" "[FIX ] installed the recommended NVIDIA driver; reboot, then run this again" + assert_not_contains "$out" "Reading package lists" +} + test_proc_sys_is_probed_under_vm() { load assert_eq "$PROC_SYS_PROBE" "/proc/sys/vm/overcommit_memory" @@ -213,6 +248,7 @@ stub_governor() { CPU_SYSFS="$WORK_DIR/cpu" SYSTEMD_DIR="$WORK_DIR/systemd" GOVERNOR_SCRIPT="$WORK_DIR/bin/5stack-cpu-governor.sh" + CPUFREQUTILS_DEFAULT="$WORK_DIR/cpufrequtils" RESTART_SETS_GOVERNOR="$1" systemctl() { echo "systemctl $*" >>"$WORK_DIR/systemctl.log" @@ -237,6 +273,27 @@ test_the_governor_unit_is_the_panels_and_restarted() { return 0 } +# The panel's tuning points cpufrequtils's boot script at performance too. +test_the_governor_fix_sets_cpufrequtils_to_performance() { + load + stub_governor true + printf 'ENABLE="true"\nGOVERNOR="ondemand"\nMAX_SPEED="0"\n' >"$CPUFREQUTILS_DEFAULT" + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + assert_eq "$(cat "$CPUFREQUTILS_DEFAULT")" 'ENABLE="true" +MAX_SPEED="0" +GOVERNOR="performance"' + [ -e "$CPUFREQUTILS_DEFAULT.5stack-new" ] && fail "the temp file was left behind" + return 0 +} + +test_the_governor_fix_adds_no_cpufrequtils_settings() { + load + stub_governor true + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + [ -e "$CPUFREQUTILS_DEFAULT" ] && fail "cpufrequtils settings were created" + return 0 +} + test_the_governor_fix_fails_when_the_governor_did_not_change() { load stub_governor false From bb0b1e1688d2a0d49f39ddc4c14e638f45dca668 Mon Sep 17 00:00:00 2001 From: Flegma Date: Mon, 2 Nov 2026 23:53:12 +0100 Subject: [PATCH 07/10] docs: the GPU node README hedges the k3s pin and names both disk minimums --- gpu-node/README.md | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/gpu-node/README.md b/gpu-node/README.md index fc18408..07a870b 100644 --- a/gpu-node/README.md +++ b/gpu-node/README.md @@ -28,19 +28,26 @@ curl -fsSLO https://raw.githubusercontent.com/5stackgg/5stack-panel/main/gpu-nod - **k3s version.** The join command installs the latest k3s, and a kubelet newer than the panel's API server is unsupported. If the panel runs an older - k3s, read its version with `kubectl version` on the panel (`Server Version`) - and put it after `sudo` in the join command, since `sudo` drops a variable - exported before it: + k3s, read its version with `kubectl version` on the panel (`Server Version`). + The k3s installer reads `INSTALL_K3S_VERSION`, but the join hands off to the + 5Stack agent and the panel's node script, so the variable only takes effect + if they pass it through to the installer. That is untested. To try it, put + it after `sudo`, since `sudo` drops a variable exported before it, and check + `k3s --version` afterwards: ```bash curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join --panel-key ``` -- **Disk.** The join wants 120 GB free on `/`, on top of what the system uses. +- **Disk.** The join wants 120 GB free on `/` before CS2 is installed and 60 GB + once it is, on top of what the system uses. These are the panel's + "Required disk space" settings for fresh and existing installs (Settings, + Application, Servers). - **CPU governor.** Render pods drop frames while the CPU ramps up, so the governor is set to `performance` with `5stack-cpu-governor.service`, the - unit the panel's CPU governor tuning installs. In an LXC container that - happens on the Proxmox host. + unit the panel's CPU governor tuning installs, and, where + `/etc/default/cpufrequtils` exists, its `GOVERNOR` is set to `performance` + too. In an LXC container that happens on the Proxmox host. ## Dedicated machine or VM @@ -55,8 +62,8 @@ curl -fsSLO https://raw.githubusercontent.com/5stackgg/5stack-panel/main/gpu-nod 3. Join the node: on the panel's **Game Server Nodes** page (`/game-server-nodes`), create a node and run the command it shows on this - machine, with `INSTALL_K3S_VERSION` after `sudo` as above. Then run the check - again: some checks only apply once k3s is there. + machine (to try the k3s pin, with `INSTALL_K3S_VERSION` after `sudo` as + above). Then run the check again: some checks only apply once k3s is there. 4. On the panel, run `setup-gpu-nodes.sh` and pick the vendor and nodes: from the panel checkout as `./setup-gpu-nodes.sh`, or on a panel that runs as the 5Stack service, `sudo 5stack run setup-gpu-nodes.sh`. All GPU nodes of a @@ -111,8 +118,9 @@ the device with the container; do not bind the GPU to `vfio-pci`. governor, creates the NVIDIA device nodes at boot, turns on nesting, and adds these lines to `/etc/pve/lxc/120.conf`. It backs the config up to `/root` first and changes nothing in the container if that fails, writes the new - config beside the old one and renames it over, and stops at an unprivileged - container: + config beside the old one and renames it over. For a missing or + unprivileged container it changes nothing, on the host or in the container, + and only checks: ``` lxc.apparmor.profile: unconfined From 069b8b0a561e7a8546fb6c191daa0d6f678d15e7 Mon Sep 17 00:00:00 2001 From: Flegma Date: Mon, 2 Nov 2026 23:57:40 +0100 Subject: [PATCH 08/10] fix: no --apply hints for a container the pre-flight refused check_container stops after the unprivileged failure when --apply was refused, like it does under --apply, so the report no longer offers fixes that a rerun with --apply would refuse again. gpu-node-check.sh's apply_step returns 1 on failure, like the Proxmox script's. --- gpu-node/gpu-node-check.sh | 4 +++- gpu-node/proxmox-gpu-lxc.sh | 4 +++- tests/scripts/proxmox_gpu_lxc_test.sh | 14 ++++++++++++++ 3 files changed, 20 insertions(+), 2 deletions(-) diff --git a/gpu-node/gpu-node-check.sh b/gpu-node/gpu-node-check.sh index 59de8da..7c8d21a 100755 --- a/gpu-node/gpu-node-check.sh +++ b/gpu-node/gpu-node-check.sh @@ -102,7 +102,8 @@ apt_install() { } # apply_step DESCRIPTION COMMAND... -# Runs an --apply step and reports it as fixed or failed. +# Runs an --apply step and reports it as fixed or failed, returning 1 when it +# failed. apply_step() { local description="$1" shift @@ -110,6 +111,7 @@ apply_step() { fixed "$description" else fail "could not apply: $description" + return 1 fi } diff --git a/gpu-node/proxmox-gpu-lxc.sh b/gpu-node/proxmox-gpu-lxc.sh index b43f3a3..f4b7d05 100755 --- a/gpu-node/proxmox-gpu-lxc.sh +++ b/gpu-node/proxmox-gpu-lxc.sh @@ -529,7 +529,9 @@ check_container() { if [ "$(conf_value "$conf" unprivileged)" = 1 ]; then fail "the container is unprivileged; k3s needs a privileged container" hint "Back it up and restore it as privileged: vzdump $CTID, then pct restore with --unprivileged 0." - if [ "$APPLY" = true ]; then + # Past this point every hint says what --apply would change, and the + # pre-flight has already refused --apply for this container. + if [ "$APPLY" = true ] || [ "$APPLY_REFUSED" = true ]; then hint "--apply changes nothing in this container until it is privileged." return fi diff --git a/tests/scripts/proxmox_gpu_lxc_test.sh b/tests/scripts/proxmox_gpu_lxc_test.sh index 7ec03e3..8342303 100644 --- a/tests/scripts/proxmox_gpu_lxc_test.sh +++ b/tests/scripts/proxmox_gpu_lxc_test.sh @@ -307,6 +307,20 @@ test_apply_stops_at_an_unprivileged_container() { assert_eq "$(find "$WORK_DIR/backup" -type f | wc -l)" "0" } +# A refused --apply runs the checks in report mode, so the container's +# hints must not offer --apply for a container it will refuse again. +test_refused_apply_offers_no_apply_fixes() { + load + write_container 1 + stub_pct stopped + APPLY=false + APPLY_REFUSED=true + local out + out=$(script check_container) + assert_contains "$out" "[FAIL] the container is unprivileged" + assert_not_contains "$out" "(--apply" +} + # The host steps run before the container's, so --apply has to stop before # them, not only in check_container. test_apply_turns_off_before_the_host_steps_for_an_unprivileged_container() { From 0a0277937a1e40fd0a891ea899615a4631a574a3 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Mon, 2 Nov 2026 21:04:40 -0500 Subject: [PATCH 09/10] docs: shorter GPU node comments and README, no k3s pin caveat --- gpu-node/README.md | 42 +++++-------- gpu-node/gpu-node-check.sh | 69 +++------------------ gpu-node/proxmox-gpu-lxc.sh | 93 ++++++---------------------- tests/scripts/gpu_node_check_test.sh | 1 - 4 files changed, 42 insertions(+), 163 deletions(-) diff --git a/gpu-node/README.md b/gpu-node/README.md index 07a870b..59ce34a 100644 --- a/gpu-node/README.md +++ b/gpu-node/README.md @@ -26,28 +26,20 @@ curl -fsSLO https://raw.githubusercontent.com/5stackgg/5stack-panel/main/gpu-nod ## Before you start -- **k3s version.** The join command installs the latest k3s, and a kubelet - newer than the panel's API server is unsupported. If the panel runs an older - k3s, read its version with `kubectl version` on the panel (`Server Version`). - The k3s installer reads `INSTALL_K3S_VERSION`, but the join hands off to the - 5Stack agent and the panel's node script, so the variable only takes effect - if they pass it through to the installer. That is untested. To try it, put - it after `sudo`, since `sudo` drops a variable exported before it, and check - `k3s --version` afterwards: +- **k3s version.** The join installs the latest k3s, and a kubelet newer than + the panel's API server is unsupported. If the panel runs an older k3s, pin + the join to its `Server Version` from `kubectl version`. The pin goes after + `sudo`, which drops exported variables: ```bash curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join --panel-key ``` - **Disk.** The join wants 120 GB free on `/` before CS2 is installed and 60 GB - once it is, on top of what the system uses. These are the panel's - "Required disk space" settings for fresh and existing installs (Settings, - Application, Servers). -- **CPU governor.** Render pods drop frames while the CPU ramps up, so the - governor is set to `performance` with `5stack-cpu-governor.service`, the - unit the panel's CPU governor tuning installs, and, where - `/etc/default/cpufrequtils` exists, its `GOVERNOR` is set to `performance` - too. In an LXC container that happens on the Proxmox host. + once it is, the panel's "Required disk space" settings. +- **CPU governor.** Render pods drop frames while the CPU ramps up, so `--apply` + sets `performance` with the panel's `5stack-cpu-governor.service`. In an LXC + container that happens on the Proxmox host. ## Dedicated machine or VM @@ -62,8 +54,7 @@ curl -fsSLO https://raw.githubusercontent.com/5stackgg/5stack-panel/main/gpu-nod 3. Join the node: on the panel's **Game Server Nodes** page (`/game-server-nodes`), create a node and run the command it shows on this - machine (to try the k3s pin, with `INSTALL_K3S_VERSION` after `sudo` as - above). Then run the check again: some checks only apply once k3s is there. + machine. Then run the check again: some checks only apply once k3s is there. 4. On the panel, run `setup-gpu-nodes.sh` and pick the vendor and nodes: from the panel checkout as `./setup-gpu-nodes.sh`, or on a panel that runs as the 5Stack service, `sudo 5stack run setup-gpu-nodes.sh`. All GPU nodes of a @@ -114,13 +105,9 @@ the device with the container; do not bind the GPU to `vfio-pci`. pct reboot 120 ``` - `--apply` loads the kernel modules k3s needs (and at every boot), sets the CPU - governor, creates the NVIDIA device nodes at boot, turns on nesting, and adds - these lines to `/etc/pve/lxc/120.conf`. It backs the config up to `/root` - first and changes nothing in the container if that fails, writes the new - config beside the old one and renames it over. For a missing or - unprivileged container it changes nothing, on the host or in the container, - and only checks: + `--apply` loads the kernel modules k3s needs, sets the CPU governor, creates + the NVIDIA device nodes at boot, turns on nesting, backs up + `/etc/pve/lxc/120.conf` to `/root`, and adds these lines to it: ``` lxc.apparmor.profile: unconfined @@ -139,9 +126,8 @@ the device with the container; do not bind the GPU to `vfio-pci`. > **`proc:rw` changes the whole host.** It lets the kubelet set > `vm.overcommit_memory=1`, `kernel.panic=10` and `kernel.panic_on_oops=1`, > which are not per container: the Proxmox host then reboots 10 seconds - > after a kernel oops, and every guest on it goes down. On a host shared - > with other guests, decide whether that is acceptable before you restart - > the container. `--apply` prints this warning too. + > after a kernel oops, and every guest on it goes down. Decide whether that + > is acceptable before you restart the container. 4. Inside the container, run `gpu-node-check.sh --apply`. For NVIDIA it installs the host driver's exact version of the NVIDIA libraries, without diff --git a/gpu-node/gpu-node-check.sh b/gpu-node/gpu-node-check.sh index 7c8d21a..1f153ad 100755 --- a/gpu-node/gpu-node-check.sh +++ b/gpu-node/gpu-node-check.sh @@ -1,19 +1,9 @@ #!/bin/bash -# Checks that a machine can run 5stack GPU render pods, and with --apply fixes -# what it safely can. Run it as root on the GPU node itself: a dedicated -# machine, a VM with the GPU passed through, or a Proxmox LXC container (run -# gpu-node/proxmox-gpu-lxc.sh on the Proxmox host first). -# -# Run it before the panel's node setup command, and again after, since some -# checks (the k3s container runtime) only mean something once k3s is there. +# Checks that a GPU node can run 5stack render pods. Run it as root on the node, +# before the panel's join command and again after it. See README.md. # # gpu-node-check.sh [--apply] [--vendor nvidia|amd] [--k3s-version v1.36.5+k3s1] -# -# Exit code: 0 when nothing failed, 1 when a check failed, 2 on bad usage. -# -# It never touches the GPU's power or clock settings, the desktop session or -# Tailscale, and it never installs a kernel driver inside a container. APPLY=false VENDOR="" @@ -101,9 +91,6 @@ apt_install() { DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold "$@" /dev/null } -# apply_step DESCRIPTION COMMAND... -# Runs an --apply step and reports it as fixed or failed, returning 1 when it -# failed. apply_step() { local description="$1" shift @@ -115,7 +102,6 @@ apply_step() { fi } -# The kernel log, when this machine may read it. A container usually may not. kernel_log() { dmesg 2>/dev/null || journalctl -k -b 0 --no-pager 2>/dev/null } @@ -147,7 +133,6 @@ check_machine() { fi info "$OS_ID $OS_VERSION" - # GPUs, from sysfs, which a container sees too. for dev in /sys/bus/pci/devices/*; do class=$(cat "$dev/class" 2>/dev/null) case "$class" in @@ -183,8 +168,6 @@ cmdline_has() { tr ' ' '\n' /dev/null | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1 @@ -383,11 +360,7 @@ install_nvidia_driver() { DEBIAN_FRONTEND=noninteractive ubuntu-drivers install /dev/null } -# nvidia_nodes_hint MISSING -# The nvidia-modprobe commands that create the MISSING device nodes on a -# machine that owns the GPU: -c0 makes nvidia0 and nvidiactl, -c0 -u the uvm -# nodes instead, and only -m makes nvidia-modeset, which a headless node -# without a display often lacks. +# -c0 makes nvidia0 and nvidiactl, -c0 -u the uvm nodes, and only -m makes nvidia-modeset. nvidia_nodes_hint() { local missing="$1" commands=() case "$missing" in *"/dev/nvidia0"* | *"/dev/nvidiactl"*) commands+=("nvidia-modprobe -c0") ;; esac @@ -411,7 +384,6 @@ install_nvidia_container_toolkit() { curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list /etc/apt/sources.list.d/nvidia-container-toolkit.list || return 1 - # The new repository needs its own package list. APT_UPDATED=false apt_install nvidia-container-toolkit } @@ -517,10 +489,6 @@ check_nvidia() { fi } -# --- Common ---------------------------------------------------------------- - -# The k3s unit: k3s-agent on a node that joined the panel, k3s when the GPU -# node is the panel's server itself. k3s_service() { if systemctl is-enabled --quiet k3s 2>/dev/null; then echo k3s @@ -529,10 +497,8 @@ k3s_service() { fi } -# The same script and unit the panel's CPU governor tuning installs, so a -# node that later gets that tuning keeps one unit, not two, and the same -# change to cpufrequtils's settings, whose boot script would set its own -# governor. +# Same script and unit as the panel's CPU governor tuning, so a node never has two. +# cpufrequtils's boot script would otherwise set its own governor. install_governor_unit() { local unit="$SYSTEMD_DIR/5stack-cpu-governor.service" governor mkdir -p "${GOVERNOR_SCRIPT%/*}" || return 1 @@ -560,8 +526,7 @@ UNIT { grep -v '^GOVERNOR=' "$CPUFREQUTILS_DEFAULT" || true; echo 'GOVERNOR="performance"'; } >"$CPUFREQUTILS_DEFAULT.5stack-new" && mv -f "$CPUFREQUTILS_DEFAULT.5stack-new" "$CPUFREQUTILS_DEFAULT" || return 1 fi - # restart, not enable --now: --now leaves an active unit alone, and the - # governor may have been changed since it ran. + # restart, not enable --now, which leaves an active unit alone. systemctl daemon-reload && systemctl enable 5stack-cpu-governor.service >/dev/null 2>&1 && systemctl restart 5stack-cpu-governor.service >/dev/null 2>&1 || return 1 @@ -574,19 +539,11 @@ UNIT done } -# k3s_join_hint VERSION -# The panel's join command with a k3s version pin. sudo drops exported -# variables, so the pin goes after sudo. The k3s installer reads it, but the -# join reaches that installer through the 5Stack agent and the panel's node -# script, and whether they pass it on has not been tested. +# sudo drops exported variables, so the pin goes after it. k3s_join_hint() { echo "curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=$1 sh -s -- join --panel-key " } -k3s_pin_caveat() { - hint "Untested: the pin only takes effect if the join passes INSTALL_K3S_VERSION on to the k3s installer. Check 'k3s --version' afterwards." -} - check_common() { step "Node" @@ -631,7 +588,6 @@ check_common() { [ "$(printf '%s\n' "$installed" "$K3S_VERSION" | sort -V | tail -1)" = "$installed" ]; then warn "k3s $installed is newer than the panel's $K3S_VERSION; a kubelet newer than its API server is unsupported" hint "Re-join with the version pinned: $(k3s_join_hint "$K3S_VERSION")" - k3s_pin_caveat else pass "k3s $installed installed" fi @@ -641,14 +597,11 @@ check_common() { fail "$service is not running; see 'journalctl -u $service -n 50'" fi else - info "k3s not installed yet. The join command installs the latest k3s; to try to match the panel, join with the version pinned:" + info "k3s not installed yet. The join command installs the latest k3s; to match the panel, join with the version pinned:" hint "$(k3s_join_hint "${K3S_VERSION:-}")" - k3s_pin_caveat fi } -# k3s in an LXC needs a privileged container with the host's kernel features -# opened up; proxmox-gpu-lxc.sh sets all of this on the host. check_lxc() { step "LXC container" @@ -704,10 +657,8 @@ check_lxc() { fail "kernel module br_netfilter is not loaded; load it on the host" fi - # The kubelet's static CPU manager finds CPUs through sysfs, which shows - # every host CPU even where lxcfs narrows /proc/cpuinfo to the container's - # cores, so a container held to fewer cores gets pods pinned to CPUs it - # cannot use. + # The kubelet's static CPU manager reads sysfs, which shows every host CPU even + # where lxcfs narrows /proc/cpuinfo, so it can pin pods to CPUs the container lacks. local allowed host_cpus allowed=$(nproc 2>/dev/null) host_cpus=$(find /sys/devices/system/cpu -maxdepth 1 -name 'cpu[0-9]*' | wc -l) diff --git a/gpu-node/proxmox-gpu-lxc.sh b/gpu-node/proxmox-gpu-lxc.sh index f4b7d05..cacaee4 100755 --- a/gpu-node/proxmox-gpu-lxc.sh +++ b/gpu-node/proxmox-gpu-lxc.sh @@ -1,15 +1,9 @@ #!/bin/bash -# Prepares a Proxmox host and one of its LXC containers to run a 5stack GPU -# node: k3s inside the container, rendering on the host's NVIDIA or AMD GPU. -# Run it as root on the Proxmox host. With --apply it also fixes what it can; -# it backs up the container's config first and never restarts the container. +# Prepares a Proxmox host and one of its LXC containers to run a 5stack GPU node. +# Run it as root on the host, then gpu-node-check.sh in the container. See README.md. # # proxmox-gpu-lxc.sh CTID [--apply] [--vendor nvidia|amd] -# -# Then, inside the container, run gpu-node/gpu-node-check.sh. -# -# Exit code: 0 when nothing failed, 1 when a check failed, 2 on bad usage. CTID="" APPLY=false @@ -23,17 +17,14 @@ CPUFREQUTILS_DEFAULT="${CPUFREQUTILS_DEFAULT:-/etc/default/cpufrequtils}" PROC_CMDLINE="${PROC_CMDLINE:-/proc/cmdline}" NVIDIA_PROC_VERSION="${NVIDIA_PROC_VERSION:-/proc/driver/nvidia/version}" -# The node setup command stops unless / has this much free (df -BG): the -# panel's "Required disk space - fresh install" setting before CS2 is -# installed, "Required disk space - existing install" once it is. These are -# their defaults. The system, k3s and the render images need room on top. +# The defaults of the panel's "Required disk space" settings, which the node setup +# enforces on / (df -BG): fresh install, then with CS2 installed. REQUIRED_FREE_GB=120 REQUIRED_FREE_EXISTING_GB=60 SYSTEM_HEADROOM_GB=20 CS2_DIR=/opt/5stack/serverfiles/game/csgo -# The kernel modules k3s, flannel and kube-proxy load. A container cannot load -# modules, so the host loads them, now and at every boot. +# A container cannot load modules, so the host loads what k3s, flannel and kube-proxy need. K3S_MODULES="overlay br_netfilter nf_conntrack vxlan ip_tables iptable_nat iptable_filter iptable_mangle xt_conntrack xt_mark xt_comment nf_tables tun" MODULES_FILE=/etc/modules-load.d/5stack-k3s.conf @@ -58,9 +49,6 @@ usage() { exit 2 } -# apply_step DESCRIPTION COMMAND... -# Runs an --apply step and reports it as fixed or failed, returning 1 when it -# failed. apply_step() { local description="$1" shift @@ -72,16 +60,12 @@ apply_step() { fi } -# conf_value FILE KEY -# The KEY's value from the container's own section (snapshots follow it). +# Reads only the container's own section; snapshot sections follow it. conf_value() { awk -v key="$2" '/^\[/ { exit } index($0, key ": ") == 1 { print substr($0, length(key) + 3); exit }' "$1" } -# size_gb SIZE -# A Proxmox disk size in whole GiB, rounded down: 128G -> 128, 2T -> 2048, -# 131072M -> 128, 512M -> 0. Without a unit the size is in bytes, as -# pve-common's parse_size reads it. +# Without a unit a Proxmox size is in bytes, as pve-common's parse_size reads it. size_gb() { local size="$1" number unit number="${size%[KMGT]}" @@ -96,8 +80,6 @@ size_gb() { }' } -# features_with_nesting FEATURES -# keyctl=1,fuse=1 -> keyctl=1,fuse=1,nesting=1, keeping every other feature. features_with_nesting() { local feature out="" local IFS=, @@ -108,8 +90,6 @@ features_with_nesting() { echo "${out:+$out,}nesting=1" } -# lxc_lines VENDOR -# The raw LXC lines that let k3s run in the container and reach the GPU. lxc_lines() { echo "lxc.apparmor.profile: unconfined" echo "lxc.cgroup2.devices.allow: a" @@ -133,9 +113,7 @@ lxc_lines() { esac } -# conf_has_line FILE LINE -# True when the container's own section has LINE. Proxmox rewrites the file on -# every change, so trailing spaces do not count. +# Proxmox rewrites the file on every change, so trailing spaces do not count. conf_has_line() { LINE="$2" awk ' function trim(s) { sub(/[ \t]+$/, "", s); return s } @@ -145,8 +123,6 @@ conf_has_line() { ' "$1" } -# missing_lines FILE LINE... -# Prints the LINEs the container's own section lacks. missing_lines() { local file="$1" line shift @@ -155,19 +131,13 @@ missing_lines() { done } -# add_lxc_lines FILE LINE... -# Adds the LINEs FILE lacks to the container's own section, which ends where -# the first snapshot section starts. FILE is either replaced whole or left -# as it was. add_lxc_lines() { local file="$1" missing tmp before after added shift missing=$(missing_lines "$file" "$@") || return 1 [ -n "$missing" ] || return 0 - # Written the way pve-common's file_set_contents writes a guest config: - # a temporary file in the same directory, renamed over the old one, so a - # failed write never leaves a cut-off config behind. + # Like pve-common's file_set_contents: renamed over, so a failed write never cuts the config off. tmp="$file.tmp.$$" rm -f "$tmp" # Blank lines are held back until the next line shows whether they end @@ -183,7 +153,6 @@ add_lxc_lines() { return 1 fi - # Every old line plus the new ones, or the result is not used. before=$(grep -c '' "$file") after=$(grep -c '' "$tmp") added=$(printf '%s\n' "$missing" | wc -l) @@ -199,8 +168,7 @@ add_lxc_lines() { fi } -# locked_add_lxc_lines FILE LINE... -# add_lxc_lines under the lock pct and backups take on the container's config. +# The lock pct and backups take on the container's config. locked_add_lxc_lines() { local lock="/run/lock/lxc/pve-config-$CTID.lock" mkdir -p "${lock%/*}" @@ -263,8 +231,6 @@ check_host_gpu() { } check_host_amd() { - # These turn off what amdgpu needs to render. NVIDIA's driver does not - # read them, so they are only checked on an AMD host. local arg for arg in nomodeset amdgpu.modeset=0 amdgpu.dpm=0; do if tr ' ' '\n' <"$PROC_CMDLINE" | grep -qx "$arg"; then @@ -303,10 +269,8 @@ check_host_nvidia() { fi pass "NVIDIA driver $version; the container needs the same version's libraries (gpu-node-check.sh --apply installs them)" - # The NVIDIA module creates no device nodes itself; they appear once - # something opens the GPU. The container cannot create them, so the host - # makes them at boot: nvidia-smi the per-GPU nodes and nvidiactl, - # nvidia-modprobe the uvm and modeset ones. + # The module creates no device nodes until something opens the GPU, and the + # container cannot create them, so the host makes them at boot. local missing missing=$(missing_nvidia_nodes) if [ -z "$missing" ] && [ -f "$NVIDIA_UNIT" ]; then @@ -403,9 +367,8 @@ check_host_kernel() { fi } -# The same script and unit the panel's CPU governor tuning installs on a -# node, so a machine never ends up with two of them, and the same change to -# cpufrequtils's settings, whose boot script would set its own governor. +# Same script and unit as the panel's CPU governor tuning, so a node never has two. +# cpufrequtils's boot script would otherwise set its own governor. install_governor_unit() { local unit="$SYSTEMD_DIR/5stack-cpu-governor.service" governor mkdir -p "${GOVERNOR_SCRIPT%/*}" || return 1 @@ -433,8 +396,7 @@ UNIT { grep -v '^GOVERNOR=' "$CPUFREQUTILS_DEFAULT" || true; echo 'GOVERNOR="performance"'; } >"$CPUFREQUTILS_DEFAULT.5stack-new" && mv -f "$CPUFREQUTILS_DEFAULT.5stack-new" "$CPUFREQUTILS_DEFAULT" || return 1 fi - # restart, not enable --now: --now leaves an active unit alone, and the - # governor may have been changed since it ran. + # restart, not enable --now, which leaves an active unit alone. systemctl daemon-reload && systemctl enable 5stack-cpu-governor.service >/dev/null 2>&1 && systemctl restart 5stack-cpu-governor.service >/dev/null 2>&1 || return 1 @@ -447,17 +409,11 @@ UNIT done } -# container_free_gb -# GB free on / inside the container (df -BG, as the node setup reads it), or -# nothing while the container is not running. container_free_gb() { pct status "$CTID" 2>/dev/null | grep -q 'status: running' || return 0 pct exec "$CTID" -- df -BG / 2>/dev/null | awk 'NR == 2 { sub(/G$/, "", $4); print $4 }' } -# container_has_cs2 -# True when CS2 is installed in the running container, where the node setup -# looks for it. container_has_cs2() { pct exec "$CTID" -- test -d "$CS2_DIR" 2>/dev/null } @@ -466,10 +422,6 @@ disk_settings_hint() { hint "The panel sets both minimums: \"Required disk space - fresh install\" ($REQUIRED_FREE_GB GB by default) and \"Required disk space - existing install\" ($REQUIRED_FREE_EXISTING_GB GB), under Settings, Application, Servers." } -# check_disk ROOTFS -# The node setup needs REQUIRED_FREE_GB free on /, or REQUIRED_FREE_EXISTING_GB -# once CS2 is installed, so a running container is judged by its free space, -# and a stopped one by its size. check_disk() { local rootfs="$1" size gb free want size=$(echo "$rootfs" | grep -oE 'size=[0-9.]+[KMGT]?' | cut -d= -f2) @@ -508,12 +460,9 @@ check_disk() { fi } -# proc:rw is what lets the kubelet change kernel values, and the ones it sets -# are not namespaced, so they change the Proxmox host itself. proc_rw_warning() { warn "lxc.mount.auto: proc:rw lets the kubelet set vm.overcommit_memory=1, kernel.panic=10 and kernel.panic_on_oops=1, which apply to the whole host" - hint "The host then reboots 10 seconds after a kernel oops, taking every guest on it down." - hint "On a host shared with other guests, decide whether that is acceptable before restarting the container." + hint "The host then reboots 10 seconds after a kernel oops, taking every guest down. Decide if that is acceptable before restarting the container." } check_container() { @@ -529,8 +478,7 @@ check_container() { if [ "$(conf_value "$conf" unprivileged)" = 1 ]; then fail "the container is unprivileged; k3s needs a privileged container" hint "Back it up and restore it as privileged: vzdump $CTID, then pct restore with --unprivileged 0." - # Past this point every hint says what --apply would change, and the - # pre-flight has already refused --apply for this container. + # Every hint past here offers --apply, which this container refuses. if [ "$APPLY" = true ] || [ "$APPLY_REFUSED" = true ]; then hint "--apply changes nothing in this container until it is privileged." return @@ -602,9 +550,6 @@ check_container() { fi } -# backup_conf FILE -# Copies FILE to BACKUP_DIR once per run. --apply changes nothing in the -# container without a backup, so a failed copy fails the run. BACKED_UP=false backup_conf() { [ "$BACKED_UP" = true ] && return 0 @@ -619,9 +564,7 @@ backup_conf() { return 1 } -# preflight_apply -# --apply changes the host only for a container it can then finish, so a -# missing or unprivileged container turns it off before any host step runs. +# --apply changes the host only for a container it can then finish. APPLY_REFUSED=false preflight_apply() { local conf="$PVE_LXC_DIR/$CTID.conf" diff --git a/tests/scripts/gpu_node_check_test.sh b/tests/scripts/gpu_node_check_test.sh index fbd423b..ffa2691 100644 --- a/tests/scripts/gpu_node_check_test.sh +++ b/tests/scripts/gpu_node_check_test.sh @@ -167,7 +167,6 @@ test_without_k3s_the_join_hint_has_the_pin() { out=$(script check_common) assert_contains "$out" "sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join" assert_not_contains "$out" "INSTALL_K3S_VERSION=v1.36.5+k3s1 curl" - assert_contains "$out" "Untested: the pin only takes effect if the join passes INSTALL_K3S_VERSION on to the k3s installer" } # ubuntu-drivers on PATH, writing to stdout as the real one does. From 6572980c4ac547eaa9881cbf7265a6720d251d5c Mon Sep 17 00:00:00 2001 From: Flegma Date: Tue, 3 Nov 2026 13:09:56 +0100 Subject: [PATCH 10/10] fix: GPU node checks accept Proxmox device passthrough and find k3s off PATH proxmox-gpu-lxc.sh counted a container that gets its GPU through Proxmox's devN: entries (dev0: /dev/kfd,gid=993, dev1: /dev/dri/renderD128) as missing the /dev/dri and /dev/kfd binds, and --apply would have added them a second time. A devN entry for the device, or for a render node in the case of /dev/dri, now counts. gpu-node-check.sh looked for k3s on PATH only, which pct exec and some sudo setups leave without /usr/local/bin, so an installed k3s read as not installed. --- gpu-node/gpu-node-check.sh | 7 +++++++ gpu-node/proxmox-gpu-lxc.sh | 27 +++++++++++++++++++++++++-- tests/scripts/gpu_node_check_test.sh | 8 ++++++++ tests/scripts/proxmox_gpu_lxc_test.sh | 26 ++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 2 deletions(-) diff --git a/gpu-node/gpu-node-check.sh b/gpu-node/gpu-node-check.sh index 1f153ad..c7fd4b9 100755 --- a/gpu-node/gpu-node-check.sh +++ b/gpu-node/gpu-node-check.sh @@ -19,6 +19,13 @@ CPUFREQUTILS_DEFAULT="${CPUFREQUTILS_DEFAULT:-/etc/default/cpufrequtils}" # LXC's proc:mixed leaves /proc/sys/net writable but not /proc/sys/vm, and # the kubelet writes to /proc/sys/vm, so that is what is tested. PROC_SYS_PROBE="${PROC_SYS_PROBE:-/proc/sys/vm/overcommit_memory}" +# The k3s installer puts k3s in /usr/local/bin, which pct exec and some sudo +# setups leave off PATH. +K3S_BIN_DIR="${K3S_BIN_DIR-/usr/local/bin}" +case ":$PATH:" in + *":$K3S_BIN_DIR:"*) ;; + *) [ -n "$K3S_BIN_DIR" ] && PATH="$PATH:$K3S_BIN_DIR" ;; +esac usage() { echo "usage: gpu-node-check.sh [--apply] [--vendor nvidia|amd] [--k3s-version v1.36.5+k3s1]" >&2 diff --git a/gpu-node/proxmox-gpu-lxc.sh b/gpu-node/proxmox-gpu-lxc.sh index cacaee4..5fac0bd 100755 --- a/gpu-node/proxmox-gpu-lxc.sh +++ b/gpu-node/proxmox-gpu-lxc.sh @@ -123,11 +123,34 @@ conf_has_line() { ' "$1" } +# Proxmox's own passthrough (devN: /dev/kfd,gid=993) gives the container the +# device too; for /dev/dri a render node is the one the GPU needs. +conf_passes_device() { + DEVICE="$2" awk ' + /^\[/ { exit } + /^dev[0-9]+: / { + path = substr($0, index($0, " ") + 1) + sub(/^path=/, "", path) + sub(/,.*/, "", path) + want = ENVIRON["DEVICE"] + if (path == want || (want == "/dev/dri" && index(path, "/dev/dri/renderD") == 1)) { found = 1; exit } + } + END { exit !found } + ' "$1" +} + missing_lines() { - local file="$1" line + local file="$1" line device shift for line in "$@"; do - conf_has_line "$file" "$line" || echo "$line" + conf_has_line "$file" "$line" && continue + case "$line" in + "lxc.mount.entry: /dev/"*) + device=${line#lxc.mount.entry: } + conf_passes_device "$file" "${device%% *}" && continue + ;; + esac + echo "$line" done } diff --git a/tests/scripts/gpu_node_check_test.sh b/tests/scripts/gpu_node_check_test.sh index ffa2691..f5c4047 100644 --- a/tests/scripts/gpu_node_check_test.sh +++ b/tests/scripts/gpu_node_check_test.sh @@ -303,4 +303,12 @@ test_the_governor_fix_fails_when_the_governor_did_not_change() { assert_not_contains "$out" "[FIX ]" } +test_k3s_is_found_in_usr_local_bin_when_path_leaves_it_out() { + ( + PATH=/usr/bin:/bin + load + assert_contains ":$PATH:" ":/usr/local/bin:" + ) +} + run_tests diff --git a/tests/scripts/proxmox_gpu_lxc_test.sh b/tests/scripts/proxmox_gpu_lxc_test.sh index 8342303..82e3b54 100644 --- a/tests/scripts/proxmox_gpu_lxc_test.sh +++ b/tests/scripts/proxmox_gpu_lxc_test.sh @@ -120,6 +120,32 @@ test_nvidia_lines_bind_the_nvidia_devices() { assert_not_contains "$(lxc_lines nvidia)" "/dev/kfd" } +test_proxmox_device_passthrough_counts_for_the_amd_binds() { + load + write_conf + sed -i '/^unprivileged: 0$/a dev0: /dev/kfd,gid=993\ndev1: path=/dev/dri/renderD128,gid=993\ndev2: /dev/dri/card1,gid=44' "$WORK_DIR/100.conf" + local lines + mapfile -t lines < <(lxc_lines amd) + assert_not_contains "$(missing_lines "$WORK_DIR/100.conf" "${lines[@]}")" "/dev/dri" + assert_not_contains "$(missing_lines "$WORK_DIR/100.conf" "${lines[@]}")" "/dev/kfd" + add_lxc_lines "$WORK_DIR/100.conf" "${lines[@]}" + assert_eq "$(grep -c '^lxc.mount.entry: /dev/kfd' "$WORK_DIR/100.conf")" "0" +} + +test_a_card_node_alone_does_not_count_for_dri() { + load + write_conf + sed -i '/^unprivileged: 0$/a dev0: /dev/dri/card1,gid=44' "$WORK_DIR/100.conf" + assert_eq "$(missing_lines "$WORK_DIR/100.conf" "lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir")" "lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir" +} + +test_device_passthrough_in_a_snapshot_does_not_count() { + load + write_conf + echo "dev0: /dev/kfd,gid=993" >>"$WORK_DIR/100.conf" + assert_eq "$(missing_lines "$WORK_DIR/100.conf" "lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file")" "lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file" +} + # An awk that fails only for the rewrite in add_lxc_lines, the one call that # passes ADD_LINES. MODE is fail (writes half the file, exits 1, as on a full # disk), empty (writes nothing, exits 0) or short (writes 2 lines, exits 0).