diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml index f6b1580..b7f97e4 100644 --- a/.github/workflows/panel-scripts.yml +++ b/.github/workflows/panel-scripts.yml @@ -10,6 +10,7 @@ on: - plugin.sh - service.sh - setup-gpu-nodes.sh + - "gpu-node/**" - update.sh - Tiltfile - "utils/**" @@ -29,6 +30,7 @@ on: - plugin.sh - service.sh - setup-gpu-nodes.sh + - "gpu-node/**" - update.sh - Tiltfile - "utils/**" @@ -46,7 +48,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: shellcheck -x service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh tests/scripts/*.sh tests/scripts/stub/kubectl + - run: shellcheck -x service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh gpu-node/*.sh tests/scripts/*.sh tests/scripts/stub/kubectl - run: bash tests/scripts/setup_env_test.sh - run: bash tests/scripts/apply_overlay_test.sh - run: bash tests/scripts/debug_test.sh @@ -61,3 +63,5 @@ jobs: - run: bash tests/scripts/service_test.sh - run: bash tests/scripts/postgres_backup_test.sh - run: bash tests/scripts/choose_node_test.sh + - run: bash tests/scripts/proxmox_gpu_lxc_test.sh + - run: bash tests/scripts/gpu_node_check_test.sh diff --git a/gpu-node/README.md b/gpu-node/README.md new file mode 100644 index 0000000..59ce34a --- /dev/null +++ b/gpu-node/README.md @@ -0,0 +1,146 @@ +# GPU nodes + +A GPU node renders demos, highlights and clips. It can be a dedicated machine, +a VM with the GPU passed through, or a Proxmox LXC container. NVIDIA and AMD +GPUs are both supported, one vendor per panel (`GPU_VENDOR`). + +| Script | Runs on | Does | +| --- | --- | --- | +| `gpu-node-check.sh` | the GPU node | checks the driver, firmware, container runtime, CPU governor, disk and k3s version; `--apply` fixes what it safely can | +| `proxmox-gpu-lxc.sh CTID` | the Proxmox host | checks the host driver and kernel modules and the container's settings; `--apply` writes them | +| `setup-gpu-nodes.sh` | the panel | labels the GPU nodes for the panel's vendor and turns that vendor's support on | + +The two check scripts only report unless you pass `--apply`, and exit 1 while +something still fails. None of these scripts change the GPU's power or clock +settings. + +## Getting the scripts + +A GPU node or a Proxmox host has no panel checkout, so download the script you +need there: + +```bash +curl -fsSLO https://raw.githubusercontent.com/5stackgg/5stack-panel/main/gpu-node/gpu-node-check.sh +curl -fsSLO https://raw.githubusercontent.com/5stackgg/5stack-panel/main/gpu-node/proxmox-gpu-lxc.sh +``` + +## Before you start + +- **k3s version.** The join installs the latest k3s, and a kubelet newer than + the panel's API server is unsupported. If the panel runs an older k3s, pin + the join to its `Server Version` from `kubectl version`. The pin goes after + `sudo`, which drops exported variables: + + ```bash + curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join --panel-key + ``` + +- **Disk.** The join wants 120 GB free on `/` before CS2 is installed and 60 GB + once it is, the panel's "Required disk space" settings. +- **CPU governor.** Render pods drop frames while the CPU ramps up, so `--apply` + sets `performance` with the panel's `5stack-cpu-governor.service`. In an LXC + container that happens on the Proxmox host. + +## Dedicated machine or VM + +1. Install Ubuntu 24.04 or newer, server or desktop. A headless install is + recommended: a desktop session keeps hold of the GPU's display. +2. Run the check, then let it fix what it can and reboot if it asks: + + ```bash + sudo bash gpu-node-check.sh + sudo bash gpu-node-check.sh --apply --k3s-version v1.36.5+k3s1 + ``` + +3. Join the node: on the panel's **Game Server Nodes** page + (`/game-server-nodes`), create a node and run the command it shows on this + machine. Then run the check again: some checks only apply once k3s is there. +4. On the panel, run `setup-gpu-nodes.sh` and pick the vendor and nodes: from + the panel checkout as `./setup-gpu-nodes.sh`, or on a panel that runs as the + 5Stack service, `sudo 5stack run setup-gpu-nodes.sh`. All GPU nodes of a + panel use the same vendor. + +### NVIDIA + +- `--apply` installs the recommended driver (`ubuntu-drivers install`) and the + NVIDIA container toolkit, and restarts k3s-agent if it started before the + toolkit was there. +- With Secure Boot on, use Ubuntu's signed prebuilt driver modules + (`ubuntu-drivers install` picks them) or enroll the DKMS key with `mokutil`. +- After a driver upgrade, `nvidia-smi` fails until the next reboot. +- A headless node may have no `/dev/nvidia-modeset`; `nvidia-modprobe -m` + creates it. + +### AMD + +- `--apply` installs the AMD firmware package. +- On Ubuntu 24.04, use the HWE kernel (`linux-generic-hwe-24.04`); the original + 6.8 kernel predates recent Radeons. +- Never boot with `nomodeset`, `amdgpu.modeset=0` or `amdgpu.dpm=0`. There is + no kernel parameter to add. +- For a discrete card, enable Above 4G decoding and Resizable BAR in the + firmware. +- Tick a node in the integrated list of `setup-gpu-nodes.sh` only when its + integrated GPU (an APU, such as Ryzen AI Max) is its only AMD GPU. Leave a + node with a discrete Radeon card unticked, even when its CPU also has an + integrated GPU: ticked, it renders on the integrated GPU and leaves the card + unused. The check says which case a node is. + +## Proxmox LXC container + +k3s needs a **privileged** container. The host keeps the GPU driver and shares +the device with the container; do not bind the GPU to `vfio-pci`. + +1. On the host, install the GPU driver (NVIDIA: the proxmox headers and + NVIDIA's `.run` installer; AMD: the kernel driver is built in, firmware comes + from `pve-firmware`). +2. Create a privileged Ubuntu container with at least 140 GB of disk (120 GB + free for the join, plus the system) and no core limit (the kubelet pins pods + to CPUs by number and sees every host CPU). +3. On the host: + + ```bash + bash proxmox-gpu-lxc.sh 120 + bash proxmox-gpu-lxc.sh 120 --apply + pct reboot 120 + ``` + + `--apply` loads the kernel modules k3s needs, sets the CPU governor, creates + the NVIDIA device nodes at boot, turns on nesting, backs up + `/etc/pve/lxc/120.conf` to `/root`, and adds these lines to it: + + ``` + lxc.apparmor.profile: unconfined + lxc.cgroup2.devices.allow: a + lxc.cap.drop: + lxc.mount.auto: proc:rw sys:rw + lxc.mount.entry: /dev/kmsg dev/kmsg none bind,optional,create=file + lxc.mount.entry: /dev/net/tun dev/net/tun none bind,optional,create=file + # AMD + lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir + lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file + # NVIDIA: /dev/nvidia0, nvidiactl, nvidia-uvm, nvidia-uvm-tools, + # nvidia-modeset and nvidia-caps, the same way + ``` + + > **`proc:rw` changes the whole host.** It lets the kubelet set + > `vm.overcommit_memory=1`, `kernel.panic=10` and `kernel.panic_on_oops=1`, + > which are not per container: the Proxmox host then reboots 10 seconds + > after a kernel oops, and every guest on it goes down. Decide whether that + > is acceptable before you restart the container. + +4. Inside the container, run `gpu-node-check.sh --apply`. For NVIDIA it + installs the host driver's exact version of the NVIDIA libraries, without + the kernel module, plus the container toolkit. +5. Continue with step 3 of the dedicated machine steps. + +If NVIDIA pods fail to start in the container with a device cgroup error, set +`no-cgroups = true` under `[nvidia-container-cli]` in +`/etc/nvidia-container-runtime/config.toml` and restart k3s-agent. Leave it at +the default otherwise: the NVIDIA device plugin runs unprivileged and relies +on it. + +## Reporting a problem + +Include the output of `gpu-node-check.sh`, `proxmox-gpu-lxc.sh` for a container, +and `journalctl -k -b | grep -iE "amdgpu|nvidia|nvrm"`. diff --git a/gpu-node/gpu-node-check.sh b/gpu-node/gpu-node-check.sh new file mode 100755 index 0000000..c7fd4b9 --- /dev/null +++ b/gpu-node/gpu-node-check.sh @@ -0,0 +1,714 @@ +#!/bin/bash + +# Checks that a GPU node can run 5stack render pods. Run it as root on the node, +# before the panel's join command and again after it. See README.md. +# +# gpu-node-check.sh [--apply] [--vendor nvidia|amd] [--k3s-version v1.36.5+k3s1] + +APPLY=false +VENDOR="" +K3S_VERSION="" +IN_LXC=false +ON_HOST="" +NVIDIA_GPUS=() +AMD_GPUS=() +SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}" +GOVERNOR_SCRIPT="${GOVERNOR_SCRIPT:-/usr/local/bin/5stack-cpu-governor.sh}" +CPU_SYSFS="${CPU_SYSFS:-/sys/devices/system/cpu}" +CPUFREQUTILS_DEFAULT="${CPUFREQUTILS_DEFAULT:-/etc/default/cpufrequtils}" +# LXC's proc:mixed leaves /proc/sys/net writable but not /proc/sys/vm, and +# the kubelet writes to /proc/sys/vm, so that is what is tested. +PROC_SYS_PROBE="${PROC_SYS_PROBE:-/proc/sys/vm/overcommit_memory}" +# The k3s installer puts k3s in /usr/local/bin, which pct exec and some sudo +# setups leave off PATH. +K3S_BIN_DIR="${K3S_BIN_DIR-/usr/local/bin}" +case ":$PATH:" in + *":$K3S_BIN_DIR:"*) ;; + *) [ -n "$K3S_BIN_DIR" ] && PATH="$PATH:$K3S_BIN_DIR" ;; +esac + +usage() { + echo "usage: gpu-node-check.sh [--apply] [--vendor nvidia|amd] [--k3s-version v1.36.5+k3s1]" >&2 + exit 2 +} + +parse_args() { + while [ $# -gt 0 ]; do + case "$1" in + --apply) APPLY=true ;; + --vendor) + VENDOR="$2" + shift + ;; + --k3s-version) + K3S_VERSION="$2" + shift + ;; + -h | --help) usage ;; + *) + echo "unknown option: $1" >&2 + usage + ;; + esac + shift + done + + case "$VENDOR" in + "" | nvidia | amd) ;; + *) + echo "--vendor must be nvidia or amd" >&2 + exit 2 + ;; + esac +} + +if [ -t 1 ]; then + C_RESET=$'\033[0m' C_STEP=$'\033[1;36m' C_OK=$'\033[0;32m' C_WARN=$'\033[1;33m' C_ERR=$'\033[0;31m' +else + C_RESET='' C_STEP='' C_OK='' C_WARN='' C_ERR='' +fi + +FAILS=0 +WARNS=0 +step() { echo; echo "${C_STEP}==> $1${C_RESET}"; } +pass() { echo "${C_OK} [PASS] $1${C_RESET}"; } +info() { echo " [INFO] $1"; } +warn() { WARNS=$((WARNS + 1)); echo "${C_WARN} [WARN] $1${C_RESET}"; } +fail() { FAILS=$((FAILS + 1)); echo "${C_ERR} [FAIL] $1${C_RESET}"; } +fixed() { echo "${C_OK} [FIX ] $1${C_RESET}"; } +hint() { echo " $1"; } + +apt_has() { + command -v apt-cache >/dev/null 2>&1 && + apt-cache policy "$1" 2>/dev/null | grep -q 'Candidate: [^(]' +} + +pkg_installed() { + dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -q 'install ok installed' +} + +# Package installs never prompt: under "curl ... | bash" a prompt would read +# the rest of this script as its answer. +APT_UPDATED=false +apt_install() { + if [ "$APT_UPDATED" = false ]; then + DEBIAN_FRONTEND=noninteractive apt-get update /dev/null || return 1 + APT_UPDATED=true + fi + DEBIAN_FRONTEND=noninteractive apt-get install -y -o Dpkg::Options::=--force-confold "$@" /dev/null +} + +apply_step() { + local description="$1" + shift + if "$@"; then + fixed "$description" + else + fail "could not apply: $description" + return 1 + fi +} + +kernel_log() { + dmesg 2>/dev/null || journalctl -k -b 0 --no-pager 2>/dev/null +} + +check_machine() { + local container vm dev class + + step "Machine" + + container=$(systemd-detect-virt -c 2>/dev/null || true) + vm=$(systemd-detect-virt -v 2>/dev/null || true) + case "$container" in + lxc*) + IN_LXC=true + ON_HOST=" on the host" + ;; + none | "") ;; + *) + fail "running in a $container container; only bare metal, VMs and LXC are supported" + ;; + esac + + if [ "$IN_LXC" = true ]; then + info "LXC container on kernel $(uname -r); the host owns the GPU driver, firmware and CPU governor" + elif [ -n "$vm" ] && [ "$vm" != none ]; then + info "$vm VM on kernel $(uname -r); the GPU must be passed through to it" + else + info "bare metal, kernel $(uname -r)" + fi + info "$OS_ID $OS_VERSION" + + for dev in /sys/bus/pci/devices/*; do + class=$(cat "$dev/class" 2>/dev/null) + case "$class" in + 0x0300* | 0x0302* | 0x0380*) ;; + *) continue ;; + esac + case "$(cat "$dev/vendor" 2>/dev/null)" in + 0x10de) NVIDIA_GPUS+=("$dev") ;; + 0x1002) AMD_GPUS+=("$dev") ;; + esac + done + + if [ -z "$VENDOR" ]; then + if [ ${#NVIDIA_GPUS[@]} -gt 0 ] && [ ${#AMD_GPUS[@]} -gt 0 ]; then + fail "found both NVIDIA and AMD GPUs; pass --vendor nvidia or --vendor amd" + VENDOR=none + elif [ ${#NVIDIA_GPUS[@]} -gt 0 ]; then + VENDOR=nvidia + elif [ ${#AMD_GPUS[@]} -gt 0 ]; then + VENDOR=amd + else + fail "no NVIDIA or AMD GPU found on the PCI bus" + VENDOR=none + fi + fi +} + +driver_of() { + basename "$(readlink "$1/driver" 2>/dev/null)" 2>/dev/null +} + +cmdline_has() { + tr ' ' '\n' /dev/null)" = vddnb ] && return 0 + done + rev=$(od -An -tu1 -j2 -N1 "$dev/gpu_metrics" 2>/dev/null | tr -d ' ') + [ -n "$rev" ] && [ "$rev" -ge 2 ] +} + +amd_integrated_note() { + if [ "$2" = true ]; then + echo "$1 is integrated, and this node has a discrete AMD GPU on amdgpu: leave it unticked in setup-gpu-nodes.sh's integrated list so it renders on the discrete card" + else + echo "$1 is integrated and the only AMD GPU on amdgpu: tick this node in setup-gpu-nodes.sh's integrated list (5stack-gpu-integrated=true)" + fi +} + +check_amd() { + local dev pci driver node render integrated bar0 size level has_discrete=false + + step "AMD GPU" + + # A card on vfio-pci or without a driver renders nothing here, so it does + # not count as a discrete GPU. + for dev in "${AMD_GPUS[@]}"; do + [ "$(driver_of "$dev")" = amdgpu ] || continue + amd_integrated "$dev" || has_discrete=true + done + + if [ -d /sys/module/amdgpu ]; then + pass "amdgpu driver loaded" + else + fail "amdgpu driver not loaded" + [ "$IN_LXC" = true ] && hint "Load it on the host; a container cannot." + fi + + for dev in "${AMD_GPUS[@]}"; do + pci=$(basename "$dev") + driver=$(driver_of "$dev") + integrated=false + amd_integrated "$dev" && integrated=true + + if [ "$driver" != amdgpu ]; then + fail "$pci is bound to '${driver:-nothing}', not amdgpu" + [ "$driver" = vfio-pci ] && hint "It is reserved for VM passthrough; an LXC or this host cannot use it." + continue + fi + + render="" + for node in "$dev"/drm/renderD*; do + [ -e "$node" ] && render=$(basename "$node") && break + done + if [ -z "$render" ]; then + fail "$pci has no render node" + continue + fi + if [ -c "/dev/dri/$render" ]; then + pass "$pci: /dev/dri/$render ($([ "$integrated" = true ] && echo integrated || echo discrete))" + else + fail "$pci: /dev/dri/$render is missing" + [ "$IN_LXC" = true ] && hint "Bind /dev/dri into the container (proxmox-gpu-lxc.sh does)." + fi + + if [ "$integrated" = true ]; then + info "$(amd_integrated_note "$pci" "$has_discrete")" + else + # BAR0 is the CPU's window into VRAM. 256 MiB means Above 4G + # decoding (and Resizable BAR) is off in the firmware. + bar0=$(head -1 "$dev/resource" 2>/dev/null) + size=$(( $(echo "$bar0" | awk '{print $2}') - $(echo "$bar0" | awk '{print $1}') + 1 )) + if [ "$size" -gt 0 ] && [ "$size" -le $((256 * 1024 * 1024)) ]; then + warn "$pci: VRAM window is $((size / 1024 / 1024)) MiB; enable Above 4G decoding and Resizable BAR in the firmware" + fi + fi + + level=$(cat "$dev/power_dpm_force_performance_level" 2>/dev/null) + if [ -n "$level" ] && [ "$level" != auto ]; then + warn "$pci: power_dpm_force_performance_level is '$level'; 5stack expects 'auto' and never changes it" + fi + done + + for arg in nomodeset amdgpu.modeset=0 amdgpu.dpm=0; do + if cmdline_has "$arg"; then + fail "kernel command line has $arg, which disables what the streamer needs; remove it" + fi + done + + if [ -e /dev/kfd ]; then + info "/dev/kfd present (not needed for rendering)" + fi + + local log + log=$(kernel_log) + if [ ! -d /sys/module/amdgpu ]; then + : + elif [ -z "$log" ]; then + info "kernel log not readable here; check firmware errors with 'dmesg | grep amdgpu' on the host" + elif echo "$log" | grep -q "Direct firmware load for amdgpu/.* failed"; then + fail "amdgpu firmware is missing: $(echo "$log" | grep -o "amdgpu/[^ ]*" | head -1)" + hint "Install the AMD firmware package$ON_HOST, then reboot it." + elif echo "$log" | grep -q "amdgpu.*Fatal error during GPU init"; then + fail "amdgpu failed to start the GPU this boot; see 'dmesg | grep amdgpu'" + else + pass "no amdgpu firmware or init errors this boot" + if echo "$log" | grep -qE "amdgpu.*(GPU reset begin|ring .* timeout)"; then + warn "the GPU has reset this boot; see 'dmesg | grep -E \"reset|timeout\"'" + fi + fi + + if [ "$IN_LXC" = false ] && command -v apt-cache >/dev/null 2>&1; then + local firmware="" + for candidate in linux-firmware-amd-graphics firmware-amd-graphics pve-firmware linux-firmware; do + if pkg_installed "$candidate"; then + firmware="$candidate" + break + fi + done + if [ -n "$firmware" ]; then + pass "firmware package $firmware installed" + else + for candidate in linux-firmware-amd-graphics firmware-amd-graphics; do + if apt_has "$candidate"; then + if [ "$APPLY" = true ]; then + apply_step "installed $candidate; reboot to load it" apt_install "$candidate" + else + warn "$candidate is not installed (--apply installs it)" + fi + break + fi + done + fi + fi + + if [ "$IN_LXC" = false ] && [ "$OS_ID" = ubuntu ] && [ "$OS_VERSION" = "24.04" ]; then + case "$(uname -r)" in + 6.8.*) warn "Ubuntu 24.04's original 6.8 kernel predates recent Radeons; install the HWE kernel (linux-generic-hwe-24.04)" ;; + esac + fi + + if [ "$IN_LXC" = false ]; then + local dm + for dm in gdm gdm3 sddm lightdm display-manager; do + if systemctl is-active --quiet "$dm" 2>/dev/null; then + warn "$dm is running; a desktop session holds the GPU's display, so a headless install is recommended" + break + fi + done + fi +} + +# 595.99.02 from "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 595.99.02 Release Build ..." +nvidia_kernel_version() { + head -1 /proc/driver/nvidia/version 2>/dev/null | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1 +} + +install_nvidia_userspace() { + # The container shares the host's kernel module, so it needs the same + # version's libraries and nothing else. The .run installer is the one + # source that has every version, matching the host whatever installed it. + local version="$1" run opt + run="/root/NVIDIA-Linux-x86_64-$version.run" + if [ ! -f "$run" ]; then + command -v curl >/dev/null 2>&1 || apt_install curl || { + fail "could not install curl to download the NVIDIA $version installer" + return 1 + } + # Download beside the target, so an interrupted download is never + # mistaken for the installer on the next run. + if ! curl -fsSL -o "$run.part" "https://us.download.nvidia.com/XFree86/Linux-x86_64/$version/NVIDIA-Linux-x86_64-$version.run" /dev/null | grep -oE -- '--no-kernel-modules?' | head -1) + if sh "$run" --silent "${opt:---no-kernel-module}" /dev/null 2>&1; then + fixed "installed the NVIDIA $version libraries" + else + fail "the NVIDIA $version installer failed; see /var/log/nvidia-installer.log" + return 1 + fi +} + +# The installer's output goes to /dev/null here, not on the apply_step call, +# which would swallow apply_step's own result line with it. +install_nvidia_driver() { + DEBIAN_FRONTEND=noninteractive ubuntu-drivers install /dev/null +} + +# -c0 makes nvidia0 and nvidiactl, -c0 -u the uvm nodes, and only -m makes nvidia-modeset. +nvidia_nodes_hint() { + local missing="$1" commands=() + case "$missing" in *"/dev/nvidia0"* | *"/dev/nvidiactl"*) commands+=("nvidia-modprobe -c0") ;; esac + case "$missing" in *"/dev/nvidia-uvm"*) commands+=("nvidia-modprobe -c0 -u") ;; esac + case "$missing" in *"/dev/nvidia-modeset"*) commands+=("nvidia-modprobe -m") ;; esac + local joined="" command + for command in "${commands[@]}"; do + joined+="${joined:+; }$command" + done + echo "Create them with '$joined'. They are gone after a reboot, so run that at boot too (a oneshot systemd unit)." +} + +install_nvidia_container_toolkit() { + if [ "$OS_ID" != ubuntu ] && [ "$OS_ID" != debian ]; then + fail "install nvidia-container-toolkit by hand on $OS_ID: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html" + return 1 + fi + apt_install curl gnupg ca-certificates && + curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey /etc/apt/sources.list.d/nvidia-container-toolkit.list || return 1 + APT_UPDATED=false + apt_install nvidia-container-toolkit +} + +check_nvidia() { + local dev pci driver kernel_version smi_version missing node + + step "NVIDIA GPU" + + for dev in "${NVIDIA_GPUS[@]}"; do + pci=$(basename "$dev") + driver=$(driver_of "$dev") + case "$driver" in + nvidia) pass "$pci bound to nvidia" ;; + nouveau) + fail "$pci is driven by nouveau; install the NVIDIA driver$ON_HOST" + ;; + vfio-pci) + fail "$pci is reserved for VM passthrough (vfio-pci); an LXC or this host cannot use it" + ;; + *) + fail "$pci has no driver" + if [ "$IN_LXC" = false ] && mokutil --sb-state 2>/dev/null | grep -q enabled; then + hint "Secure Boot is on: use the signed prebuilt modules ('ubuntu-drivers install') or enroll the DKMS key with mokutil." + fi + ;; + esac + done + + kernel_version=$(nvidia_kernel_version) + if [ -z "$kernel_version" ]; then + fail "the NVIDIA kernel module is not loaded" + if [ "$IN_LXC" = false ] && [ "$APPLY" = true ] && [ "$OS_ID" = ubuntu ] && command -v ubuntu-drivers >/dev/null 2>&1; then + apply_step "installed the recommended NVIDIA driver; reboot, then run this again" install_nvidia_driver + elif [ "$IN_LXC" = false ] && [ "$OS_ID" = ubuntu ]; then + hint "Install it with 'ubuntu-drivers install' (--apply does), then reboot." + fi + else + pass "kernel module $kernel_version" + fi + + missing="" + for node in /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-modeset; do + [ -c "$node" ] || missing+=" $node" + done + ls /dev/nvidia[0-9]* >/dev/null 2>&1 || missing+=" /dev/nvidia0" + if [ -z "$missing" ]; then + pass "device nodes present" + else + fail "missing device nodes:$missing" + if [ "$IN_LXC" = true ]; then + hint "Bind them into the container (proxmox-gpu-lxc.sh does), and create them on the host first." + else + hint "$(nvidia_nodes_hint "$missing")" + fi + fi + + if [ -n "$kernel_version" ]; then + smi_version=$(nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1) + if [ "$smi_version" = "$kernel_version" ]; then + pass "nvidia-smi sees $(nvidia-smi --query-gpu=name --format=csv,noheader | head -1), driver $smi_version" + elif [ "$IN_LXC" = true ]; then + if [ "$APPLY" = true ]; then + install_nvidia_userspace "$kernel_version" + else + fail "the container's NVIDIA libraries ($( [ -n "$smi_version" ] && echo "$smi_version" || echo none)) do not match the host's module $kernel_version (--apply installs $kernel_version)" + fi + else + fail "nvidia-smi does not work: $(nvidia-smi 2>&1 | head -1)" + hint "A library/module version mismatch clears after a reboot." + fi + fi + + if command -v nvidia-container-runtime >/dev/null 2>&1; then + pass "nvidia-container-toolkit installed" + elif [ "$APPLY" = true ]; then + apply_step "installed nvidia-container-toolkit" install_nvidia_container_toolkit + else + fail "nvidia-container-toolkit is not installed (--apply installs it)" + fi + + if command -v nvidia-container-cli >/dev/null 2>&1; then + if nvidia-container-cli info >/dev/null 2>&1; then + pass "nvidia-container-cli reaches the GPU" + else + fail "nvidia-container-cli cannot reach the GPU: $(nvidia-container-cli info 2>&1 | tail -1)" + fi + fi + + # k3s writes an nvidia runtime into its containerd config only when it + # finds nvidia-container-runtime at start, so a toolkit installed after + # the join needs a k3s restart. + local containerd=/var/lib/rancher/k3s/agent/etc/containerd/config.toml service + service=$(k3s_service) + if [ -f "$containerd" ]; then + if grep -q nvidia-container-runtime "$containerd"; then + pass "k3s containerd has the nvidia runtime" + elif [ "$APPLY" = true ] && command -v nvidia-container-runtime >/dev/null 2>&1; then + apply_step "restarted $service to pick up the nvidia runtime" systemctl restart "$service" + else + fail "k3s containerd has no nvidia runtime; restart $service after installing the toolkit" + fi + fi +} + +k3s_service() { + if systemctl is-enabled --quiet k3s 2>/dev/null; then + echo k3s + else + echo k3s-agent + fi +} + +# Same script and unit as the panel's CPU governor tuning, so a node never has two. +# cpufrequtils's boot script would otherwise set its own governor. +install_governor_unit() { + local unit="$SYSTEMD_DIR/5stack-cpu-governor.service" governor + mkdir -p "${GOVERNOR_SCRIPT%/*}" || return 1 + cat >"$GOVERNOR_SCRIPT.5stack-new" <<'SCRIPT' || return 1 +#!/bin/sh +for governor in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do + echo performance > "$governor" +done +SCRIPT + chmod 0755 "$GOVERNOR_SCRIPT.5stack-new" && mv -f "$GOVERNOR_SCRIPT.5stack-new" "$GOVERNOR_SCRIPT" || return 1 + cat >"$unit.5stack-new" <"$CPUFREQUTILS_DEFAULT.5stack-new" && + mv -f "$CPUFREQUTILS_DEFAULT.5stack-new" "$CPUFREQUTILS_DEFAULT" || return 1 + fi + # restart, not enable --now, which leaves an active unit alone. + systemctl daemon-reload && + systemctl enable 5stack-cpu-governor.service >/dev/null 2>&1 && + systemctl restart 5stack-cpu-governor.service >/dev/null 2>&1 || return 1 + for governor in "$CPU_SYSFS"/cpu*/cpufreq/scaling_governor; do + [ -f "$governor" ] || continue + if [ "$(cat "$governor")" != performance ]; then + hint "$governor is still $(cat "$governor")" + return 1 + fi + done +} + +# sudo drops exported variables, so the pin goes after it. +k3s_join_hint() { + echo "curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=$1 sh -s -- join --panel-key " +} + +check_common() { + step "Node" + + local tool + for tool in curl jq; do + if command -v "$tool" >/dev/null 2>&1; then + pass "$tool installed" + elif [ "$APPLY" = true ] && command -v apt-get >/dev/null 2>&1; then + apply_step "installed $tool" apt_install "$tool" + else + fail "$tool is not installed (--apply installs it); the node setup command needs it" + fi + done + + local free_gb + free_gb=$(df -BG / | awk 'NR==2 {print $4}' | tr -d 'G') + if [ "${free_gb:-0}" -ge 120 ]; then + pass "${free_gb} GB free on /" + else + warn "${free_gb} GB free on /; the node setup asks for 120 GB on a fresh node" + fi + + local governor + governor=$(cat "$CPU_SYSFS/cpu0/cpufreq/scaling_governor" 2>/dev/null) + if [ -z "$governor" ]; then + info "no CPU frequency scaling exposed" + elif [ "$governor" = performance ]; then + pass "CPU governor is performance" + elif [ "$IN_LXC" = true ]; then + warn "CPU governor is '$governor'; set it to performance on the host (proxmox-gpu-lxc.sh --apply does)" + elif [ "$APPLY" = true ]; then + apply_step "CPU governor set to performance, at every boot (5stack-cpu-governor.service)" install_governor_unit + else + warn "CPU governor is '$governor'; render pods drop frames on a ramping CPU (--apply sets performance at every boot)" + fi + + if command -v k3s >/dev/null 2>&1; then + local installed service + installed=$(k3s --version 2>/dev/null | awk 'NR==1 {print $3}') + service=$(k3s_service) + if [ -n "$K3S_VERSION" ] && [ "$installed" != "$K3S_VERSION" ] && + [ "$(printf '%s\n' "$installed" "$K3S_VERSION" | sort -V | tail -1)" = "$installed" ]; then + warn "k3s $installed is newer than the panel's $K3S_VERSION; a kubelet newer than its API server is unsupported" + hint "Re-join with the version pinned: $(k3s_join_hint "$K3S_VERSION")" + else + pass "k3s $installed installed" + fi + if systemctl is-active --quiet "$service" 2>/dev/null; then + pass "$service running" + else + fail "$service is not running; see 'journalctl -u $service -n 50'" + fi + else + info "k3s not installed yet. The join command installs the latest k3s; to match the panel, join with the version pinned:" + hint "$(k3s_join_hint "${K3S_VERSION:-}")" + fi +} + +check_lxc() { + step "LXC container" + + if awk '$1 == 0 && $2 == 0 { found = 1 } END { exit !found }' /proc/self/uid_map; then + pass "privileged container" + else + fail "unprivileged container; k3s needs a privileged one" + fi + + local profile + profile=$(cat /proc/self/attr/current 2>/dev/null) + case "$profile" in + unconfined | "") pass "AppArmor unconfined" ;; + *) fail "AppArmor profile '$profile' blocks containerd; set lxc.apparmor.profile: unconfined" ;; + esac + + if [ -e /dev/kmsg ]; then + pass "/dev/kmsg present" + else + fail "/dev/kmsg missing; the kubelet needs it" + fi + if [ -c /dev/net/tun ]; then + pass "/dev/net/tun present" + else + fail "/dev/net/tun missing; Tailscale needs it" + fi + + if [ -w "$PROC_SYS_PROBE" ]; then + pass "/proc/sys/vm writable" + else + fail "/proc/sys/vm is read-only, so the kubelet cannot set its kernel values; set lxc.mount.auto: proc:rw sys:rw on the host" + fi + + local controllers controller + controllers=$(cat /sys/fs/cgroup/cgroup.controllers 2>/dev/null) + for controller in cpuset cpu memory pids; do + if [[ " $controllers " != *" $controller "* ]]; then + fail "cgroup controller '$controller' is not delegated to the container" + fi + done + + # Check what each module provides, not /sys/module: a module built into + # the host kernel (br_netfilter on Proxmox) has no /sys/module entry, and + # the container has no /lib/modules for the host kernel to ask modinfo. + if grep -qw overlay /proc/filesystems 2>/dev/null; then + pass "kernel module overlay loaded" + else + fail "kernel module overlay is not loaded; load it on the host" + fi + if [ -e /proc/sys/net/bridge/bridge-nf-call-iptables ]; then + pass "kernel module br_netfilter loaded" + else + fail "kernel module br_netfilter is not loaded; load it on the host" + fi + + # The kubelet's static CPU manager reads sysfs, which shows every host CPU even + # where lxcfs narrows /proc/cpuinfo, so it can pin pods to CPUs the container lacks. + local allowed host_cpus + allowed=$(nproc 2>/dev/null) + host_cpus=$(find /sys/devices/system/cpu -maxdepth 1 -name 'cpu[0-9]*' | wc -l) + if [ -n "$allowed" ] && [ "$host_cpus" -gt "$allowed" ]; then + warn "the container may use $allowed of the host's $host_cpus CPUs; remove its core limit (pct set --delete cores), or the kubelet can pin pods to CPUs it cannot use" + else + pass "the container may use every CPU ($allowed)" + fi +} + +main() { + parse_args "$@" + + if [ "$APPLY" = true ] && [ "$(id -u)" -ne 0 ]; then + echo "--apply must run as root" >&2 + exit 2 + fi + + # shellcheck source=/dev/null + . /etc/os-release 2>/dev/null + OS_ID="${ID:-unknown}" + OS_VERSION="${VERSION_ID:-}" + + check_machine + [ "$IN_LXC" = true ] && check_lxc + case "$VENDOR" in + amd) check_amd ;; + nvidia) check_nvidia ;; + esac + check_common + + echo + if [ "$FAILS" -gt 0 ]; then + echo "${C_ERR}$FAILS failed, $WARNS warnings${C_RESET}" + [ "$APPLY" = false ] && echo "Run with --apply to fix what this script can, then run it again." + exit 1 + fi + echo "${C_OK}ready for the ${VENDOR} GPU setup ($WARNS warnings)${C_RESET}" + echo "Next: join this node from the panel's Game Server Nodes page, then run setup-gpu-nodes.sh on the panel." +} + +# The tests source this file for its functions. Comparing BASH_SOURCE with $0 +# cannot tell that apart from "curl ... | bash", where both differ. +if [ -z "${GPU_NODE_CHECK_LIB:-}" ]; then + main "$@" +fi diff --git a/gpu-node/proxmox-gpu-lxc.sh b/gpu-node/proxmox-gpu-lxc.sh new file mode 100755 index 0000000..5fac0bd --- /dev/null +++ b/gpu-node/proxmox-gpu-lxc.sh @@ -0,0 +1,657 @@ +#!/bin/bash + +# Prepares a Proxmox host and one of its LXC containers to run a 5stack GPU node. +# Run it as root on the host, then gpu-node-check.sh in the container. See README.md. +# +# proxmox-gpu-lxc.sh CTID [--apply] [--vendor nvidia|amd] + +CTID="" +APPLY=false +VENDOR="" +PVE_LXC_DIR="${PVE_LXC_DIR:-/etc/pve/lxc}" +BACKUP_DIR="${BACKUP_DIR:-/root}" +SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}" +GOVERNOR_SCRIPT="${GOVERNOR_SCRIPT:-/usr/local/bin/5stack-cpu-governor.sh}" +CPU_SYSFS="${CPU_SYSFS:-/sys/devices/system/cpu}" +CPUFREQUTILS_DEFAULT="${CPUFREQUTILS_DEFAULT:-/etc/default/cpufrequtils}" +PROC_CMDLINE="${PROC_CMDLINE:-/proc/cmdline}" +NVIDIA_PROC_VERSION="${NVIDIA_PROC_VERSION:-/proc/driver/nvidia/version}" + +# The defaults of the panel's "Required disk space" settings, which the node setup +# enforces on / (df -BG): fresh install, then with CS2 installed. +REQUIRED_FREE_GB=120 +REQUIRED_FREE_EXISTING_GB=60 +SYSTEM_HEADROOM_GB=20 +CS2_DIR=/opt/5stack/serverfiles/game/csgo + +# A container cannot load modules, so the host loads what k3s, flannel and kube-proxy need. +K3S_MODULES="overlay br_netfilter nf_conntrack vxlan ip_tables iptable_nat iptable_filter iptable_mangle xt_conntrack xt_mark xt_comment nf_tables tun" +MODULES_FILE=/etc/modules-load.d/5stack-k3s.conf + +if [ -t 1 ]; then + C_RESET=$'\033[0m' C_STEP=$'\033[1;36m' C_OK=$'\033[0;32m' C_WARN=$'\033[1;33m' C_ERR=$'\033[0;31m' +else + C_RESET='' C_STEP='' C_OK='' C_WARN='' C_ERR='' +fi + +FAILS=0 +WARNS=0 +step() { echo; echo "${C_STEP}==> $1${C_RESET}"; } +pass() { echo "${C_OK} [PASS] $1${C_RESET}"; } +info() { echo " [INFO] $1"; } +warn() { WARNS=$((WARNS + 1)); echo "${C_WARN} [WARN] $1${C_RESET}"; } +fail() { FAILS=$((FAILS + 1)); echo "${C_ERR} [FAIL] $1${C_RESET}"; } +fixed() { echo "${C_OK} [FIX ] $1${C_RESET}"; } +hint() { echo " $1"; } + +usage() { + echo "usage: proxmox-gpu-lxc.sh CTID [--apply] [--vendor nvidia|amd]" >&2 + exit 2 +} + +apply_step() { + local description="$1" + shift + if "$@"; then + fixed "$description" + else + fail "could not apply: $description" + return 1 + fi +} + +# Reads only the container's own section; snapshot sections follow it. +conf_value() { + awk -v key="$2" '/^\[/ { exit } index($0, key ": ") == 1 { print substr($0, length(key) + 3); exit }' "$1" +} + +# Without a unit a Proxmox size is in bytes, as pve-common's parse_size reads it. +size_gb() { + local size="$1" number unit + number="${size%[KMGT]}" + unit="${size#"$number"}" + case "$number" in + '' | .* | *. | *.*.* | *[!0-9.]*) return ;; + esac + awk -v n="$number" -v unit="$unit" 'BEGIN { + divisor["T"] = 1 / 1024; divisor["G"] = 1; divisor["M"] = 1024 + divisor["K"] = 1024 * 1024; divisor[""] = 1024 * 1024 * 1024 + printf "%d\n", n / divisor[unit] + }' +} + +features_with_nesting() { + local feature out="" + local IFS=, + for feature in $1; do + case "$feature" in nesting=*) continue ;; esac + out+="${out:+,}$feature" + done + echo "${out:+$out,}nesting=1" +} + +lxc_lines() { + echo "lxc.apparmor.profile: unconfined" + echo "lxc.cgroup2.devices.allow: a" + echo "lxc.cap.drop: " + echo "lxc.mount.auto: proc:rw sys:rw" + echo "lxc.mount.entry: /dev/kmsg dev/kmsg none bind,optional,create=file" + echo "lxc.mount.entry: /dev/net/tun dev/net/tun none bind,optional,create=file" + case "$1" in + amd) + echo "lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir" + echo "lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file" + ;; + nvidia) + local node + for node in /dev/nvidia[0-9]* /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-uvm-tools /dev/nvidia-modeset; do + case "$node" in '/dev/nvidia[0-9]*') node=/dev/nvidia0 ;; esac + echo "lxc.mount.entry: $node ${node#/} none bind,optional,create=file" + done + echo "lxc.mount.entry: /dev/nvidia-caps dev/nvidia-caps none bind,optional,create=dir" + ;; + esac +} + +# Proxmox rewrites the file on every change, so trailing spaces do not count. +conf_has_line() { + LINE="$2" awk ' + function trim(s) { sub(/[ \t]+$/, "", s); return s } + /^\[/ { exit } + trim($0) == trim(ENVIRON["LINE"]) { found = 1; exit } + END { exit !found } + ' "$1" +} + +# Proxmox's own passthrough (devN: /dev/kfd,gid=993) gives the container the +# device too; for /dev/dri a render node is the one the GPU needs. +conf_passes_device() { + DEVICE="$2" awk ' + /^\[/ { exit } + /^dev[0-9]+: / { + path = substr($0, index($0, " ") + 1) + sub(/^path=/, "", path) + sub(/,.*/, "", path) + want = ENVIRON["DEVICE"] + if (path == want || (want == "/dev/dri" && index(path, "/dev/dri/renderD") == 1)) { found = 1; exit } + } + END { exit !found } + ' "$1" +} + +missing_lines() { + local file="$1" line device + shift + for line in "$@"; do + conf_has_line "$file" "$line" && continue + case "$line" in + "lxc.mount.entry: /dev/"*) + device=${line#lxc.mount.entry: } + conf_passes_device "$file" "${device%% *}" && continue + ;; + esac + echo "$line" + done +} + +add_lxc_lines() { + local file="$1" missing tmp before after added + shift + missing=$(missing_lines "$file" "$@") || return 1 + [ -n "$missing" ] || return 0 + + # Like pve-common's file_set_contents: renamed over, so a failed write never cuts the config off. + tmp="$file.tmp.$$" + rm -f "$tmp" + # Blank lines are held back until the next line shows whether they end + # the container's section. + if ! ADD_LINES="$missing" awk ' + !done && /^$/ { blanks = blanks "\n"; next } + !done && /^\[/ { print ENVIRON["ADD_LINES"]; done = 1 } + { printf "%s", blanks; blanks = ""; print } + END { if (!done) print ENVIRON["ADD_LINES"]; printf "%s", blanks } + ' "$file" >"$tmp"; then + hint "could not write $tmp; $file is unchanged" + rm -f "$tmp" + return 1 + fi + + before=$(grep -c '' "$file") + after=$(grep -c '' "$tmp") + added=$(printf '%s\n' "$missing" | wc -l) + if [ ! -s "$tmp" ] || [ "$after" -ne $((before + added)) ]; then + hint "the new config came out wrong ($after lines, expected $((before + added))); $file is unchanged" + rm -f "$tmp" + return 1 + fi + if ! mv -f "$tmp" "$file"; then + hint "could not rename $tmp over $file; $file is unchanged" + rm -f "$tmp" + return 1 + fi +} + +# The lock pct and backups take on the container's config. +locked_add_lxc_lines() { + local lock="/run/lock/lxc/pve-config-$CTID.lock" + mkdir -p "${lock%/*}" + ( + flock -w 30 9 || exit 1 + add_lxc_lines "$@" + ) 9>"$lock" +} + +check_host_gpu() { + local dev class driver pci + local nvidia=() amd=() + + step "Host GPU" + + for dev in /sys/bus/pci/devices/*; do + class=$(cat "$dev/class" 2>/dev/null) + case "$class" in 0x0300* | 0x0302* | 0x0380*) ;; *) continue ;; esac + case "$(cat "$dev/vendor" 2>/dev/null)" in + 0x10de) nvidia+=("$dev") ;; + 0x1002) amd+=("$dev") ;; + esac + done + + if [ -z "$VENDOR" ]; then + if [ ${#nvidia[@]} -gt 0 ] && [ ${#amd[@]} -gt 0 ]; then + fail "found both NVIDIA and AMD GPUs; pass --vendor nvidia or --vendor amd" + return + elif [ ${#nvidia[@]} -gt 0 ]; then + VENDOR=nvidia + elif [ ${#amd[@]} -gt 0 ]; then + VENDOR=amd + else + fail "no NVIDIA or AMD GPU found on the PCI bus" + return + fi + fi + + local devices=("${amd[@]}") expected=amdgpu + [ "$VENDOR" = nvidia ] && devices=("${nvidia[@]}") expected=nvidia + + for dev in "${devices[@]}"; do + pci=$(basename "$dev") + driver=$(basename "$(readlink "$dev/driver" 2>/dev/null)" 2>/dev/null) + if [ "$driver" = "$expected" ]; then + pass "$pci bound to $driver" + elif [ "$driver" = vfio-pci ]; then + fail "$pci is bound to vfio-pci for VM passthrough; a container shares the host's driver instead" + hint "Remove it from the vfio-pci ids (/etc/modprobe.d) and any VM's hostpci, then reboot." + else + fail "$pci is bound to '${driver:-nothing}', not $expected" + fi + done + + if [ "$VENDOR" = amd ]; then + check_host_amd + else + check_host_nvidia + fi +} + +check_host_amd() { + local arg + for arg in nomodeset amdgpu.modeset=0 amdgpu.dpm=0; do + if tr ' ' '\n' <"$PROC_CMDLINE" | grep -qx "$arg"; then + fail "the kernel command line has $arg; remove it" + fi + done + + if ls /dev/dri/renderD* >/dev/null 2>&1; then + pass "render nodes: $(cd /dev/dri && echo renderD*)" + else + fail "no /dev/dri/renderD* on the host" + fi + + local log + log=$(journalctl -k -b 0 --no-pager 2>/dev/null) + if echo "$log" | grep -q "Direct firmware load for amdgpu/.* failed"; then + fail "amdgpu firmware is missing: $(echo "$log" | grep -o "amdgpu/[^ ]*" | head -1)" + hint "Proxmox ships GPU firmware in pve-firmware; update it ('apt install pve-firmware') and the kernel, then reboot." + elif echo "$log" | grep -q "amdgpu.*Fatal error during GPU init"; then + fail "amdgpu failed to start the GPU this boot; see 'journalctl -k -b | grep amdgpu'" + else + pass "no amdgpu firmware or init errors this boot" + fi +} + +NVIDIA_UNIT="$SYSTEMD_DIR/5stack-nvidia-devices.service" +NVIDIA_NODES="/dev/nvidia0 /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-uvm-tools /dev/nvidia-modeset" + +check_host_nvidia() { + local version + version=$(head -1 "$NVIDIA_PROC_VERSION" 2>/dev/null | grep -oE '[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1) + if [ -z "$version" ]; then + fail "the NVIDIA driver is not loaded on the host" + hint "Install it on the host with the proxmox-headers package and NVIDIA's .run installer, then reboot." + return + fi + pass "NVIDIA driver $version; the container needs the same version's libraries (gpu-node-check.sh --apply installs them)" + + # The module creates no device nodes until something opens the GPU, and the + # container cannot create them, so the host makes them at boot. + local missing + missing=$(missing_nvidia_nodes) + if [ -z "$missing" ] && [ -f "$NVIDIA_UNIT" ]; then + pass "device nodes present, and created at every boot" + elif [ "$APPLY" = true ] && command -v nvidia-modprobe >/dev/null 2>&1; then + apply_step "device nodes created, and at every boot ($NVIDIA_UNIT)" install_nvidia_unit + elif [ "$APPLY" = true ]; then + if [ -n "$missing" ]; then + fail "missing device nodes:$missing, and --apply cannot create them without nvidia-modprobe" + else + warn "nothing creates the NVIDIA device nodes at boot, and --apply cannot add the unit that does without nvidia-modprobe" + fi + hint "NVIDIA's .run installer installs nvidia-modprobe; install the driver with it, then run this again." + elif [ -n "$missing" ]; then + fail "missing device nodes:$missing (--apply creates them at every boot with nvidia-modprobe)" + else + warn "nothing creates the NVIDIA device nodes at boot, so a container that starts first misses them (--apply adds a unit)" + fi +} + +missing_nvidia_nodes() { + local node + for node in $NVIDIA_NODES; do + [ -c "$node" ] || printf ' %s' "$node" + done +} + +install_nvidia_unit() { + local missing + cat >"$NVIDIA_UNIT" <<'UNIT' || return 1 +[Unit] +Description=Create the NVIDIA device nodes for 5stack GPU containers +After=systemd-modules-load.service +Before=pve-guests.service + +[Service] +Type=oneshot +ExecStart=-/usr/bin/nvidia-smi -L +ExecStart=-/usr/bin/nvidia-modprobe -c0 +ExecStart=-/usr/bin/nvidia-modprobe -c0 -u +ExecStart=-/usr/bin/nvidia-modprobe -m +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target +UNIT + # restart, not enable --now, which leaves an active unit alone. + systemctl daemon-reload && + systemctl enable 5stack-nvidia-devices.service >/dev/null 2>&1 && + systemctl restart 5stack-nvidia-devices.service >/dev/null 2>&1 || return 1 + missing=$(missing_nvidia_nodes) + if [ -n "$missing" ]; then + hint "still missing:$missing" + return 1 + fi +} + +write_modules_file() { + # shellcheck disable=SC2086 # one module per line + printf '%s\n' $K3S_MODULES >"$MODULES_FILE" +} + +check_host_kernel() { + step "Host kernel modules" + + local module missing=() + for module in $K3S_MODULES; do + [ -d "/sys/module/$module" ] || modinfo "$module" 2>/dev/null | grep -q '^filename:.*(builtin)' || missing+=("$module") + done + local persisted=true + [ -f "$MODULES_FILE" ] || persisted=false + + if [ ${#missing[@]} -eq 0 ] && [ "$persisted" = true ]; then + pass "k3s modules loaded, and at every boot" + elif [ "$APPLY" = true ]; then + apply_step "k3s modules load at every boot ($MODULES_FILE)" write_modules_file + for module in "${missing[@]}"; do + apply_step "loaded $module" modprobe "$module" + done + elif [ ${#missing[@]} -gt 0 ]; then + fail "not loaded: ${missing[*]} (--apply loads them, and at every boot)" + else + warn "the k3s modules are loaded now but not at every boot (--apply writes $MODULES_FILE)" + fi + + local governor + governor=$(cat "$CPU_SYSFS/cpu0/cpufreq/scaling_governor" 2>/dev/null) + if [ -z "$governor" ] || [ "$governor" = performance ]; then + pass "CPU governor ${governor:-not exposed}" + elif [ "$APPLY" = true ]; then + apply_step "CPU governor set to performance, at every boot (5stack-cpu-governor.service)" install_governor_unit + else + warn "CPU governor is '$governor'; render pods drop frames on a ramping CPU (--apply sets performance at every boot)" + fi +} + +# Same script and unit as the panel's CPU governor tuning, so a node never has two. +# cpufrequtils's boot script would otherwise set its own governor. +install_governor_unit() { + local unit="$SYSTEMD_DIR/5stack-cpu-governor.service" governor + mkdir -p "${GOVERNOR_SCRIPT%/*}" || return 1 + cat >"$GOVERNOR_SCRIPT.5stack-new" <<'SCRIPT' || return 1 +#!/bin/sh +for governor in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do + echo performance > "$governor" +done +SCRIPT + chmod 0755 "$GOVERNOR_SCRIPT.5stack-new" && mv -f "$GOVERNOR_SCRIPT.5stack-new" "$GOVERNOR_SCRIPT" || return 1 + cat >"$unit.5stack-new" <"$CPUFREQUTILS_DEFAULT.5stack-new" && + mv -f "$CPUFREQUTILS_DEFAULT.5stack-new" "$CPUFREQUTILS_DEFAULT" || return 1 + fi + # restart, not enable --now, which leaves an active unit alone. + systemctl daemon-reload && + systemctl enable 5stack-cpu-governor.service >/dev/null 2>&1 && + systemctl restart 5stack-cpu-governor.service >/dev/null 2>&1 || return 1 + for governor in "$CPU_SYSFS"/cpu*/cpufreq/scaling_governor; do + [ -f "$governor" ] || continue + if [ "$(cat "$governor")" != performance ]; then + hint "$governor is still $(cat "$governor")" + return 1 + fi + done +} + +container_free_gb() { + pct status "$CTID" 2>/dev/null | grep -q 'status: running' || return 0 + pct exec "$CTID" -- df -BG / 2>/dev/null | awk 'NR == 2 { sub(/G$/, "", $4); print $4 }' +} + +container_has_cs2() { + pct exec "$CTID" -- test -d "$CS2_DIR" 2>/dev/null +} + +disk_settings_hint() { + hint "The panel sets both minimums: \"Required disk space - fresh install\" ($REQUIRED_FREE_GB GB by default) and \"Required disk space - existing install\" ($REQUIRED_FREE_EXISTING_GB GB), under Settings, Application, Servers." +} + +check_disk() { + local rootfs="$1" size gb free want + size=$(echo "$rootfs" | grep -oE 'size=[0-9.]+[KMGT]?' | cut -d= -f2) + gb=$(size_gb "$size") + if [ -z "$gb" ]; then + warn "could not read the root disk size from '$rootfs'" + return + fi + want=$((REQUIRED_FREE_GB + SYSTEM_HEADROOM_GB)) + free=$(container_free_gb) + case "$free" in *[!0-9]*) free="" ;; esac + + if [ -n "$free" ]; then + local need=$REQUIRED_FREE_GB when="before CS2 is installed" + if container_has_cs2; then + need=$REQUIRED_FREE_EXISTING_GB + when="with CS2 already installed" + fi + if [ "$free" -ge "$need" ]; then + pass "root disk $size, ${free} GB free ($when the node setup needs $need GB)" + else + fail "root disk $size has ${free} GB free; $when the node setup needs $need GB free" + hint "Grow it: pct resize $CTID rootfs +$((need - free + SYSTEM_HEADROOM_GB))G" + disk_settings_hint + fi + elif [ "$gb" -lt "$REQUIRED_FREE_GB" ]; then + fail "root disk is $size; before CS2 is installed the node setup needs $REQUIRED_FREE_GB GB free on it" + hint "Grow it: pct resize $CTID rootfs ${want}G" + hint "If CS2 is already installed in it, start the container and run this again: $REQUIRED_FREE_EXISTING_GB GB free is then enough." + disk_settings_hint + elif [ "$gb" -lt "$want" ]; then + warn "root disk is $size; the node setup needs $REQUIRED_FREE_GB GB free, which leaves little for the system (${want}G is safer)" + hint "Start the container and run this again to check its free space." + else + pass "root disk $size (start the container to check its free space)" + fi +} + +proc_rw_warning() { + warn "lxc.mount.auto: proc:rw lets the kubelet set vm.overcommit_memory=1, kernel.panic=10 and kernel.panic_on_oops=1, which apply to the whole host" + hint "The host then reboots 10 seconds after a kernel oops, taking every guest down. Decide if that is acceptable before restarting the container." +} + +check_container() { + local conf="$PVE_LXC_DIR/$CTID.conf" + + step "Container $CTID" + + if [ ! -f "$conf" ]; then + fail "no container $CTID ($conf)" + return + fi + + if [ "$(conf_value "$conf" unprivileged)" = 1 ]; then + fail "the container is unprivileged; k3s needs a privileged container" + hint "Back it up and restore it as privileged: vzdump $CTID, then pct restore with --unprivileged 0." + # Every hint past here offers --apply, which this container refuses. + if [ "$APPLY" = true ] || [ "$APPLY_REFUSED" = true ]; then + hint "--apply changes nothing in this container until it is privileged." + return + fi + else + pass "privileged container" + fi + + local rootfs storage + rootfs=$(conf_value "$conf" rootfs) + check_disk "$rootfs" + + storage=${rootfs%%:*} + if command -v pvesm >/dev/null 2>&1 && pvesm status 2>/dev/null | awk -v s="$storage" '$1 == s { print $2 }' | grep -qx zfspool; then + local zfs + zfs=$(zfs version 2>/dev/null | head -1 | grep -oE '[0-9]+\.[0-9]+' | head -1) + if [ -n "$zfs" ] && [ "$(printf '%s\n' 2.2 "$zfs" | sort -V | head -1)" != 2.2 ]; then + fail "the root disk is on ZFS $zfs; containerd's overlayfs needs ZFS 2.2 or newer" + fi + fi + + local cores host_cpus + cores=$(conf_value "$conf" cores) + host_cpus=$(nproc --all) + if [ -n "$cores" ] && [ "$cores" -lt "$host_cpus" ]; then + warn "the container is limited to $cores of $host_cpus cores; the kubelet's static CPU manager can pin pods to cores it cannot use" + hint "Remove the limit: pct set $CTID --delete cores" + fi + + # A change to a running container waits in its pending section until the + # next restart; pct config shows it, conf_value does not. + local features pending + features=$(conf_value "$conf" features) + pending=$(pct config "$CTID" 2>/dev/null | sed -n 's/^features: //p') + if [[ ",$features," == *",nesting=1,"* ]]; then + pass "nesting enabled" + elif [[ ",$pending," == *",nesting=1,"* ]]; then + warn "nesting turns on at the next container restart" + RESTART_NEEDED=true + elif [ "$APPLY" = true ]; then + backup_conf "$conf" || return + apply_step "enabled nesting" pct set "$CTID" --features "$(features_with_nesting "$features")" && + RESTART_NEEDED=true + else + fail "nesting is off (--apply turns it on)" + fi + + [ -z "$VENDOR" ] && return + + local lines=() line missing=() + mapfile -t lines < <(lxc_lines "$VENDOR") + mapfile -t missing < <(missing_lines "$conf" "${lines[@]}") + if [ ${#missing[@]} -eq 0 ]; then + pass "LXC settings for k3s and the $VENDOR GPU in place" + return + fi + for line in "${missing[@]}"; do + [ "$line" = "lxc.mount.auto: proc:rw sys:rw" ] && proc_rw_warning + done + if [ "$APPLY" = true ]; then + backup_conf "$conf" || return + apply_step "added ${#missing[@]} LXC lines for k3s and the $VENDOR GPU" locked_add_lxc_lines "$conf" "${lines[@]}" && + RESTART_NEEDED=true + else + fail "${#missing[@]} LXC lines for k3s and the $VENDOR GPU are missing (--apply adds them):" + for line in "${missing[@]}"; do + hint "$line" + done + fi +} + +BACKED_UP=false +backup_conf() { + [ "$BACKED_UP" = true ] && return 0 + local backup + backup="$BACKUP_DIR/5stack-lxc-$CTID.conf.$(date +%Y%m%d-%H%M%S)" + if cp "$1" "$backup" 2>/dev/null && cmp -s "$1" "$backup"; then + BACKED_UP=true + info "backed up the container config to $backup" + return 0 + fi + fail "could not back up $1 to $backup, so --apply changes nothing in the container" + return 1 +} + +# --apply changes the host only for a container it can then finish. +APPLY_REFUSED=false +preflight_apply() { + local conf="$PVE_LXC_DIR/$CTID.conf" + [ "$APPLY" = true ] || return 0 + if [ -f "$conf" ] && [ "$(conf_value "$conf" unprivileged)" != 1 ]; then + return 0 + fi + APPLY=false + APPLY_REFUSED=true + echo "${C_WARN}--apply changes nothing on the host or in container $CTID until the container exists and is privileged; only checking${C_RESET}" +} + +main() { + while [ $# -gt 0 ]; do + case "$1" in + --apply) APPLY=true ;; + --vendor) + VENDOR="$2" + shift + ;; + -h | --help) usage ;; + -*) + echo "unknown option: $1" >&2 + usage + ;; + *) CTID="$1" ;; + esac + shift + done + + case "$CTID" in '' | *[!0-9]*) usage ;; esac + case "$VENDOR" in + "" | nvidia | amd) ;; + *) + echo "--vendor must be nvidia or amd" >&2 + exit 2 + ;; + esac + if [ "$(id -u)" -ne 0 ] || ! command -v pct >/dev/null 2>&1; then + echo "run this as root on the Proxmox host" >&2 + exit 2 + fi + + RESTART_NEEDED=false + preflight_apply + check_host_gpu + check_host_kernel + check_container + + echo + if [ "$RESTART_NEEDED" = true ]; then + echo "${C_WARN}Restart the container to apply its new settings: pct reboot $CTID${C_RESET}" + fi + if [ "$FAILS" -gt 0 ]; then + echo "${C_ERR}$FAILS failed, $WARNS warnings${C_RESET}" + [ "$APPLY" = false ] && [ "$APPLY_REFUSED" = false ] && echo "Run with --apply to fix what this script can, then run it again." + exit 1 + fi + echo "${C_OK}host and container $CTID ready ($WARNS warnings)${C_RESET}" + echo "Next, inside the container: gpu-node-check.sh --apply, then join it from the panel's Game Server Nodes page." +} + +# The tests source this file for its functions. Comparing BASH_SOURCE with $0 +# cannot tell that apart from "curl ... | bash", where both differ. +if [ -z "${PROXMOX_GPU_LXC_LIB:-}" ]; then + main "$@" +fi diff --git a/tests/scripts/gpu_node_check_test.sh b/tests/scripts/gpu_node_check_test.sh new file mode 100644 index 0000000..f5c4047 --- /dev/null +++ b/tests/scripts/gpu_node_check_test.sh @@ -0,0 +1,314 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +load() { + # shellcheck source=gpu-node/gpu-node-check.sh + GPU_NODE_CHECK_LIB=1 source "$TEST_ROOT/gpu-node/gpu-node-check.sh" + # The script has its own fail(); the asserts need the one that exits. + SCRIPT_FAIL=$(declare -f fail) + # shellcheck source=tests/scripts/lib.sh + source "$TEST_ROOT/tests/scripts/lib.sh" +} + +# script FUNCTION ARGS... +# Runs one of the script's functions with the script's own fail(), which +# records a [FAIL] line instead of ending the test. +script() { + ( + eval "$SCRIPT_FAIL" + "$@" + ) +} + +# fake_amd_gpu NAME integrated|discrete [DRIVER] +# A PCI device directory as sysfs shows an AMD GPU, with a render node, bound +# to DRIVER (amdgpu unless given). +fake_amd_gpu() { + local dev="$WORK_DIR/pci/$1" driver="${3:-amdgpu}" + mkdir -p "$dev/drm/renderD128" "$dev/hwmon/hwmon0" + ln -s "$WORK_DIR/drivers/$driver" "$dev/driver" + mkdir -p "$WORK_DIR/drivers/$driver" + if [ "$2" = integrated ]; then + echo vddnb >"$dev/hwmon/hwmon0/in1_label" + else + echo vddgfx >"$dev/hwmon/hwmon0/in1_label" + fi + echo "$dev" +} + +test_the_script_runs_nothing_when_sourced() { + local out + out=$(GPU_NODE_CHECK_LIB=1 bash -c "source '$TEST_ROOT/gpu-node/gpu-node-check.sh'; echo sourced" 2>&1) + assert_eq "$out" "sourced" +} + +test_a_bad_vendor_is_a_usage_error() { + load + local status + (parse_args --vendor intel) 2>/dev/null + status=$? + assert_eq "$status" "2" + (parse_args --bogus) 2>/dev/null + status=$? + assert_eq "$status" "2" +} + +test_an_apu_is_recognised_by_its_vddnb_rail() { + load + local apu card + apu=$(fake_amd_gpu 0000:c5:00.0 integrated) + card=$(fake_amd_gpu 0000:03:00.0 discrete) + amd_integrated "$apu" || fail "the APU was not seen as integrated" + if amd_integrated "$card"; then + fail "the discrete card was seen as integrated" + fi +} + +test_an_apu_is_recognised_by_its_gpu_metrics_revision() { + load + local dev="$WORK_DIR/pci/0000:c5:00.0" + mkdir -p "$dev" + printf '\x00\x00\x02\x00' >"$dev/gpu_metrics" + amd_integrated "$dev" || fail "gpu_metrics revision 2 was not seen as integrated" + printf '\x00\x00\x01\x00' >"$dev/gpu_metrics" + if amd_integrated "$dev"; then + fail "gpu_metrics revision 1 was seen as integrated" + fi +} + +test_the_integrated_note_depends_on_a_discrete_card() { + load + local alone beside + alone=$(amd_integrated_note 0000:c5:00.0 false) + beside=$(amd_integrated_note 0000:c5:00.0 true) + assert_contains "$alone" "tick this node" + assert_contains "$beside" "leave it unticked" + assert_contains "$beside" "renders on the discrete card" + assert_not_contains "$beside" "tick this node" +} + +# check_amd reads the kernel log and /proc/cmdline; these keep it on the +# fake devices. +quiet_amd_host() { + IN_LXC=true + # shellcheck disable=SC2317 # called by the function under test + kernel_log() { :; } + # shellcheck disable=SC2317 # called by the function under test + cmdline_has() { return 1; } +} + +test_an_apu_beside_a_discrete_card_stays_unticked() { + load + quiet_amd_host + AMD_GPUS=("$(fake_amd_gpu 0000:c5:00.0 integrated)" "$(fake_amd_gpu 0000:03:00.0 discrete)") + local out + out=$(script check_amd) + assert_contains "$out" "0000:c5:00.0 is integrated, and this node has a discrete AMD GPU on amdgpu: leave it unticked" + assert_not_contains "$out" "tick this node" + assert_not_contains "$out" "runpm" +} + +test_an_apu_on_its_own_is_ticked() { + load + quiet_amd_host + AMD_GPUS=("$(fake_amd_gpu 0000:c5:00.0 integrated)") + local out + out=$(script check_amd) + assert_contains "$out" "0000:c5:00.0 is integrated and the only AMD GPU on amdgpu: tick this node" + assert_not_contains "$out" "runpm" +} + +# A card held for VM passthrough renders nothing on this node, so the APU is +# still the GPU to tick. +test_a_discrete_card_on_vfio_pci_does_not_count() { + load + quiet_amd_host + AMD_GPUS=("$(fake_amd_gpu 0000:c5:00.0 integrated)" "$(fake_amd_gpu 0000:03:00.0 discrete vfio-pci)") + local out + out=$(script check_amd) + assert_contains "$out" "[FAIL] 0000:03:00.0 is bound to 'vfio-pci', not amdgpu" + assert_contains "$out" "0000:c5:00.0 is integrated and the only AMD GPU on amdgpu: tick this node" + assert_not_contains "$out" "leave it unticked" +} + +test_the_nvidia_node_hint_uses_m_for_modeset() { + load + local out + out=$(nvidia_nodes_hint " /dev/nvidia-modeset") + assert_contains "$out" "'nvidia-modprobe -m'" + assert_not_contains "$out" "-c0" + out=$(nvidia_nodes_hint " /dev/nvidiactl /dev/nvidia-uvm /dev/nvidia-modeset /dev/nvidia0") + assert_contains "$out" "'nvidia-modprobe -c0; nvidia-modprobe -c0 -u; nvidia-modprobe -m'" + out=$(nvidia_nodes_hint " /dev/nvidia-uvm") + assert_contains "$out" "'nvidia-modprobe -c0 -u'" + assert_not_contains "$out" "modprobe -m" +} + +test_the_k3s_pin_goes_after_sudo() { + load + assert_eq "$(k3s_join_hint v1.36.5+k3s1)" \ + "curl -fsSL https://get.5stack.gg | sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join --panel-key " +} + +test_without_k3s_the_join_hint_has_the_pin() { + load + # shellcheck disable=SC2317 # called by the function under test + command() { + if [ "$1" = -v ] && [ "$2" = k3s ]; then return 1; fi + builtin command "$@" + } + # shellcheck disable=SC2317 # called by the function under test + df() { printf 'Filesystem 1G-blocks Used Available Use%% Mounted on\n/dev/root 500G 10G 490G 2%% /\n'; } + CPU_SYSFS="$WORK_DIR/cpu" + K3S_VERSION=v1.36.5+k3s1 + local out + out=$(script check_common) + assert_contains "$out" "sudo INSTALL_K3S_VERSION=v1.36.5+k3s1 sh -s -- join" + assert_not_contains "$out" "INSTALL_K3S_VERSION=v1.36.5+k3s1 curl" +} + +# ubuntu-drivers on PATH, writing to stdout as the real one does. +shim_ubuntu_drivers() { + mkdir -p "$WORK_DIR/bin" + cat >"$WORK_DIR/bin/ubuntu-drivers" <"$WORK_DIR/overcommit_memory" + chmod 0444 "$WORK_DIR/overcommit_memory" + PROC_SYS_PROBE="$WORK_DIR/overcommit_memory" + local out + out=$(script check_lxc) + assert_contains "$out" "[FAIL] /proc/sys/vm is read-only" + assert_not_contains "$out" "[PASS] /proc/sys/vm writable" + + chmod 0644 "$WORK_DIR/overcommit_memory" + out=$(script check_lxc) + assert_contains "$out" "[PASS] /proc/sys/vm writable" +} + +# systemctl for the governor unit: a restart either sets every governor or, +# like an already-active unit that enable --now skips, changes nothing. +stub_governor() { + mkdir -p "$WORK_DIR/cpu/cpu0/cpufreq" "$WORK_DIR/cpu/cpu1/cpufreq" "$WORK_DIR/systemd" + echo powersave >"$WORK_DIR/cpu/cpu0/cpufreq/scaling_governor" + echo powersave >"$WORK_DIR/cpu/cpu1/cpufreq/scaling_governor" + CPU_SYSFS="$WORK_DIR/cpu" + SYSTEMD_DIR="$WORK_DIR/systemd" + GOVERNOR_SCRIPT="$WORK_DIR/bin/5stack-cpu-governor.sh" + CPUFREQUTILS_DEFAULT="$WORK_DIR/cpufrequtils" + RESTART_SETS_GOVERNOR="$1" + systemctl() { + echo "systemctl $*" >>"$WORK_DIR/systemctl.log" + if [ "$1" = restart ] && [ "$RESTART_SETS_GOVERNOR" = true ]; then + local g + for g in "$CPU_SYSFS"/cpu*/cpufreq/scaling_governor; do echo performance >"$g"; done + fi + return 0 + } +} + +test_the_governor_unit_is_the_panels_and_restarted() { + load + stub_governor true + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + assert_contains "$(cat "$WORK_DIR/systemd/5stack-cpu-governor.service")" "ExecStart=$GOVERNOR_SCRIPT" + # shellcheck disable=SC2016 # the literal line the unit's script holds + assert_contains "$(cat "$GOVERNOR_SCRIPT")" 'echo performance > "$governor"' + assert_contains "$(cat "$WORK_DIR/systemctl.log")" "systemctl restart 5stack-cpu-governor.service" + assert_not_contains "$(cat "$WORK_DIR/systemctl.log")" "--now" + [ -e "$WORK_DIR/systemd/5stack-cpu-governor.service.5stack-new" ] && fail "the temp unit was left behind" + return 0 +} + +# The panel's tuning points cpufrequtils's boot script at performance too. +test_the_governor_fix_sets_cpufrequtils_to_performance() { + load + stub_governor true + printf 'ENABLE="true"\nGOVERNOR="ondemand"\nMAX_SPEED="0"\n' >"$CPUFREQUTILS_DEFAULT" + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + assert_eq "$(cat "$CPUFREQUTILS_DEFAULT")" 'ENABLE="true" +MAX_SPEED="0" +GOVERNOR="performance"' + [ -e "$CPUFREQUTILS_DEFAULT.5stack-new" ] && fail "the temp file was left behind" + return 0 +} + +test_the_governor_fix_adds_no_cpufrequtils_settings() { + load + stub_governor true + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + [ -e "$CPUFREQUTILS_DEFAULT" ] && fail "cpufrequtils settings were created" + return 0 +} + +test_the_governor_fix_fails_when_the_governor_did_not_change() { + load + stub_governor false + local out + out=$(script apply_step "CPU governor set to performance" install_governor_unit) + assert_contains "$out" "[FAIL] could not apply: CPU governor set to performance" + assert_contains "$out" "is still powersave" + assert_not_contains "$out" "[FIX ]" +} + +test_k3s_is_found_in_usr_local_bin_when_path_leaves_it_out() { + ( + PATH=/usr/bin:/bin + load + assert_contains ":$PATH:" ":/usr/local/bin:" + ) +} + +run_tests diff --git a/tests/scripts/proxmox_gpu_lxc_test.sh b/tests/scripts/proxmox_gpu_lxc_test.sh new file mode 100644 index 0000000..82e3b54 --- /dev/null +++ b/tests/scripts/proxmox_gpu_lxc_test.sh @@ -0,0 +1,589 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +load() { + # shellcheck source=gpu-node/proxmox-gpu-lxc.sh + PROXMOX_GPU_LXC_LIB=1 source "$TEST_ROOT/gpu-node/proxmox-gpu-lxc.sh" + # The script has its own fail(); the asserts need the one that exits. + SCRIPT_FAIL=$(declare -f fail) + # shellcheck source=tests/scripts/lib.sh + source "$TEST_ROOT/tests/scripts/lib.sh" +} + +# script FUNCTION ARGS... +# Runs one of the script's functions with the script's own fail(), which +# records a [FAIL] line instead of ending the test. +script() { + ( + eval "$SCRIPT_FAIL" + "$@" + ) +} + +write_conf() { + cat >"$WORK_DIR/100.conf" <<'CONF' +arch: amd64 +cores: 8 +features: keyctl=1 +hostname: gpu-node +rootfs: local-lvm:vm-100-disk-0,size=128G +unprivileged: 0 + +[before-k3s] +arch: amd64 +rootfs: local-lvm:vm-100-disk-0,size=32G +unprivileged: 1 +CONF +} + +test_conf_value_reads_the_container_not_its_snapshots() { + load + write_conf + assert_eq "$(conf_value "$WORK_DIR/100.conf" rootfs)" "local-lvm:vm-100-disk-0,size=128G" + assert_eq "$(conf_value "$WORK_DIR/100.conf" unprivileged)" "0" + assert_eq "$(conf_value "$WORK_DIR/100.conf" nesting)" "" +} + +test_size_gb_reads_proxmox_disk_sizes() { + load + assert_eq "$(size_gb 128G)" "128" + assert_eq "$(size_gb 2T)" "2048" + assert_eq "$(size_gb 131072M)" "128" + assert_eq "$(size_gb 512M)" "0" + assert_eq "$(size_gb 134217728K)" "128" + assert_eq "$(size_gb 137438953472)" "128" + assert_eq "$(size_gb 8.5G)" "8" + assert_eq "$(size_gb "")" "" + assert_eq "$(size_gb abc)" "" + assert_eq "$(size_gb 1.2.3G)" "" +} + +test_nesting_is_added_without_dropping_other_features() { + load + assert_eq "$(features_with_nesting "keyctl=1,fuse=1")" "keyctl=1,fuse=1,nesting=1" + assert_eq "$(features_with_nesting "nesting=0,keyctl=1")" "keyctl=1,nesting=1" + assert_eq "$(features_with_nesting "")" "nesting=1" +} + +test_lxc_lines_go_before_the_first_snapshot() { + load + write_conf + add_lxc_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined" "lxc.cap.drop: " || fail "add_lxc_lines failed" + assert_eq "$(sed -n '6,10p' "$WORK_DIR/100.conf" | sed 's/ *$//')" "unprivileged: 0 +lxc.apparmor.profile: unconfined +lxc.cap.drop: + +[before-k3s]" + assert_eq "$(tail -1 "$WORK_DIR/100.conf")" "unprivileged: 1" +} + +# Proxmox drops the trailing space of an empty value when it rewrites the file. +test_lxc_lines_are_added_once() { + load + write_conf + add_lxc_lines "$WORK_DIR/100.conf" "lxc.cap.drop: " + sed -i 's/^lxc.cap.drop: $/lxc.cap.drop:/' "$WORK_DIR/100.conf" + add_lxc_lines "$WORK_DIR/100.conf" "lxc.cap.drop: " || fail "add_lxc_lines failed" + assert_eq "$(grep -c '^lxc.cap.drop' "$WORK_DIR/100.conf")" "1" +} + +test_a_line_in_a_snapshot_does_not_count() { + load + write_conf + echo "lxc.apparmor.profile: unconfined" >>"$WORK_DIR/100.conf" + assert_eq "$(missing_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined")" "lxc.apparmor.profile: unconfined" +} + +test_a_config_without_snapshots_gets_the_lines_at_the_end() { + load + printf 'arch: amd64\nunprivileged: 0\n' >"$WORK_DIR/101.conf" + add_lxc_lines "$WORK_DIR/101.conf" "lxc.mount.auto: proc:rw sys:rw" + assert_eq "$(cat "$WORK_DIR/101.conf")" "arch: amd64 +unprivileged: 0 +lxc.mount.auto: proc:rw sys:rw" +} + +test_amd_lines_bind_dri_and_kfd() { + load + assert_contains "$(lxc_lines amd)" "lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir" + assert_contains "$(lxc_lines amd)" "lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file" + assert_not_contains "$(lxc_lines amd)" "nvidia" +} + +test_nvidia_lines_bind_the_nvidia_devices() { + load + assert_contains "$(lxc_lines nvidia)" "lxc.mount.entry: /dev/nvidiactl dev/nvidiactl none bind,optional,create=file" + assert_contains "$(lxc_lines nvidia)" "lxc.mount.entry: /dev/nvidia-uvm dev/nvidia-uvm none bind,optional,create=file" + assert_contains "$(lxc_lines nvidia)" "/dev/nvidia0 dev/nvidia0" + assert_not_contains "$(lxc_lines nvidia)" "/dev/kfd" +} + +test_proxmox_device_passthrough_counts_for_the_amd_binds() { + load + write_conf + sed -i '/^unprivileged: 0$/a dev0: /dev/kfd,gid=993\ndev1: path=/dev/dri/renderD128,gid=993\ndev2: /dev/dri/card1,gid=44' "$WORK_DIR/100.conf" + local lines + mapfile -t lines < <(lxc_lines amd) + assert_not_contains "$(missing_lines "$WORK_DIR/100.conf" "${lines[@]}")" "/dev/dri" + assert_not_contains "$(missing_lines "$WORK_DIR/100.conf" "${lines[@]}")" "/dev/kfd" + add_lxc_lines "$WORK_DIR/100.conf" "${lines[@]}" + assert_eq "$(grep -c '^lxc.mount.entry: /dev/kfd' "$WORK_DIR/100.conf")" "0" +} + +test_a_card_node_alone_does_not_count_for_dri() { + load + write_conf + sed -i '/^unprivileged: 0$/a dev0: /dev/dri/card1,gid=44' "$WORK_DIR/100.conf" + assert_eq "$(missing_lines "$WORK_DIR/100.conf" "lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir")" "lxc.mount.entry: /dev/dri dev/dri none bind,optional,create=dir" +} + +test_device_passthrough_in_a_snapshot_does_not_count() { + load + write_conf + echo "dev0: /dev/kfd,gid=993" >>"$WORK_DIR/100.conf" + assert_eq "$(missing_lines "$WORK_DIR/100.conf" "lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file")" "lxc.mount.entry: /dev/kfd dev/kfd none bind,optional,create=file" +} + +# An awk that fails only for the rewrite in add_lxc_lines, the one call that +# passes ADD_LINES. MODE is fail (writes half the file, exits 1, as on a full +# disk), empty (writes nothing, exits 0) or short (writes 2 lines, exits 0). +shim_awk() { + local real + real=$(command -v awk) + mkdir -p "$WORK_DIR/bin" + cat >"$WORK_DIR/bin/awk" <"$WORK_DIR/out"; then + fail "add_lxc_lines succeeded although awk failed" + fi + assert_conf_untouched + assert_contains "$(cat "$WORK_DIR/out")" "is unchanged" +} + +test_an_empty_rewrite_is_refused() { + load + write_conf + cp "$WORK_DIR/100.conf" "$WORK_DIR/100.conf.orig" + shim_awk empty + if add_lxc_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined" >/dev/null; then + fail "add_lxc_lines took an empty result" + fi + assert_conf_untouched +} + +test_a_cut_off_rewrite_is_refused() { + load + write_conf + cp "$WORK_DIR/100.conf" "$WORK_DIR/100.conf.orig" + shim_awk short + if add_lxc_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined" >/dev/null; then + fail "add_lxc_lines took a cut-off result" + fi + assert_conf_untouched +} + +# A directory that takes no new file, like a full disk or /etc/pve without +# quorum. Root ignores the permission, so this only runs unprivileged. +test_a_directory_that_takes_no_temp_file_leaves_the_config_as_it_was() { + load + [ "$(id -u)" -eq 0 ] && return 0 + mkdir "$WORK_DIR/pve" + write_conf + mv "$WORK_DIR/100.conf" "$WORK_DIR/pve/100.conf" + cp "$WORK_DIR/pve/100.conf" "$WORK_DIR/100.conf.orig" + chmod 0555 "$WORK_DIR/pve" + if add_lxc_lines "$WORK_DIR/pve/100.conf" "lxc.apparmor.profile: unconfined" >/dev/null 2>&1; then + chmod 0755 "$WORK_DIR/pve" + fail "add_lxc_lines succeeded in a read-only directory" + fi + chmod 0755 "$WORK_DIR/pve" + assert_eq "$(cat "$WORK_DIR/pve/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" +} + +test_the_rewrite_replaces_the_file_by_rename() { + load + write_conf + local inode + inode=$(stat -c %i "$WORK_DIR/100.conf") + add_lxc_lines "$WORK_DIR/100.conf" "lxc.apparmor.profile: unconfined" || fail "add_lxc_lines failed" + [ "$(stat -c %i "$WORK_DIR/100.conf")" != "$inode" ] || fail "the config was rewritten in place" + assert_eq "$(find "$WORK_DIR" -maxdepth 1 -name '100.conf.tmp.*' | wc -l)" "0" +} + +# pct as the container tests see it: STATUS is what pct status says, FREE the +# free GB df reports inside, CS2 whether CS2 is installed there (true or +# false), and every call is logged. PCT_SET_FAILS=true makes pct set fail. +stub_pct() { + PCT_STATUS="$1" + PCT_FREE="${2:-}" + PCT_CS2="${3:-false}" + PCT_SET_FAILS=false + # shellcheck disable=SC2317 # called by the function under test + pct() { + echo "pct $*" >>"$WORK_DIR/pct.log" + case "$1" in + status) echo "status: $PCT_STATUS" ;; + set) [ "$PCT_SET_FAILS" != true ] ;; + exec) + case "$4" in + test) [ "$PCT_CS2" = true ] ;; + *) + printf 'Filesystem 1G-blocks Used Available Use%% Mounted on\n' + printf '/dev/loop0 200G 10G %sG 10%% /\n' "$PCT_FREE" + ;; + esac + ;; + esac + } + touch "$WORK_DIR/pct.log" +} + +test_a_running_container_is_judged_by_its_free_space() { + load + CTID=100 + stub_pct running 130 + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=128G")" "[PASS] root disk 128G, 130 GB free" + stub_pct running 100 + local out + out=$(script check_disk "local-lvm:vm-100-disk-0,size=128G") + assert_contains "$out" "[FAIL] root disk 128G has 100 GB free; before CS2 is installed the node setup needs 120 GB free" + assert_contains "$out" "pct resize 100 rootfs +40G" + assert_contains "$out" '"Required disk space - fresh install" (120 GB by default) and "Required disk space - existing install" (60 GB)' + assert_contains "$(cat "$WORK_DIR/pct.log")" "pct exec 100 -- test -d /opt/5stack/serverfiles/game/csgo" +} + +# With CS2 installed the node setup takes the existing install minimum. +test_a_container_with_cs2_installed_needs_60_gb_free() { + load + CTID=100 + stub_pct running 100 true + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=128G")" "[PASS] root disk 128G, 100 GB free (with CS2 already installed the node setup needs 60 GB)" + stub_pct running 50 true + local out + out=$(script check_disk "local-lvm:vm-100-disk-0,size=128G") + assert_contains "$out" "[FAIL] root disk 128G has 50 GB free; with CS2 already installed the node setup needs 60 GB free" + assert_contains "$out" "pct resize 100 rootfs +30G" + assert_contains "$out" '"Required disk space - existing install" (60 GB)' +} + +test_a_stopped_container_is_judged_by_its_size() { + load + CTID=100 + stub_pct stopped + local out + out=$(script check_disk "local-lvm:vm-100-disk-0,size=100G") + assert_contains "$out" "[FAIL] root disk is 100G" + assert_contains "$out" "If CS2 is already installed in it, start the container and run this again: 60 GB free is then enough." + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=120G")" "[WARN] root disk is 120G" + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=131072M")" "[WARN] root disk is 131072M" + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0,size=150G")" "[PASS] root disk 150G" + assert_contains "$(script check_disk "local-lvm:vm-100-disk-0")" "[WARN] could not read the root disk size" +} + +write_container() { + write_conf + mkdir -p "$WORK_DIR/lxc" "$WORK_DIR/backup" + sed "s/^unprivileged: 0$/unprivileged: $1/" "$WORK_DIR/100.conf" >"$WORK_DIR/lxc/100.conf" + cp "$WORK_DIR/lxc/100.conf" "$WORK_DIR/100.conf.orig" + PVE_LXC_DIR="$WORK_DIR/lxc" + BACKUP_DIR="$WORK_DIR/backup" + CTID=100 + VENDOR=amd + APPLY=true + # The lock lives under /run/lock; the tests take none. + # shellcheck disable=SC2317 # called by the function under test + locked_add_lxc_lines() { add_lxc_lines "$@"; } +} + +test_apply_stops_at_an_unprivileged_container() { + load + write_container 1 + stub_pct stopped + local out + out=$(script check_container) + assert_contains "$out" "[FAIL] the container is unprivileged" + assert_contains "$out" "--apply changes nothing in this container until it is privileged" + assert_eq "$(cat "$WORK_DIR/lxc/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" + assert_not_contains "$(cat "$WORK_DIR/pct.log")" "pct set" + assert_eq "$(find "$WORK_DIR/backup" -type f | wc -l)" "0" +} + +# A refused --apply runs the checks in report mode, so the container's +# hints must not offer --apply for a container it will refuse again. +test_refused_apply_offers_no_apply_fixes() { + load + write_container 1 + stub_pct stopped + APPLY=false + APPLY_REFUSED=true + local out + out=$(script check_container) + assert_contains "$out" "[FAIL] the container is unprivileged" + assert_not_contains "$out" "(--apply" +} + +# The host steps run before the container's, so --apply has to stop before +# them, not only in check_container. +test_apply_turns_off_before_the_host_steps_for_an_unprivileged_container() { + load + write_container 1 + local out + out=$(preflight_apply) + assert_contains "$out" "--apply changes nothing on the host or in container 100 until the container exists and is privileged" + preflight_apply >/dev/null + assert_eq "$APPLY" "false" + assert_eq "$APPLY_REFUSED" "true" + + MODULES_FILE="$WORK_DIR/5stack-k3s.conf" + # shellcheck disable=SC2317 # called by the function under test + modprobe() { echo "modprobe $*" >>"$WORK_DIR/modprobe.log"; } + stub_governor true + out=$(script check_host_kernel) + assert_not_contains "$out" "[FIX ]" + [ -e "$MODULES_FILE" ] && fail "the modules file was written" + [ -e "$WORK_DIR/modprobe.log" ] && fail "a module was loaded" + [ -e "$WORK_DIR/systemd/5stack-cpu-governor.service" ] && fail "the governor unit was installed" + return 0 +} + +# main as root on a Proxmox host, with an unprivileged container: the refusal +# comes before the first host step, and nothing on the host changes. +test_main_refuses_apply_before_the_host_steps() { + load + write_container 1 + stub_pct stopped + stub_governor true + MODULES_FILE="$WORK_DIR/5stack-k3s.conf" + # shellcheck disable=SC2317 # called by the function under test + id() { echo 0; } + # shellcheck disable=SC2317 # called by the function under test + modprobe() { echo "modprobe $*" >>"$WORK_DIR/modprobe.log"; } + local out + out=$( (main 100 --apply --vendor amd) 2>&1) + assert_contains "$out" "--apply changes nothing on the host or in container 100" + assert_contains "$(echo "$out" | sed -n '1,/==> Host GPU/p')" "--apply changes nothing on the host" + assert_not_contains "$out" "[FIX ]" + assert_not_contains "$out" "Run with --apply" + [ -e "$MODULES_FILE" ] && fail "the modules file was written" + [ -e "$WORK_DIR/modprobe.log" ] && fail "a module was loaded" + [ -e "$WORK_DIR/systemd/5stack-cpu-governor.service" ] && fail "the governor unit was installed" + assert_eq "$(cat "$WORK_DIR/lxc/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" +} + +test_apply_goes_ahead_for_a_privileged_container() { + load + write_container 0 + preflight_apply >"$WORK_DIR/out" + assert_eq "$(cat "$WORK_DIR/out")" "" + assert_eq "$APPLY" "true" + CTID=101 + preflight_apply >/dev/null + assert_eq "$APPLY" "false" +} + +test_a_failed_change_asks_for_no_restart() { + load + write_container 0 + stub_pct stopped + PCT_SET_FAILS=true + # shellcheck disable=SC2317 # called by the function under test + locked_add_lxc_lines() { return 1; } + local out + out=$( + eval "$SCRIPT_FAIL" + RESTART_NEEDED=false + check_container + echo "RESTART_NEEDED=$RESTART_NEEDED" + ) + assert_contains "$out" "[FAIL] could not apply: enabled nesting" + assert_contains "$out" "[FAIL] could not apply: added 8 LXC lines" + assert_contains "$out" "RESTART_NEEDED=false" +} + +test_apply_backs_up_then_adds_the_lines_and_warns_about_proc_rw() { + load + write_container 0 + stub_pct stopped + local out + out=$(script check_container) + assert_contains "$out" "[INFO] backed up the container config to $WORK_DIR/backup/5stack-lxc-100.conf." + assert_contains "$out" "[WARN] lxc.mount.auto: proc:rw lets the kubelet set vm.overcommit_memory=1, kernel.panic=10 and kernel.panic_on_oops=1" + assert_contains "$out" "reboots 10 seconds after a kernel oops" + assert_contains "$out" "[FIX ] added 8 LXC lines for k3s and the amd GPU" + assert_contains "$(cat "$WORK_DIR/pct.log")" "pct set 100 --features keyctl=1,nesting=1" + assert_eq "$(cat "$WORK_DIR/backup"/5stack-lxc-100.conf.*)" "$(cat "$WORK_DIR/100.conf.orig")" + local lines + mapfile -t lines < <(lxc_lines amd) + assert_eq "$(missing_lines "$WORK_DIR/lxc/100.conf" "${lines[@]}")" "" +} + +test_apply_changes_nothing_when_the_backup_fails() { + load + write_container 0 + stub_pct stopped + BACKUP_DIR="$WORK_DIR/no-such-dir" + local out + out=$(script check_container) + assert_contains "$out" "[FAIL] could not back up $WORK_DIR/lxc/100.conf" + assert_not_contains "$out" "[FIX ]" + assert_not_contains "$(cat "$WORK_DIR/pct.log")" "pct set" + assert_eq "$(cat "$WORK_DIR/lxc/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" +} + +test_report_mode_lists_the_lines_and_warns_about_proc_rw() { + load + write_container 0 + stub_pct stopped + APPLY=false + local out + out=$(script check_container) + assert_contains "$out" "[FAIL] 8 LXC lines for k3s and the amd GPU are missing" + assert_contains "$out" "[WARN] lxc.mount.auto: proc:rw lets the kubelet set" + assert_eq "$(cat "$WORK_DIR/lxc/100.conf")" "$(cat "$WORK_DIR/100.conf.orig")" +} + +# nvidia_host DRIVER_VERSION +# check_host_nvidia's view of a host with the driver loaded, no device nodes +# and no nvidia-modprobe. +nvidia_host() { + echo "NVRM version: NVIDIA UNIX Open Kernel Module for x86_64 $1 Release Build" >"$WORK_DIR/nvidia-version" + NVIDIA_PROC_VERSION="$WORK_DIR/nvidia-version" + NVIDIA_NODES="$WORK_DIR/dev/nvidia0 $WORK_DIR/dev/nvidiactl" + SYSTEMD_DIR="$WORK_DIR/systemd" + NVIDIA_UNIT="$SYSTEMD_DIR/5stack-nvidia-devices.service" + # shellcheck disable=SC2317 # called by the function under test + command() { + if [ "$1" = -v ] && [ "$2" = nvidia-modprobe ]; then return 1; fi + builtin command "$@" + } +} + +test_apply_without_nvidia_modprobe_says_it_cannot_create_the_nodes() { + load + nvidia_host 595.99.02 + APPLY=true + local out + out=$(script check_host_nvidia) + assert_contains "$out" "[PASS] NVIDIA driver 595.99.02" + assert_contains "$out" "[FAIL] missing device nodes: $WORK_DIR/dev/nvidia0 $WORK_DIR/dev/nvidiactl, and --apply cannot create them without nvidia-modprobe" + assert_contains "$out" "NVIDIA's .run installer installs nvidia-modprobe" + assert_not_contains "$out" "--apply creates them" + [ -e "$NVIDIA_UNIT" ] && fail "the unit was written" + return 0 +} + +test_report_mode_without_device_nodes_offers_apply() { + load + nvidia_host 595.99.02 + APPLY=false + assert_contains "$(script check_host_nvidia)" "(--apply creates them at every boot with nvidia-modprobe)" +} + +test_nomodeset_only_fails_an_amd_host() { + load + echo "BOOT_IMAGE=/vmlinuz root=/dev/sda1 nomodeset quiet" >"$WORK_DIR/cmdline" + PROC_CMDLINE="$WORK_DIR/cmdline" + assert_contains "$(script check_host_amd)" "[FAIL] the kernel command line has nomodeset" + VENDOR=nvidia + assert_not_contains "$(script check_host_gpu)" "nomodeset" +} + +# systemctl for the governor unit: a restart either sets every governor or, +# like an already-active unit that enable --now skips, changes nothing. +stub_governor() { + mkdir -p "$WORK_DIR/cpu/cpu0/cpufreq" "$WORK_DIR/cpu/cpu1/cpufreq" "$WORK_DIR/systemd" + echo powersave >"$WORK_DIR/cpu/cpu0/cpufreq/scaling_governor" + echo powersave >"$WORK_DIR/cpu/cpu1/cpufreq/scaling_governor" + CPU_SYSFS="$WORK_DIR/cpu" + SYSTEMD_DIR="$WORK_DIR/systemd" + GOVERNOR_SCRIPT="$WORK_DIR/bin/5stack-cpu-governor.sh" + CPUFREQUTILS_DEFAULT="$WORK_DIR/cpufrequtils" + RESTART_SETS_GOVERNOR="$1" + systemctl() { + echo "systemctl $*" >>"$WORK_DIR/systemctl.log" + if [ "$1" = restart ] && [ "$RESTART_SETS_GOVERNOR" = true ]; then + local g + for g in "$CPU_SYSFS"/cpu*/cpufreq/scaling_governor; do echo performance >"$g"; done + fi + return 0 + } +} + +test_the_governor_unit_is_the_panels_and_restarted() { + load + stub_governor true + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + assert_contains "$(cat "$WORK_DIR/systemd/5stack-cpu-governor.service")" "ExecStart=$GOVERNOR_SCRIPT" + # shellcheck disable=SC2016 # the literal line the unit's script holds + assert_contains "$(cat "$GOVERNOR_SCRIPT")" 'echo performance > "$governor"' + assert_contains "$(cat "$WORK_DIR/systemctl.log")" "systemctl restart 5stack-cpu-governor.service" + assert_not_contains "$(cat "$WORK_DIR/systemctl.log")" "--now" +} + +test_the_governor_check_reads_the_cpu_sysfs_it_is_given() { + load + stub_governor true + MODULES_FILE="$WORK_DIR/5stack-k3s.conf" + APPLY=false + assert_contains "$(script check_host_kernel)" "[WARN] CPU governor is 'powersave'" + echo performance >"$WORK_DIR/cpu/cpu0/cpufreq/scaling_governor" + assert_contains "$(script check_host_kernel)" "[PASS] CPU governor performance" +} + +# The panel's tuning points cpufrequtils's boot script at performance too. +test_the_governor_fix_sets_cpufrequtils_to_performance() { + load + stub_governor true + printf 'ENABLE="true"\nGOVERNOR="ondemand"\nMAX_SPEED="0"\n' >"$CPUFREQUTILS_DEFAULT" + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + assert_eq "$(cat "$CPUFREQUTILS_DEFAULT")" 'ENABLE="true" +MAX_SPEED="0" +GOVERNOR="performance"' + [ -e "$CPUFREQUTILS_DEFAULT.5stack-new" ] && fail "the temp file was left behind" + return 0 +} + +test_the_governor_fix_adds_no_cpufrequtils_settings() { + load + stub_governor true + install_governor_unit >"$WORK_DIR/out" || fail "install_governor_unit failed: $(cat "$WORK_DIR/out")" + [ -e "$CPUFREQUTILS_DEFAULT" ] && fail "cpufrequtils settings were created" + return 0 +} + +test_the_governor_fix_fails_when_the_governor_did_not_change() { + load + stub_governor false + local out + out=$(script apply_step "CPU governor set to performance" install_governor_unit) + assert_contains "$out" "[FAIL] could not apply: CPU governor set to performance" + assert_contains "$out" "is still powersave" + assert_not_contains "$out" "[FIX ]" +} + +run_tests