diff --git a/.github/scripts/image_updates.sh b/.github/scripts/image_updates.sh new file mode 100755 index 0000000..32627f6 --- /dev/null +++ b/.github/scripts/image_updates.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# Finds newer releases of the third-party images the panel pins, for +# .github/workflows/image-updates.yml. Run it from inside the repository. +# +# image_updates.sh plan one line per update: +# image_updates.sh apply OLD NEW rewrites every pin of OLD to NEW +# image_updates.sh title the commit title for the plan lines on stdin +# +# A pin is a tracked `image: ` line or a shell `_IMAGE=` +# assignment whose tag is a version. An update keeps the tag's shape: its `v`, +# its number of version parts and the suffix after the version (`-alpine`, +# `-pg18`, `-k3s1`) stay the same, so a tag only moves to another release of +# the same build, and prereleases (`-rc1`, `.rc22`) never match. A pinned +# digest is re-pinned to the new tag's. An update is major when the first +# version part changes, or below 1.0 the second. +set -euo pipefail + +# Pins that only move by hand. +IGNORED_PATHS=( + # The upstream cert-manager release file, held on 1.17: from 1.18 the + # HTTP-01 solver's Ingress is rejected by the older ingress-nginx some + # panels still run, and renewals quietly stop. + ':!overlays/cert-manager-crds/cert-manager.yaml' + # The system-upgrade-controller's release assets, which + # tests/scripts/system_upgrade_test.sh checks against their sha256. + ':!base/system-upgrade/' + # The old releases the TimescaleDB upgrade tests start from. + ':!tests/timescaledb/' + # This script's own test fixtures. + ':!tests/scripts/image_updates_test.sh' + # Workflows, which the app's token can't push. + ':!.github/' +) + +die() { + echo "image_updates.sh: $*" >&2 + exit 1 +} + +cd "$(git rev-parse --show-toplevel)" + +# Every distinct ref pinned in the tracked files. +pins() { + git grep -I -h -o -E \ + -e 'image:[[:space:]]*"?[^[:space:]"]+' \ + -e '^[[:space:]]*[A-Z0-9_]+_IMAGE=[^[:space:]"]+' \ + -- "${IGNORED_PATHS[@]}" | + sed -E 's/^.*(image:[[:space:]]*"?|_IMAGE=)//' | + sort -u +} + +# The newest tags in the list on stdin that keep the shape of tag $1, as +# "minor " and "major " lines. +newer_tags() { + awk -v current="$1" ' + # Splits a tag into its v, version and suffix. 0 for a tag that is not + # a plain version, like latest, 19beta1-alpine or 31.0.rc22-arm64. + function parse(tag, out) { + out["v"] = "" + if (tag ~ /^v[0-9]/) { + out["v"] = "v" + tag = substr(tag, 2) + } + if (!match(tag, /^[0-9]+(\.[0-9]+)*/)) return 0 + out["version"] = substr(tag, 1, RLENGTH) + out["suffix"] = substr(tag, RLENGTH + 1) + return out["suffix"] == "" || out["suffix"] ~ /^-/ + } + # 1 when version a is newer than version b, which has as many parts. + function newer(a, b, x, y, n, i) { + n = split(a, x, ".") + split(b, y, ".") + for (i = 1; i <= n; i++) { + if (x[i] + 0 != y[i] + 0) return x[i] + 0 > y[i] + 0 + } + return 0 + } + BEGIN { + if (!parse(current, cur)) exit + parts = split(cur["version"], c, ".") + } + { + if (!parse($0, cand)) next + if (cand["v"] != cur["v"] || cand["suffix"] != cur["suffix"]) next + if (split(cand["version"], n, ".") != parts) next + if (!newer(cand["version"], cur["version"])) next + kind = "minor" + if (n[1] + 0 != c[1] + 0 || (c[1] + 0 == 0 && parts > 1 && n[2] + 0 != c[2] + 0)) kind = "major" + if (!(kind in best) || newer(cand["version"], best[kind])) { + best[kind] = cand["version"] + best_tag[kind] = $0 + } + } + END { + if ("minor" in best_tag) print "minor", best_tag["minor"] + if ("major" in best_tag) print "major", best_tag["major"] + } + ' +} + +plan() { + local ref name tag digest kind new_tag new_digest tags + pins | while IFS= read -r ref; do + digest="" + if [[ $ref == *@* ]]; then + digest="${ref#*@}" + fi + name="${ref%%@*}" + tag="${name##*:}" + # name@digest, a registry port with no tag after it, or a tag like + # latest that no release moves + if [[ $name != *:* || ! $tag =~ ^v?[0-9]+(\.[0-9]+)*(-.*)?$ ]]; then + continue + fi + name="${name%:*}" + if ! tags="$(crane ls "$name" 2>&1 &2 + continue + fi + newer_tags "$tag" <<<"$tags" | while read -r kind new_tag; do + if [ -z "$digest" ]; then + echo "$kind $ref $name:$new_tag" + elif new_digest="$(crane digest "$name:$new_tag" 2>&1 &2 + fi + done + done +} + +apply() { + local old="$1" new="$2" files=() before + while IFS= read -r file; do + files+=("$file") + done < <(git grep -I -l -F -e "$old" -- "${IGNORED_PATHS[@]}") + [ ${#files[@]} -gt 0 ] || die "$old is not pinned anywhere" + before="$(cat -- "${files[@]}" | git hash-object --stdin)" + OLD="$old" NEW="$new" perl -pi -e \ + 's/(image:[ \t]*"?|_IMAGE=)\Q$ENV{OLD}\E(?=["\s]|$)/$1$ENV{NEW}/g' -- "${files[@]}" + if [ "$(cat -- "${files[@]}" | git hash-object --stdin)" = "$before" ]; then + die "$old is mentioned, but not pinned, in ${files[*]}" + fi +} + +# "redis 8.12.0-alpine" for docker.io/library/redis:8.12.0-alpine@sha256:... +label() { + local ref="${1%%@*}" name + name="${ref%:*}" + case "${name%%/*}" in + *.* | *:* | localhost) [[ $name == */* ]] && name="${name#*/}" ;; + esac + name="${name#library/}" + echo "$name ${ref##*:}" +} + +title() { + local kind old new labels=() list + while read -r kind old new; do + [ -n "$new" ] || continue + labels+=("$(label "$new")") + done + [ ${#labels[@]} -gt 0 ] || die "no updates on stdin" + list="${labels[0]}" + if [ ${#labels[@]} -gt 1 ]; then + list="$(printf '%s, ' "${labels[@]:0:${#labels[@]}-1}")" + list="${list%, } and ${labels[${#labels[@]}-1]}" + fi + if [ ${#list} -gt 200 ]; then + list="${#labels[@]} images" + fi + echo "chore: $list" +} + +case "${1:-}" in + plan) plan ;; + apply) + [ $# -eq 3 ] || die "usage: image_updates.sh apply OLD NEW" + apply "$2" "$3" + ;; + title) title ;; + *) die "usage: image_updates.sh plan | apply OLD NEW | title" ;; +esac diff --git a/.github/workflows/gateway.yml b/.github/workflows/gateway.yml index 7cf8de2..d621532 100644 --- a/.github/workflows/gateway.yml +++ b/.github/workflows/gateway.yml @@ -64,6 +64,18 @@ jobs: - uses: actions/checkout@v4 - run: bash tests/gateway/upgrade_test.sh + upgrade-from-servicelb: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash tests/gateway/servicelb_upgrade_test.sh + + upgrade-k3s-to-its-own-gateway-api: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash tests/gateway/k3s_upgrade_test.sh + cert-manager: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/image-updates.yml b/.github/workflows/image-updates.yml new file mode 100644 index 0000000..d164791 --- /dev/null +++ b/.github/workflows/image-updates.yml @@ -0,0 +1,203 @@ +name: Image updates + +# Moves the third-party images the panel pins to their newest releases; see +# .github/scripts/image_updates.sh for what counts as one and which pins only +# move by hand. Minor and patch releases go into one PR that merges itself +# once its checks pass. Each major release gets a PR of its own, assigned to +# REVIEWERS to check that the panel still works on it. Closing a major PR skips +# that release; a newer major release opens a new PR. +on: + schedule: + - cron: "0 13 * * 1" + workflow_dispatch: + +permissions: {} + +concurrency: + group: image-updates + +env: + REVIEWERS: lukepolo + CRANE_VERSION: v0.22.1 + CRANE_SHA256: 0ab7a1d6932a213aed964ce97666c3077fe691c8606413674a8b3e0b9ec4cda0 + GH_REPO: ${{ github.repository }} + +defaults: + run: + shell: bash + +jobs: + update: + if: github.repository == '5stackgg/5stack-panel' + runs-on: ubuntu-latest + timeout-minutes: 45 + # GITHUB_TOKEN only reads checks and keeps the labels. Branches and PRs + # come from the app's token, because ones from GITHUB_TOKEN start no checks + # and no release. + permissions: + contents: read + checks: read + statuses: read + pull-requests: read + issues: write + steps: + - name: Get a token from the 5Stack Ops app + id: app + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.OPS_APP_CLIENT_ID }} + private-key: ${{ secrets.OPS_APP_KEY }} + permission-contents: write + permission-pull-requests: write + + - uses: actions/checkout@v4 + with: + token: ${{ steps.app.outputs.token }} + fetch-depth: 0 + + - name: Install crane + run: | + curl -fsSLo "$RUNNER_TEMP/crane.tar.gz" \ + "https://github.com/google/go-containerregistry/releases/download/$CRANE_VERSION/go-containerregistry_Linux_x86_64.tar.gz" + echo "$CRANE_SHA256 $RUNNER_TEMP/crane.tar.gz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/crane.tar.gz" -C "$RUNNER_TEMP" crane + echo "$RUNNER_TEMP" >>"$GITHUB_PATH" + + # The steps below switch to branches made from main, so they run a copy + # of this commit's script rather than whichever one the branch has. + - name: Keep this run's script + run: | + cp .github/scripts/image_updates.sh "$RUNNER_TEMP/image_updates.sh" + echo "SCRIPT=$RUNNER_TEMP/image_updates.sh" >>"$GITHUB_ENV" + + - name: Commit as the app + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + APP_SLUG: ${{ steps.app.outputs.app-slug }} + run: | + id="$(gh api "/users/$APP_SLUG%5Bbot%5D" --jq .id)" + git config user.name "${APP_SLUG}[bot]" + git config user.email "$id+${APP_SLUG}[bot]@users.noreply.github.com" + + - name: Keep the labels + env: + GH_TOKEN: ${{ github.token }} + run: | + gh label create image-update --force --color 0e8a16 \ + --description "Minor and patch image releases, merged once the checks pass" + gh label create image-update:major --force --color d93f0b \ + --description "A major image release to check before merging" + + - name: Merge the minor and patch releases + id: minor + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + CHECKS_TOKEN: ${{ github.token }} + run: | + branch=image-updates/minor + "$SCRIPT" plan >"$RUNNER_TEMP/plan" + cat "$RUNNER_TEMP/plan" + awk '$1 == "minor"' "$RUNNER_TEMP/plan" >"$RUNNER_TEMP/minor" + pr="$(gh pr list --head "$branch" --state open --json number --jq '.[0].number // empty')" + if [ ! -s "$RUNNER_TEMP/minor" ]; then + if [ -n "$pr" ]; then + gh pr close "$pr" --delete-branch --comment "main already has these releases." + fi + exit 0 + fi + + git switch -q -C "$branch" origin/main + while read -r _ old new <&3; do + "$SCRIPT" apply "$old" "$new" + done 3<"$RUNNER_TEMP/minor" + title="$("$SCRIPT" title <"$RUNNER_TEMP/minor")" + git commit -q -am "$title" + git push -q --force origin "$branch" + head="$(git rev-parse HEAD)" + if [ -z "$pr" ]; then + gh pr create --base main --head "$branch" --title "$title" --body "" --label image-update + pr="$(gh pr view "$branch" --json number --jq .number)" + else + gh pr edit "$pr" --title "$title" + fi + + # Let the checks the push started register before waiting on them. + sleep 90 + if ! GH_TOKEN="$CHECKS_TOKEN" gh pr checks "$pr" --watch --fail-fast --interval 30; then + gh pr edit "$pr" --add-assignee "$REVIEWERS" + echo "::error::#$pr's checks failed, so it was left open for $REVIEWERS" + exit 1 + fi + if ! gh pr merge "$pr" --squash --match-head-commit "$head"; then + gh pr edit "$pr" --add-assignee "$REVIEWERS" + echo "::error::#$pr passed its checks but could not be merged, so it was left open for $REVIEWERS" + exit 1 + fi + + - name: Open a PR for each major release + # Also after the minor releases' PR fails, but not when the setup did. + if: ${{ !cancelled() && steps.minor.outcome != 'skipped' }} + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + # true when origin's $1 changes main the way HEAD does + same_change() { + [ "$(git diff "origin/main...origin/$1" | git patch-id --stable)" = \ + "$(git diff origin/main HEAD | git patch-id --stable)" ] + } + + # Planned again on main, which may now have the minor releases. + git fetch -q origin main + git switch -q --discard-changes --detach origin/main + "$SCRIPT" plan >"$RUNNER_TEMP/plan" + awk '$1 == "major"' "$RUNNER_TEMP/plan" >"$RUNNER_TEMP/major" + cat "$RUNNER_TEMP/major" + + while read -r kind old new <&3; do + name="${new%%@*}" + tag="${name##*:}" + name="${name%:*}" + old_tag="${old%%@*}" + old_tag="${old_tag##*:}" + slug="$(printf '%s' "$name" | tr '/:' '--')" + branch="image-updates/major/$slug/$tag" + state="$(gh pr list --head "$branch" --state all --json state \ + --jq 'map(.state) | if any(. == "OPEN") then "open" elif length > 0 then "closed" else "" end')" + # Merged, or closed to skip this release. + if [ "$state" = closed ]; then + continue + fi + + git switch -q --discard-changes -C "$branch" origin/main + "$SCRIPT" apply "$old" "$new" + title="$("$SCRIPT" title <<<"$kind $old $new")" + git commit -q -am "$title" + + if [ "$state" = open ]; then + # main changed this pin since the PR was opened, as a minor + # release of the same image does: rebuild it so it still merges. + if ! same_change "$branch"; then + git push -q --force origin "$branch" + fi + continue + fi + + git push -q --force origin "$branch" + label="${title#chore: }" + gh pr create --base main --head "$branch" --title "$title" \ + --label image-update:major --assignee "$REVIEWERS" --reviewer "$REVIEWERS" \ + --body "$(printf '%s\n' \ + "$label is a new major release, up from $old_tag, so it waits for a review instead of merging itself. Before merging, check that the panel still works on it:" \ + "" \ + "- [ ] Read its release notes from $old_tag up to $tag for breaking changes and migrations." \ + "- [ ] Try it on a test panel. This branch is published to the \`branch/$branch\` channel: \`sudo ./service.sh --channel branch/$branch\`." \ + "" \ + "Closing this PR skips $tag. A newer major release opens a new PR.")" + pr="$(gh pr view "$branch" --json number --jq .number)" + + gh pr list --state open --label image-update:major --json number,headRefName \ + --jq ".[] | select(.headRefName | startswith(\"image-updates/major/$slug/\")) | select(.headRefName != \"$branch\") | .number" | + while read -r superseded; do + gh pr close "$superseded" --delete-branch --comment "Superseded by #$pr." + done + done 3<"$RUNNER_TEMP/major" diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml index fb594cb..0908be9 100644 --- a/.github/workflows/panel-scripts.yml +++ b/.github/workflows/panel-scripts.yml @@ -25,6 +25,7 @@ on: - "base/backups/**" - "base/game-server-node-connector/**" - "overlays/nvidia/**" + - ".github/scripts/**" - .github/workflows/panel-scripts.yml pull_request: paths: @@ -49,6 +50,7 @@ on: - "base/backups/**" - "base/game-server-node-connector/**" - "overlays/nvidia/**" + - ".github/scripts/**" - .github/workflows/panel-scripts.yml jobs: @@ -56,7 +58,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: shellcheck -x service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh tests/scripts/*.sh tests/scripts/stub/kubectl + - run: shellcheck -x service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh .github/scripts/image_updates.sh tests/scripts/*.sh tests/scripts/stub/kubectl tests/scripts/stub/crane - run: bash tests/scripts/setup_env_test.sh - run: bash tests/scripts/apply_overlay_test.sh - run: bash tests/scripts/debug_test.sh @@ -72,3 +74,4 @@ jobs: - run: bash tests/scripts/postgres_backup_test.sh - run: bash tests/scripts/choose_node_test.sh - run: bash tests/scripts/nvidia_fallback_test.sh + - run: bash tests/scripts/image_updates_test.sh diff --git a/base/gateway/gateway.yaml b/base/gateway/gateway.yaml index 0816337..ea15725 100644 --- a/base/gateway/gateway.yaml +++ b/base/gateway/gateway.yaml @@ -2,7 +2,7 @@ # inherit the hostname from. These are the plain-HTTP listeners of a # REVERSE_PROXY install; overlays/http/gateway-https-patch.yaml turns them into # HTTPS ones. Ports are Traefik's entrypoints (charts/gateway), not the node's: -# HTTP_PORT and HTTPS_PORT only move the LoadBalancer Service. The listeners for +# HTTP_PORT and HTTPS_PORT only move Traefik's host ports. The listeners for # every other host are on the chart's shared Gateway, traefik/traefik-gateway: # web (HTTP) and websecure (HTTPS on a self-signed certificate). apiVersion: gateway.networking.k8s.io/v1 diff --git a/charts/gateway-crds/Chart.yaml b/charts/gateway-crds/Chart.yaml index 984e129..40044eb 100644 --- a/charts/gateway-crds/Chart.yaml +++ b/charts/gateway-crds/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: gateway-crds -description: The Gateway API (v1.6.3, standard channel) and Traefik v3.7 CRDs that the gateway chart needs. Install it first. +description: The Gateway API (v1.6.3, standard channel) and Traefik v3.7 CRDs that the gateway chart needs. Install it first. Set gatewayApi=false where the cluster brings its own Gateway API CRDs, as k3s does from 1.37. type: application -version: 0.1.0 +version: 0.2.0 appVersion: v1.6.3 kubeVersion: ">=1.31.0-0" diff --git a/charts/gateway-crds/templates/gateway-api.yaml b/charts/gateway-crds/templates/gateway-api.yaml index 37d14c1..ce936d5 100644 --- a/charts/gateway-crds/templates/gateway-api.yaml +++ b/charts/gateway-crds/templates/gateway-api.yaml @@ -1,6 +1,8 @@ +{{- if .Values.gatewayApi }} # Gateway API v1.6.3, standard channel: https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.6.3/standard-install.yaml # (sha256 356c2b8e286a964dc20995435659ad1095386329974793af10bbfe3eff977fe5), with helm.sh/resource-policy: keep -# added to each CRD so an uninstall never takes every HTTPRoute on the cluster with it. +# added to every object here, so neither an uninstall nor gatewayApi: false deletes the +# CRDs, and every HTTPRoute on the cluster with them, or the safe-upgrades policy. # Copyright The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20078,6 +20080,7 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy metadata: annotations: + helm.sh/resource-policy: keep gateway.networking.k8s.io/bundle-version: v1.6.3 gateway.networking.k8s.io/channel: standard name: "safe-upgrades.gateway.networking.k8s.io" @@ -20115,6 +20118,7 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding metadata: annotations: + helm.sh/resource-policy: keep gateway.networking.k8s.io/bundle-version: v1.6.3 gateway.networking.k8s.io/channel: standard name: safe-upgrades.gateway.networking.k8s.io @@ -20127,3 +20131,4 @@ spec: apiVersions: ["v1"] resources: ["customresourcedefinitions"] operations: ["CREATE", "UPDATE"] +{{- end }} diff --git a/charts/gateway-crds/values.yaml b/charts/gateway-crds/values.yaml new file mode 100644 index 0000000..c6106ab --- /dev/null +++ b/charts/gateway-crds/values.yaml @@ -0,0 +1,4 @@ +# The Gateway API's CRDs and its safe-upgrades admission policy. Turn it off +# where something else installs them, as k3s does from 1.37 with its +# gateway-api-crd chart. Traefik's CRDs are installed either way. +gatewayApi: true diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index ccb8c6f..7cf3817 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -80,10 +80,17 @@ traefik: readTimeout: 0 writeTimeout: 0 + # install_traefik binds the host's ports straight to the pod (hostPort): + # k3s ServiceLB masquerades, so behind it every request came from its pod. service: spec: - type: LoadBalancer - externalTrafficPolicy: Local + type: ClusterIP + + # Old and new pods can't hold the same host port at once. + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 resources: requests: diff --git a/dev.sh b/dev.sh index d31fce0..cb50854 100755 --- a/dev.sh +++ b/dev.sh @@ -58,6 +58,7 @@ kubectl config use-context k3d-5stack-dev install_traefik "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1 retire_ingress_nginx --immediately || exit 1 +move_traefik_to_host_ports "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1 echo "Labeling node..." kubectl label node $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') 5stack-api=true 5stack-hasura=true 5stack-minio=true 5stack-rustfs=true 5stack-timescaledb=true 5stack-redis=true 5stack-typesense=true 5stack-web=true 5stack-dev-server=true 5stack-mediamtx=true diff --git a/tests/gateway/k3s_upgrade_test.sh b/tests/gateway/k3s_upgrade_test.sh new file mode 100644 index 0000000..6baf77e --- /dev/null +++ b/tests/gateway/k3s_upgrade_test.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# A panel installed while k3s left the Gateway API CRDs to it, then k3s +# upgraded in place as the panel's k3s updates do it: the same datastore and +# flags on a new binary. From 1.37 k3s's own gateway-api-crd chart installs +# beside gateway-crds' copies, and the next update.sh hands them over to it, +# safe-upgrades policy and all. + +# shellcheck source=tests/gateway/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +trap cleanup EXIT + +API="api.$DOMAIN" + +gateway_api_objects() { + chart_objects "$PANEL_DIR/charts/gateway-crds/templates/gateway-api.yaml" +} + +# release/namespace from an object's Helm marks, or "missing". +owner() { + kubectl get "$1" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}/{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || + echo missing +} + +expect_gateway_api_owned_by() { + local object + for object in $(gateway_api_objects); do + [ "$(owner "$object")" = "$1" ] || fail "$object belongs to $(owner "$object"), not $1" + done +} + +update_panel() { + install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" && routing_layer http +} + +# k3s runs its chart's install Job a little after it starts. +k3s_chart_installed() { + for _ in $(seq 1 60); do + [ "$(kubectl -n kube-system get job helm-install-gateway-api-crd -o jsonpath='{.status.succeeded}' 2>/dev/null)" = 1 ] && + return 0 + sleep 3 + done + return 1 +} + +crds_generation() { + kubectl -n kube-system get helmchart gateway-crds -o jsonpath='{.metadata.generation}' +} + +test_gateway_crds_holds_the_gateway_api_before_k3s_does() { + expect_gateway_api_owned_by gateway-crds/traefik + wait_for 200 http "$API" /me +} + +test_the_panel_serves_through_the_k3s_upgrade() { + k3s_chart_installed || fail "$K3S_IMAGE never installed its gateway-api-crd chart" + wait_for 200 http "$API" /me +} + +test_update_hands_the_gateway_api_to_k3s() { + update_panel || fail "update.sh's routing steps failed after the k3s upgrade" + expect_gateway_api_owned_by gateway-api-crd/kube-system + case "$(kubectl -n kube-system get helmchart gateway-crds -o jsonpath='{.spec.valuesContent}')" in + *"gatewayApi: false"*) ;; + *) fail "gateway-crds still installs the Gateway API" ;; + esac + kubectl wait --for=condition=Established crd/middlewares.traefik.io --timeout=10s >/dev/null || + fail "gateway-crds dropped Traefik's CRDs with the Gateway API's" + wait_for 200 http "$API" /me +} + +test_a_restart_keeps_the_handover() { + k3s_chart_installed || fail "k3s didn't run its gateway-api-crd chart again" + expect_gateway_api_owned_by gateway-api-crd/kube-system + wait_for 200 http "$API" /me +} + +test_another_update_changes_nothing() { + local before + before="$(crds_generation)" + update_panel || fail "update.sh's routing steps failed on a second run" + [ "$(crds_generation)" = "$before" ] || fail "a second update changed the gateway-crds HelmChart" + expect_gateway_api_owned_by gateway-api-crd/kube-system + wait_for 200 http "$API" /me +} + +K3S_IMAGE="$K3S_BEFORE_GATEWAY_API" start_cluster || exit 1 +make_panel || exit 1 +update_panel >/dev/null || exit 1 +deploy_panel_stubs || exit 1 +run_test test_gateway_crds_holds_the_gateway_api_before_k3s_does + +run_k3s "$K3S_IMAGE" || exit 1 +run_test test_the_panel_serves_through_the_k3s_upgrade +run_test test_update_hands_the_gateway_api_to_k3s + +# A reboot, or the next k3s patch. +run_k3s "$K3S_IMAGE" || exit 1 +run_test test_a_restart_keeps_the_handover +run_test test_another_update_changes_nothing + +summary diff --git a/tests/gateway/lib.sh b/tests/gateway/lib.sh index 43fd7d5..15dc9c2 100644 --- a/tests/gateway/lib.sh +++ b/tests/gateway/lib.sh @@ -2,19 +2,26 @@ # k3s in Docker, with the panel's routing layer installed by the same functions # update.sh calls, in front of stub backends named like the real Services. -# Requests come from a separate client container on the same Docker network, +# Requests come from a separate client container on a second Docker network, # so the client IP the backends see is a real peer's. One cluster at a time; # everything is named after $PREFIX and removed on exit. HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_ROOT="${TEST_ROOT:-$(cd "$HERE/../.." && pwd)}" PREFIX="${PREFIX:-gateway-test}" -K3S_IMAGE=rancher/k3s:v1.36.5-k3s1@sha256:057cfd5a9d51b89f1f0ee139c28d5d5fdd5aa7ddeac2742a0ba90c6dbc0329bb +K3S_IMAGE=rancher/k3s:v1.37.2-k3s1@sha256:21ea6e0fac4e3930c2ce8aa40ddd41a8b18cb3e9a0222c030dc005e7e88c4332 +# The last k3s that leaves the Gateway API CRDs to whoever installs Traefik; +# from 1.37 its gateway-api-crd chart installs them. Not named *_IMAGE, so +# the image-updates workflow leaves it here while K3S_IMAGE moves on. +# shellcheck disable=SC2034 # for the tests that source this +K3S_BEFORE_GATEWAY_API=rancher/k3s:v1.36.5-k3s1@sha256:057cfd5a9d51b89f1f0ee139c28d5d5fdd5aa7ddeac2742a0ba90c6dbc0329bb PYTHON_IMAGE=python:3.13.18-alpine3.24@sha256:bbc0290146ebe0a8b9a8ed07d3f2c7bdeb4d917fc93300a4e47490c8b5a11b48 DOMAIN=5stack.test WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/gateway-test.XXXXXX")" PANEL_DIR="$WORK_DIR/panel" KUBECONFIG="$WORK_DIR/kubeconfig" +# k3s keeps its datastore here, so run_k3s can bring the same cluster back. +K3S_STATE="$PREFIX-k3s-state" DEBUG=false TESTS_PASSED=0 TESTS_FAILED=0 @@ -22,7 +29,8 @@ export KUBECONFIG PANEL_DIR DEBUG cleanup() { docker rm -f -v "$PREFIX-k3s" "$PREFIX-client" >/dev/null 2>&1 - docker network rm "$PREFIX-net" >/dev/null 2>&1 + docker network rm "$PREFIX-net" "$PREFIX-public" >/dev/null 2>&1 + docker volume rm "$K3S_STATE" >/dev/null 2>&1 rm -rf "$WORK_DIR" } @@ -51,13 +59,38 @@ client() { docker exec "$PREFIX-client" "$@" } +# A cluster on K3S_IMAGE, with the client container. start_cluster() { - local port cleanup mkdir -p "$WORK_DIR" docker network create "$PREFIX-net" >/dev/null || return 1 + # Clients reach the node on an address k3s doesn't know as its InternalIP, + # as a server's public one is when k3s runs on a private or VPN address. + # kube-proxy answers the InternalIP itself, which hid that ServiceLB + # masquerades everything else. + docker network create "$PREFIX-public" >/dev/null || return 1 + run_k3s "$K3S_IMAGE" || return 1 + + docker run -d --name "$PREFIX-client" --network "$PREFIX-net" "$PYTHON_IMAGE" sleep infinity >/dev/null || return 1 + client apk add --no-cache -q curl jq openssl >/dev/null || return 1 + docker cp "$HERE/client.py" "$PREFIX-client:/client.py" >/dev/null + docker network connect "$PREFIX-public" "$PREFIX-client" || return 1 + # shellcheck disable=SC2034 + CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-client")" +} + +# (Re)starts the k3s server on the cluster's state volume. On another image +# that is the in-place upgrade the panel's k3s updates make with +# rancher/k3s-upgrade: the same datastore and flags on a new binary. Waits for +# the new kubelet itself, since the Node the old one left still says Ready. +run_k3s() { + local image="$1" version port + version="${image%%@*}" + version="${version##*:}" + version="${version/-k3s/+k3s}" + docker rm -f "$PREFIX-k3s" >/dev/null 2>&1 docker run -d --name "$PREFIX-k3s" --hostname "$PREFIX-k3s" --network "$PREFIX-net" --privileged \ - --tmpfs /run --tmpfs /var/run -p 127.0.0.1::6443 "$K3S_IMAGE" server \ + --tmpfs /run --tmpfs /var/run -v "$K3S_STATE:/var/lib/rancher/k3s" -p 127.0.0.1::6443 "$image" server \ --disable=traefik --disable=metrics-server --disable=local-storage --tls-san 127.0.0.1 >/dev/null || return 1 port="$(docker port "$PREFIX-k3s" 6443/tcp | head -n 1 | sed 's/.*://')" for _ in $(seq 1 90); do @@ -66,17 +99,19 @@ start_cluster() { done docker exec "$PREFIX-k3s" cat /etc/rancher/k3s/k3s.yaml | sed "s#https://127.0.0.1:6443#https://127.0.0.1:$port#" >"$KUBECONFIG" for _ in $(seq 1 90); do - kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null 2>&1 && break + if [ "$(kubectl get node "$PREFIX-k3s" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>/dev/null)" = "$version" ] && + [ "$(kubectl get --raw "/api/v1/nodes/$PREFIX-k3s/proxy/healthz" 2>/dev/null)" = ok ] && + kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null 2>&1; then + break + fi sleep 2 done kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null || return 1 - NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-k3s")" + [ "$(kubectl get node "$PREFIX-k3s" -o jsonpath='{.status.nodeInfo.kubeletVersion}')" = "$version" ] || return 1 - docker run -d --name "$PREFIX-client" --network "$PREFIX-net" "$PYTHON_IMAGE" sleep infinity >/dev/null || return 1 - client apk add --no-cache -q curl jq openssl >/dev/null || return 1 - docker cp "$HERE/client.py" "$PREFIX-client:/client.py" >/dev/null - # shellcheck disable=SC2034 - CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-client")" + docker network connect "$PREFIX-public" "$PREFIX-k3s" || return 1 + NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-k3s")" + [ "$NODE_IP" != "$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" ] || return 1 } set_env() { diff --git a/tests/gateway/servicelb_upgrade_test.sh b/tests/gateway/servicelb_upgrade_test.sh new file mode 100644 index 0000000..6f1d6c6 --- /dev/null +++ b/tests/gateway/servicelb_upgrade_test.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# An install whose Traefik still sits behind k3s ServiceLB on an older image, as +# the panel ran it before Traefik bound the host's ports itself, taken through +# update.sh's install_traefik. + +# shellcheck source=tests/gateway/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +trap cleanup EXIT + +API="api.$DOMAIN" +PORTS_ENV="$PANEL_DIR/overlays/config/traefik-config.env" + +install_traefik_behind_servicelb() { + local crd + install_helm_chart gateway-crds "$PANEL_DIR/charts/gateway-crds" "" >/dev/null || return 1 + for crd in gatewayclasses.gateway.networking.k8s.io gateways.gateway.networking.k8s.io \ + httproutes.gateway.networking.k8s.io middlewares.traefik.io; do + kubectl wait --for=condition=Established "crd/$crd" --timeout=120s >/dev/null || return 1 + done + install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat <<'EOF' +traefik: + versionOverride: v3.7.15 + image: + digest: sha256:0f6150202bc1fed72a7bd2107460eaa510e5b8a905f3fdb5b84f2c8619c33368 + service: + spec: {type: LoadBalancer, externalTrafficPolicy: Local} + updateStrategy: + rollingUpdate: {maxUnavailable: 0, maxSurge: 1} +EOF +)" >/dev/null || return 1 + kubectl -n traefik rollout status deployment/traefik --timeout=300s >/dev/null || return 1 + for _ in $(seq 1 90); do + [ -n "$(kubectl -n traefik get service traefik -o jsonpath='{.status.loadBalancer.ingress}')" ] && return 0 + sleep 2 + done + return 1 +} + +traefik_pod() { + kubectl -n traefik get pods -l app.kubernetes.io/name=traefik -o jsonpath='{.items[*].metadata.uid}' +} + +test_behind_servicelb_every_request_comes_from_its_pod() { + wait_for 200 https "$API" /me + case "$(json '.headers["x-forwarded-for"]')" in + 10.42.*) ;; + *) fail "expected a ServiceLB pod address, got $(json '.headers["x-forwarded-for"]')" ;; + esac +} + +test_the_move_costs_seconds() { + local longest + longest="$(jq -r .longest_outage <<<"$POLL")" + echo " outage while Traefik moved: ${longest}s ($(jq -r '"\(.failed) of \(.requests)"' <<<"$POLL") requests failed)" + awk -v t="$longest" 'BEGIN { exit !(t < 30) }' || fail "$POLL" +} + +test_servicelb_no_longer_fronts_traefik() { + [ "$(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')" = ClusterIP ] || fail "Traefik's Service is still $(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')" + [ -z "$(kubectl -n kube-system get daemonset -l svccontroller.k3s.cattle.io/svcname=traefik -o name)" ] || fail "svclb-traefik is still running" +} + +test_traefik_runs_the_new_image_pulled_beforehand() { + local want + want="docker.io/traefik@$(sed -n 's/^ *digest: *\(sha256:[0-9a-f]*\).*/\1/p' "$PANEL_DIR/charts/gateway/values.yaml")" + [ "$(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')" = "$want" ] || + fail "Traefik runs $(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')" + kubectl -n traefik wait --for=delete pod/traefik-pull --timeout=60s >/dev/null 2>&1 || fail "the pull pod was left behind" +} + +test_the_client_ip_reaches_the_backend() { + call https "$API" /me + expect_code 200 + expect_json '.headers["x-forwarded-for"]' "$CLIENT_IP" + expect_json '.headers["x-real-ip"]' "$CLIENT_IP" + call http "$API" /me + expect_code 308 +} + +test_running_the_update_again_changes_nothing() { + local before + before="$(traefik_pod)" + install_traefik "$PORTS_ENV" >/dev/null || fail "install_traefik failed" + move_traefik_to_host_ports "$PORTS_ENV" >/dev/null || fail "move_traefik_to_host_ports failed" + [ "$(traefik_pod)" = "$before" ] || fail "Traefik was restarted" + call https "$API" /me + expect_code 200 +} + +start_cluster || exit 1 +make_panel || exit 1 +install_traefik_behind_servicelb || exit 1 +deploy_panel_stubs || exit 1 +routing_layer https || exit 1 +tls_secret 5stack-ssl "$DOMAIN" "$API" "demos.$DOMAIN" "ws.$DOMAIN" "tv.$DOMAIN" "search.$DOMAIN" "console.$DOMAIN" || exit 1 +tls_secret 5stack-mediamtx-ssl "hls.$DOMAIN" || exit 1 + +run_test test_behind_servicelb_every_request_comes_from_its_pod + +client rm -f /tmp/stop +docker exec -d "$PREFIX-client" sh -c "python3 /client.py poll $NODE_IP $API /me /tmp/stop >/tmp/poll.json" +sleep 2 +install_traefik "$PORTS_ENV" || exit 1 +sleep 5 +client touch /tmp/stop +for _ in $(seq 1 10); do + client test -s /tmp/poll.json && break + sleep 1 +done +POLL="$(client cat /tmp/poll.json)" + +for name in \ + test_the_move_costs_seconds \ + test_servicelb_no_longer_fronts_traefik \ + test_traefik_runs_the_new_image_pulled_beforehand \ + test_the_client_ip_reaches_the_backend \ + test_running_the_update_again_changes_nothing; do + run_test "$name" +done + +summary diff --git a/tests/gateway/upgrade_test.sh b/tests/gateway/upgrade_test.sh index 0ab0f7e..8854126 100644 --- a/tests/gateway/upgrade_test.sh +++ b/tests/gateway/upgrade_test.sh @@ -3,7 +3,8 @@ # An install that is still on ingress-nginx -- the controller, its admission # webhook, the panel's Ingresses as they were, and another app's nginx-class # Ingress with forward-auth -- taken through update.sh's routing steps: -# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx. +# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx, +# move_traefik_to_host_ports. # shellcheck source=tests/gateway/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" @@ -38,7 +39,7 @@ install_ingress_nginx() { # traefik-crd release, which is gone. leave_k3s_traefik_crds() { local -a crds - kubectl apply --server-side -f "$PANEL_DIR/charts/gateway-crds/templates/gateway-api.yaml" >/dev/null || return 1 + grep -v "^{{" "$PANEL_DIR/charts/gateway-crds/templates/gateway-api.yaml" | kubectl apply --server-side -f - >/dev/null || return 1 kubectl delete validatingadmissionpolicybinding,validatingadmissionpolicy safe-upgrades.gateway.networking.k8s.io >/dev/null || return 1 mapfile -t crds < <(kubectl get crd -o name | grep '\.gateway\.networking\.k8s\.io$') kubectl annotate --overwrite "${crds[@]}" meta.helm.sh/release-name=traefik-crd meta.helm.sh/release-namespace=kube-system >/dev/null || return 1 @@ -200,12 +201,15 @@ test_running_the_update_again_changes_nothing() { install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "install_traefik failed" routing_layer https || fail "the overlay did not apply" retire_ingress_nginx || fail "retire_ingress_nginx failed" + move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "move_traefik_to_host_ports failed" call https "$API" /me expect_code 200 expect_other_app_routes_with_its_auth } -start_cluster || exit 1 +# k3s's Traefik only held the Gateway API CRDs before 1.37, when k3s began +# installing them as a chart of their own. +K3S_IMAGE="$K3S_BEFORE_GATEWAY_API" start_cluster || exit 1 make_panel || exit 1 install_ingress_nginx || exit 1 deploy_panel_stubs || exit 1 @@ -222,6 +226,7 @@ sleep 2 install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1 routing_layer https || exit 1 retire_ingress_nginx || exit 1 +move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1 sleep 5 client touch /tmp/stop for _ in $(seq 1 10); do diff --git a/tests/scripts/image_updates_test.sh b/tests/scripts/image_updates_test.sh new file mode 100644 index 0000000..37cac2d --- /dev/null +++ b/tests/scripts/image_updates_test.sh @@ -0,0 +1,181 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +STUB_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")/stub" && pwd)" +SCRIPT="$TEST_ROOT/.github/scripts/image_updates.sh" +OLD_DIGEST="sha256:$(printf "%064d" 1)" +NEW_DIGEST="sha256:$(printf "%064d" 2)" + +# pin FILE LINE...: a tracked file of the test repository. +pin() { + mkdir -p "$(dirname "$WORK_DIR/repo/$1")" + printf '%s\n' "${@:2}" >"$WORK_DIR/repo/$1" +} + +# tags REPOSITORY TAG...: what the registry lists for REPOSITORY. +tags() { + mkdir -p "$WORK_DIR/tags" + printf '%s\n' "${@:2}" >"$WORK_DIR/tags/$(printf '%s' "$1" | tr / _)" +} + +digest() { + echo "$1 $2" >>"$WORK_DIR/digests" +} + +image_updates() { + [ -d "$WORK_DIR/repo/.git" ] || git init -q "$WORK_DIR/repo" + ( + cd "$WORK_DIR/repo" || exit 1 + git add -A + PATH="$STUB_BIN:$PATH" bash "$SCRIPT" "$@" + ) +} + +test_the_newest_minor_and_the_newest_major_are_planned() { + pin base/typesense.yaml " image: typesense/typesense:30.2" + tags typesense/typesense 29.1 30.0 30.2 30.3 30.10 31.0 32.1 + assert_eq "$(image_updates plan)" "minor typesense/typesense:30.2 typesense/typesense:30.10 +major typesense/typesense:30.2 typesense/typesense:32.1" +} + +test_an_update_keeps_the_suffix_and_the_number_of_version_parts() { + pin base/redis.yaml " image: redis:8.10.3-alpine" + tags redis 8.10.3-alpine 8.10.4-alpine3.23 8.11-alpine 8.12.0-alpine 8.12.1 9.0.0-rc1-alpine 8.12.1-alpine-arm64 + assert_eq "$(image_updates plan)" "minor redis:8.10.3-alpine redis:8.12.0-alpine" +} + +test_an_update_keeps_the_v() { + pin base/hasura.yaml " - image: hasura/graphql-engine:v2.51.3-ce.cli-migrations-v3" + tags hasura/graphql-engine v2.51.4-ce.cli-migrations-v3 2.51.5-ce.cli-migrations-v3 v2.51.9 v2.52.0-beta.1-ce.cli-migrations-v3 + assert_eq "$(image_updates plan)" "minor hasura/graphql-engine:v2.51.3-ce.cli-migrations-v3 hasura/graphql-engine:v2.51.4-ce.cli-migrations-v3" +} + +test_prereleases_and_tags_that_are_not_versions_are_skipped() { + pin base/typesense.yaml " image: typesense/typesense:30.2" + tags typesense/typesense latest main 31.0.rc22-arm64 30.3rc1 30.2 + assert_eq "$(image_updates plan)" "" +} + +# Below 1.0 a minor release may break things, as a major does from 1.0 on. +test_below_1_0_a_new_second_part_is_major() { + pin base/rustfs.yaml " image: rustfs/rc:v0.1.36" + tags rustfs/rc v0.1.35 v0.1.37 v0.2.0 v1.0.0 + assert_eq "$(image_updates plan)" "minor rustfs/rc:v0.1.36 rustfs/rc:v0.1.37 +major rustfs/rc:v0.1.36 rustfs/rc:v1.0.0" +} + +test_a_one_part_version_only_moves_to_a_major() { + pin base/backup.yaml " image: postgres:18-alpine" + tags postgres 17-alpine 18-alpine 18.1-alpine 19-alpine 19beta1-alpine 19-alpine3.22 + assert_eq "$(image_updates plan)" "major postgres:18-alpine postgres:19-alpine" +} + +test_a_pinned_digest_is_repinned_to_the_new_tags() { + pin base/redis.yaml " image: redis:8.10.3-alpine@$OLD_DIGEST" + tags redis 8.10.4-alpine + digest redis:8.10.4-alpine "$NEW_DIGEST" + assert_eq "$(image_updates plan)" "minor redis:8.10.3-alpine@$OLD_DIGEST redis:8.10.4-alpine@$NEW_DIGEST" +} + +test_a_digest_that_is_not_a_sha256_is_not_pinned() { + pin base/redis.yaml " image: redis:8.10.3-alpine@$OLD_DIGEST" + tags redis 8.10.4-alpine + digest redis:8.10.4-alpine "sha256:\$(reboot)" + assert_eq "$(image_updates plan 2>"$WORK_DIR/stderr")" "" + assert_contains "$(cat "$WORK_DIR/stderr")" "::warning::could not read the digest of redis:8.10.4-alpine" +} + +test_shell_and_quoted_pins_are_planned() { + pin tests/lib.sh "K3S_IMAGE=rancher/k3s:v1.36.5-k3s1" + pin overlays/x.yaml ' image: "coturn/coturn:4.18.0"' + tags rancher/k3s v1.36.5-rc1-k3s1 v1.36.6-k3s1 v1.36.5-k3s2 + tags coturn/coturn 4.18.1 4.18.0-r0-debian-amd64 + assert_eq "$(image_updates plan)" "minor coturn/coturn:4.18.0 coturn/coturn:4.18.1 +minor rancher/k3s:v1.36.5-k3s1 rancher/k3s:v1.36.6-k3s1" +} + +test_refs_without_a_version_tag_never_reach_the_registry() { + pin base/api.yaml " - image: ghcr.io/5stackgg/api:latest" + # shellcheck disable=SC2016 # a pin of an unexpanded variable + pin tests/t.sh "SEED_IMAGE=timescale/timescaledb@sha256:abc" 'image: $PEBBLE_IMAGE' + assert_eq "$(image_updates plan)" "" + [ ! -e "$WORK_DIR/crane-calls" ] || fail "crane was asked: $(cat "$WORK_DIR/crane-calls")" +} + +test_held_pins_are_not_planned() { + pin overlays/cert-manager-crds/cert-manager.yaml ' image: "quay.io/jetstack/cert-manager-controller:v1.17.4"' + pin base/system-upgrade/system-upgrade-controller.yaml " image: rancher/system-upgrade-controller:v0.20.2" + pin tests/timescaledb/major_upgrade_test.sh "OLD_IMAGE=timescale/timescaledb:2.30.2-pg17" + pin tests/scripts/image_updates_test.sh "K3S_IMAGE=rancher/k3s:v1.36.5-k3s1" + pin .github/workflows/x.yml " image: rancher/k3s:v1.36.5-k3s1" + tags rancher/k3s v1.36.6-k3s1 + tags quay.io/jetstack/cert-manager-controller v1.17.4 v1.21.2 + tags rancher/system-upgrade-controller v0.20.2 v0.20.3 + tags timescale/timescaledb 2.30.2-pg17 2.31.0-pg17 + assert_eq "$(image_updates plan)" "" +} + +test_an_unreadable_repository_skips_only_its_own_pins() { + pin base/a.yaml " image: busybox:1.38" " image: coturn/coturn:4.18.0" + tags busybox 1.38 1.39 + assert_eq "$(image_updates plan 2>"$WORK_DIR/stderr")" "minor busybox:1.38 busybox:1.39" + assert_contains "$(cat "$WORK_DIR/stderr")" "::warning::could not list the tags of coturn/coturn" +} + +test_apply_rewrites_every_pin_and_nothing_else() { + pin base/a.yaml \ + " image: busybox:1.38" \ + ' - image: "busybox:1.38"' \ + " image: busybox:1.38.1" \ + " image: mybusybox:1.38" \ + " image: docker.io/busybox:1.38" + pin tests/a.sh \ + "TEST_IMAGE=busybox:1.38" \ + "OTHER=busybox:1.38" \ + 'echo "pulling image busybox:1.38"' + pin tests/timescaledb/a.sh "OLD_IMAGE=busybox:1.38" + image_updates apply busybox:1.38 busybox:1.39 || fail "apply failed" + assert_eq "$(cat "$WORK_DIR/repo/base/a.yaml")" " image: busybox:1.39 + - image: \"busybox:1.39\" + image: busybox:1.38.1 + image: mybusybox:1.38 + image: docker.io/busybox:1.38" + assert_eq "$(cat "$WORK_DIR/repo/tests/a.sh")" "TEST_IMAGE=busybox:1.39 +OTHER=busybox:1.38 +echo \"pulling image busybox:1.38\"" + assert_eq "$(cat "$WORK_DIR/repo/tests/timescaledb/a.sh")" "OLD_IMAGE=busybox:1.38" +} + +test_apply_moves_a_digest_pin_whole() { + pin base/a.yaml " image: redis:8.10.3-alpine@$OLD_DIGEST" + image_updates apply "redis:8.10.3-alpine@$OLD_DIGEST" "redis:8.10.4-alpine@$NEW_DIGEST" || fail "apply failed" + assert_eq "$(cat "$WORK_DIR/repo/base/a.yaml")" " image: redis:8.10.4-alpine@$NEW_DIGEST" +} + +test_apply_fails_when_the_ref_is_not_pinned() { + pin tests/a.sh 'echo "pulling image busybox:1.38"' + if image_updates apply busybox:1.38 busybox:1.39 2>"$WORK_DIR/stderr"; then + fail "apply succeeded" + fi + assert_contains "$(cat "$WORK_DIR/stderr")" "busybox:1.38 is mentioned, but not pinned, in tests/a.sh" + if image_updates apply busybox:1.37 busybox:1.39 2>"$WORK_DIR/stderr"; then + fail "apply succeeded" + fi + assert_contains "$(cat "$WORK_DIR/stderr")" "busybox:1.37 is not pinned anywhere" +} + +test_the_title_names_each_image_and_its_new_tag() { + pin README.md "readme" + assert_eq "$(image_updates title <<<"minor redis:8.10.3-alpine redis:8.12.0-alpine@sha256:abc")" \ + "chore: redis 8.12.0-alpine" + assert_eq "$(printf '%s\n' \ + "minor redis:8.10.3-alpine redis:8.12.0-alpine@sha256:abc" \ + "minor ghcr.io/letsencrypt/pebble:2.10.1 ghcr.io/letsencrypt/pebble:2.10.2" \ + "major docker.io/library/postgres:18-alpine docker.io/library/postgres:19-alpine" | + image_updates title)" \ + "chore: redis 8.12.0-alpine, letsencrypt/pebble 2.10.2 and postgres 19-alpine" +} + +run_tests diff --git a/tests/scripts/stub/crane b/tests/scripts/stub/crane new file mode 100755 index 0000000..3ba173f --- /dev/null +++ b/tests/scripts/stub/crane @@ -0,0 +1,13 @@ +#!/bin/bash + +# crane for tests/scripts/image_updates_test.sh: the tags of a repository come +# from $WORK_DIR/tags/, and digests from +# " " lines in $WORK_DIR/digests. + +echo "$*" >>"$WORK_DIR/crane-calls" + +case "$1" in + ls) cat "$WORK_DIR/tags/$(printf '%s' "$2" | tr / _)" ;; + digest) awk -v ref="$2" '$1 == ref { print $2; found = 1 } END { exit !found }' "$WORK_DIR/digests" 2>/dev/null ;; + *) exit 2 ;; +esac diff --git a/tests/scripts/stub/kubectl b/tests/scripts/stub/kubectl index 76e019d..4859f10 100755 --- a/tests/scripts/stub/kubectl +++ b/tests/scripts/stub/kubectl @@ -3,6 +3,7 @@ echo "$*" >>"$WORK_DIR/calls" args=" $* " +file="" pod="" container="" previous=false @@ -14,13 +15,28 @@ while [ $# -gt 0 ]; do shift ;; --previous) previous=true ;; + -f) + file="$2" + shift + ;; esac shift done case "$args" in - *" get crd helmcharts.helm.cattle.io "* | *" apply --server-side "*) ;; + *" get crd helmcharts.helm.cattle.io "*) ;; + *" apply --server-side "*) if [ -n "$file" ]; then cat "$file" >>"$WORK_DIR/server-side-applied"; fi ;; *" apply -f - "*) cat >"$WORK_DIR/applied" ;; + *" get service traefik "*'{.spec.type}'*) + if [ -f "$WORK_DIR/traefik-service-error" ]; then + echo "The connection to the server was refused" >&2 + exit 1 + fi + if [ -f "$WORK_DIR/traefik-type" ]; then cat "$WORK_DIR/traefik-type"; fi + ;; + *" get deployment traefik "*) if [ -f "$WORK_DIR/traefik-image" ]; then cat "$WORK_DIR/traefik-image"; fi ;; + *" get pods -l app.kubernetes.io/name=traefik "*) if [ -f "$WORK_DIR/traefik-node" ]; then cat "$WORK_DIR/traefik-node"; fi ;; + *" wait pod/traefik-pull "*) [ ! -f "$WORK_DIR/pull-fails" ] || exit 1 ;; *" get service traefik "*) if [ -f "$WORK_DIR/traefik-address" ]; then cat "$WORK_DIR/traefik-address"; fi ;; *" get helmchart "*) echo 1 ;; *" get gateway traefik-gateway "*) [ -f "$WORK_DIR/shared-gateway" ] || exit 1 ;; @@ -76,7 +92,14 @@ case "$args" in *'name=="https"'*) port_file="$WORK_DIR/nginx-https-port" ;; *) port_file="$WORK_DIR/nginx-http-port" ;; esac + if [ -f "$WORK_DIR/nginx-error" ]; then + echo "The connection to the server was refused" >&2 + exit 1 + fi if [ ! -f "$port_file" ]; then + case "$args" in + *" --ignore-not-found "*) exit 0 ;; + esac echo 'Error from server (NotFound): services "ingress-nginx-controller" not found' >&2 exit 1 fi diff --git a/tests/scripts/traefik_install_test.sh b/tests/scripts/traefik_install_test.sh index a0c28b7..379f038 100644 --- a/tests/scripts/traefik_install_test.sh +++ b/tests/scripts/traefik_install_test.sh @@ -143,10 +143,23 @@ hand_over() { source "$TEST_ROOT/utils/output_redirect.sh" # shellcheck source=utils/install_traefik.sh source "$TEST_ROOT/utils/install_traefik.sh" - adopt_orphaned_chart_objects "$TEST_ROOT/charts/gateway-crds" gateway-crds traefik + adopt_orphaned_chart_objects "$TEST_ROOT/charts/gateway-crds" gateway-crds traefik "$@" ) >"$WORK_DIR/output" 2>&1 } +# Runs one of install_traefik.sh's functions against the stub. +traefik_fn() { + ( + # shellcheck source=utils/colors.sh + source "$TEST_ROOT/utils/colors.sh" + # shellcheck source=utils/output_redirect.sh + source "$TEST_ROOT/utils/output_redirect.sh" + # shellcheck source=utils/install_traefik.sh + source "$TEST_ROOT/utils/install_traefik.sh" + PANEL_DIR="$TEST_ROOT" "$@" + ) +} + handovers() { grep -E ' (annotate|label) ' "$WORK_DIR/calls" 2>/dev/null || true } @@ -219,6 +232,63 @@ test_a_cluster_with_nothing_left_over_hands_nothing_over() { assert_eq "$(handovers)" "" } +K3S_GATEWAY_API_CHART="CustomResourceDefinition/httproutes.gateway.networking.k8s.io|gateway-api-crd|kube-system" + +test_k3s_installing_the_gateway_api_is_seen_from_its_chart() { + if traefik_fn k3s_installs_gateway_api; then + fail "a cluster without k3s's gateway-api-crd chart was taken for one with it" + fi + echo helmchart.helm.cattle.io/gateway-api-crd >"$WORK_DIR/helmcharts-gateway-api-crd" + traefik_fn k3s_installs_gateway_api || fail "k3s's gateway-api-crd chart went unseen" +} + +# From k3s 1.37 its gateway-api-crd release holds the Gateway API CRDs, which +# gateway-crds then leaves out rather than refusing to install. +test_a_left_out_template_is_not_checked() { + cluster_objects "$K3S_GATEWAY_API_CHART" + echo helmchart.helm.cattle.io/gateway-api-crd >"$WORK_DIR/helmcharts-gateway-api-crd" + if hand_over; then + fail "gateway-crds would install over the CRDs of k3s's gateway-api-crd release" + fi + assert_contains "$(cat "$WORK_DIR/output")" "belongs to the Helm release gateway-api-crd in kube-system" + rm "$WORK_DIR/calls" + hand_over gateway-api.yaml || fail "adopt_orphaned_chart_objects failed: $(cat "$WORK_DIR/output")" + assert_eq "$(handovers)" "" +} + +# k3s's gateway-api-crd turned off after gateway-crds stepped aside leaves the +# CRDs marked as that release's; gateway-crds takes them back on its upgrade. +test_once_installed_what_a_removed_release_left_is_handed_over() { + cluster_objects "$K3S_GATEWAY_API_CHART" \ + "CustomResourceDefinition/ingressroutes.traefik.io||" \ + "CustomResourceDefinition/gateways.gateway.networking.k8s.io|gateway-crds|traefik" + echo secret/sh.helm.release.v1.gateway-crds.v2 >"$WORK_DIR/releases-gateway-crds" + hand_over || fail "adopt_orphaned_chart_objects failed: $(cat "$WORK_DIR/output")" + assert_contains "$(handovers)" "annotate --overwrite customresourcedefinition/httproutes.gateway.networking.k8s.io meta.helm.sh/release-name=gateway-crds meta.helm.sh/release-namespace=traefik" + assert_not_contains "$(handovers)" ingressroutes + assert_not_contains "$(handovers)" gateways +} + +test_once_installed_a_live_release_is_left_alone() { + cluster_objects "$K3S_GATEWAY_API_CHART" + echo secret/sh.helm.release.v1.gateway-crds.v2 >"$WORK_DIR/releases-gateway-crds" + echo helmchart.helm.cattle.io/gateway-api-crd >"$WORK_DIR/helmcharts-gateway-api-crd" + hand_over || fail "adopt_orphaned_chart_objects failed: $(cat "$WORK_DIR/output")" + assert_eq "$(handovers)" "" +} + +test_only_gateway_api_objects_gateway_crds_holds_are_handed_to_k3s() { + cluster_objects "CustomResourceDefinition/httproutes.gateway.networking.k8s.io|gateway-crds|traefik" \ + "ValidatingAdmissionPolicy/safe-upgrades.gateway.networking.k8s.io|gateway-crds|traefik" \ + "ValidatingAdmissionPolicyBinding/safe-upgrades.gateway.networking.k8s.io|gateway-crds|traefik" \ + "CustomResourceDefinition/ingressroutes.traefik.io|gateway-crds|traefik" \ + "CustomResourceDefinition/gateways.gateway.networking.k8s.io|gateway-api-crd|kube-system" \ + "CustomResourceDefinition/grpcroutes.gateway.networking.k8s.io||" + assert_eq "$(traefik_fn gateway_api_held_by_gateway_crds)" "customresourcedefinition/httproutes.gateway.networking.k8s.io +validatingadmissionpolicy/safe-upgrades.gateway.networking.k8s.io +validatingadmissionpolicybinding/safe-upgrades.gateway.networking.k8s.io" +} + test_an_unchanged_chart_that_installed_is_left_alone() { run_install_chart assert_eq "$(calls)" "" @@ -263,4 +333,112 @@ test_dev_retires_without_waiting_for_routes() { assert_not_contains "$(cat "$WORK_DIR/calls")" "wait httproute/api" } +test_retiring_with_no_ingress_nginx_waits_for_no_address() { + run_retire || fail "retire_ingress_nginx failed: $(cat "$WORK_DIR/output")" + assert_not_contains "$(cat "$WORK_DIR/calls")" "loadBalancer.ingress" +} + +try_install() { + ( + export PANEL_DIR="$TEST_ROOT" + # shellcheck source=utils/colors.sh + source "$TEST_ROOT/utils/colors.sh" + # shellcheck source=utils/output_redirect.sh + source "$TEST_ROOT/utils/output_redirect.sh" + # shellcheck source=utils/update_env_var.sh + source "$TEST_ROOT/utils/update_env_var.sh" + # shellcheck source=utils/install_traefik.sh + source "$TEST_ROOT/utils/install_traefik.sh" + "$@" "$WORK_DIR/traefik-config.env" + ) >"$WORK_DIR/output" 2>&1 +} + +run_install() { + try_install "$@" || fail "$1 exited $?: $(cat "$WORK_DIR/output")" +} + +chart_image() { + printf 'docker.io/traefik@%s' "$(sed -n 's/^ *digest: *\(sha256:[0-9a-f]*\).*/\1/p' "$TEST_ROOT/charts/gateway/values.yaml")" +} + +gateway_values() { + sed -n '/^ name: gateway$/,$p' "$WORK_DIR/server-side-applied" 2>/dev/null | sed -n '/valuesContent/,$p' +} + +test_traefik_binds_the_host_ports_itself() { + ports_file HTTP_PORT=8080 HTTPS_PORT=8443 + run_install install_traefik + assert_contains "$(gateway_values)" "web: {exposedPort: 8080, hostPort: 8080}" + assert_contains "$(gateway_values)" "websecure: {exposedPort: 8443, hostPort: 8443}" + assert_not_contains "$(gateway_values)" "LoadBalancer" +} + +test_traefik_waits_behind_servicelb_while_ingress_nginx_holds_the_ports() { + nginx_ports 80 443 + ports_file HTTP_PORT=80 HTTPS_PORT=443 + run_install install_traefik + assert_contains "$(gateway_values)" "spec: {type: LoadBalancer, externalTrafficPolicy: Local}" + assert_not_contains "$(gateway_values)" "hostPort" +} + +test_traefik_left_behind_servicelb_moves_onto_the_host_ports() { + echo LoadBalancer >"$WORK_DIR/traefik-type" + ports_file HTTP_PORT=80 HTTPS_PORT=443 + run_install move_traefik_to_host_ports + assert_contains "$(gateway_values)" "web: {exposedPort: 80, hostPort: 80}" +} + +test_traefik_already_on_the_host_ports_is_not_reinstalled() { + echo ClusterIP >"$WORK_DIR/traefik-type" + run_install move_traefik_to_host_ports + [ ! -f "$WORK_DIR/server-side-applied" ] || fail "the gateway chart was applied again" +} + +test_a_failed_service_lookup_fails_the_move() { + touch "$WORK_DIR/traefik-service-error" + if try_install move_traefik_to_host_ports; then + fail "move_traefik_to_host_ports succeeded without knowing where Traefik is" + fi +} + +test_a_failed_ingress_nginx_lookup_stops_the_install() { + touch "$WORK_DIR/nginx-error" + if try_install install_traefik; then + fail "install_traefik succeeded without knowing whether ingress-nginx holds the ports" + fi + assert_eq "$(gateway_values)" "" +} + +test_a_new_image_is_pulled_onto_traefiks_node_before_the_upgrade() { + echo docker.io/traefik@sha256:0000 >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + run_install install_traefik + assert_contains "$(grep ' run traefik-pull ' "$WORK_DIR/calls")" "--image=$(chart_image) --restart=Never --overrides={\"spec\":{\"nodeName\":\"node-a\"" + [ "$(grep -n ' run traefik-pull ' "$WORK_DIR/calls" | cut -d: -f1)" -lt "$(grep -n ' apply --server-side ' "$WORK_DIR/calls" | tail -n 1 | cut -d: -f1)" ] || + fail "the image was pulled after the upgrade" +} + +test_a_refused_pull_leaves_traefik_alone() { + echo docker.io/traefik@sha256:0000 >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + touch "$WORK_DIR/pull-fails" + if try_install install_traefik; then + fail "install_traefik succeeded although the image never arrived" + fi + assert_eq "$(gateway_values)" "" + assert_contains "$(cat "$WORK_DIR/output")" "keeps running its current image" +} + +test_traefik_already_on_the_image_pulls_nothing() { + chart_image >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + run_install install_traefik + assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" +} + +test_a_first_install_pulls_nothing() { + run_install install_traefik + assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" +} + run_tests diff --git a/update.sh b/update.sh index 9e71aad..f963857 100755 --- a/update.sh +++ b/update.sh @@ -117,6 +117,7 @@ if [ "$REVERSE_PROXY" != true ]; then fi retire_ingress_nginx || die "Traefik did not take over ports 80/443" +move_traefik_to_host_ports || die "Traefik did not take over the host's ports" if [ "$VAULT_MANAGER" = true ]; then step "Syncing Vault secrets" diff --git a/utils/install_traefik.sh b/utils/install_traefik.sh index c8b22e3..7d848a9 100644 --- a/utils/install_traefik.sh +++ b/utils/install_traefik.sh @@ -39,8 +39,8 @@ carry_over_ingress_nginx_ports() { install_traefik() { local ports_env="${1:-$PANEL_DIR/overlays/config/traefik-config.env}" - local minor http_port https_port crd - local -a kc=() + local minor http_port https_port crd binding nginx crds_values left_out held object + local -a kc=() handed=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") # Gateway API v1.6's CRDs validate with CEL's isIP(), which the API server @@ -62,8 +62,37 @@ install_traefik() { return 1 fi - adopt_orphaned_chart_objects "$PANEL_DIR/charts/gateway-crds" gateway-crds traefik || return 1 - install_helm_chart gateway-crds "$PANEL_DIR/charts/gateway-crds" "" || return 1 + # Where k3s installs the Gateway API itself, gateway-crds brings only + # Traefik's CRDs. One that still holds k3s's objects, from before k3s was + # upgraded to 1.37, hands them over: Helm deletes what a release drops + # unless the live object is marked keep, which before gateway-crds 0.2.0 + # the safe-upgrades policy and its binding weren't. + crds_values="" + left_out="" + handed=() + if k3s_installs_gateway_api; then + crds_values="gatewayApi: false" + left_out=gateway-api.yaml + held="$(gateway_api_held_by_gateway_crds)" || { + err "could not read the cluster's Gateway API CRDs before installing gateway-crds" + return 1 + } + while IFS= read -r object; do + [ -n "$object" ] && handed+=("$object") + done <<<"$held" + if [ "${#handed[@]}" -gt 0 ]; then + output_redirect kubectl "${kc[@]}" annotate --overwrite "${handed[@]}" helm.sh/resource-policy=keep || return 1 + fi + fi + + adopt_orphaned_chart_objects "$PANEL_DIR/charts/gateway-crds" gateway-crds traefik "$left_out" || return 1 + install_helm_chart gateway-crds "$PANEL_DIR/charts/gateway-crds" "$crds_values" || return 1 + + if [ "${#handed[@]}" -gt 0 ]; then + warn "k3s installs the Gateway API CRDs now; handing ${#handed[@]} of gateway-crds' objects to its gateway-api-crd release" + output_redirect kubectl "${kc[@]}" annotate --overwrite "${handed[@]}" \ + meta.helm.sh/release-name=gateway-api-crd meta.helm.sh/release-namespace=kube-system || return 1 + fi for crd in gatewayclasses.gateway.networking.k8s.io gateways.gateway.networking.k8s.io \ httproutes.gateway.networking.k8s.io middlewares.traefik.io; do @@ -73,16 +102,29 @@ install_traefik() { fi done - # The Service is externalTrafficPolicy: Local, so only the node running - # Traefik answers on these ports -- and DNS points at the server node, not - # at a game node that joined later. + # While ingress-nginx's ServiceLB pods hold the ports, Traefik waits behind + # its own until retire_ingress_nginx hands them over, so a stalled handover + # can still give them back; move_traefik_to_host_ports finishes the move. + nginx="$(kubectl "${kc[@]}" -n ingress-nginx get service ingress-nginx-controller --ignore-not-found -o name)" || return 1 + if [ -n "$nginx" ]; then + binding=" ports: + web: {exposedPort: $http_port} + websecure: {exposedPort: $https_port} + service: + spec: {type: LoadBalancer, externalTrafficPolicy: Local}" + else + binding=" ports: + web: {exposedPort: $http_port, hostPort: $http_port} + websecure: {exposedPort: $https_port, hostPort: $https_port}" + fi + + pull_traefik_image || return 1 + + # Only the node running Traefik listens on these ports, and DNS points at + # the server node, not at a game node that joined later. install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat </dev/null)" = "$image" ]; then + return 0 + fi + node="$(kubectl "${kc[@]}" -n traefik get pods -l app.kubernetes.io/name=traefik -o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null)" + if [ -z "$node" ]; then + return 0 + fi + + echo "Pulling $image onto $node..." + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --ignore-not-found --wait=true + if ! output_redirect kubectl "${kc[@]}" -n traefik run traefik-pull --image="$image" --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"$node\",\"tolerations\":[{\"operator\":\"Exists\"}]}}" \ + --command -- traefik version || + ! output_redirect kubectl "${kc[@]}" -n traefik wait pod/traefik-pull --timeout=300s \ + --for=jsonpath='{.status.phase}'=Succeeded; then + err "could not pull $image onto $node; Traefik keeps running its current image" + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --ignore-not-found --wait=false + return 1 + fi + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --wait=false +} + +move_traefik_to_host_ports() { + local type + local -a kc=() + [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") + + type="$(kubectl "${kc[@]}" -n traefik get service traefik -o jsonpath='{.spec.type}')" || return 1 + if [ "$type" != LoadBalancer ]; then + return 0 + fi + install_traefik "$@" +} + +# The cluster-scoped objects the given chart templates define, as kind/name. chart_objects() { awk '/^kind: / { kind = $2; next } - kind && /^ name: / { name = $2; gsub(/"/, "", name); print tolower(kind) "/" name; kind = "" }' "$1"/templates/*.yaml + kind && /^ name: / { name = $2; gsub(/"/, "", name); print tolower(kind) "/" name; kind = "" }' "$@" +} + +# The cluster's CRDs, admission policies and bindings with Helm's marks, as +# kind/name|release|release namespace. +marked_objects() { + local listed + local -a kc=() + [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") + + listed="$(kubectl "${kc[@]}" get customresourcedefinitions,validatingadmissionpolicies,validatingadmissionpolicybindings \ + -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}|{.metadata.annotations.meta\.helm\.sh/release-name}|{.metadata.annotations.meta\.helm\.sh/release-namespace}{"\n"}{end}')" || return 1 + awk -F'|' -v OFS='|' 'NF { $1 = tolower($1); print }' <<<"$listed" +} + +# From 1.37 k3s installs the Gateway API CRDs and the safe-upgrades policy +# itself, as its gateway-api-crd chart, whatever --disable=traefik says, and +# takes them over from any release that holds them. Turned off with +# --disable=gateway-api-crd, the chart isn't there. +k3s_installs_gateway_api() { + local -a kc=() + [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") + + [ -n "$(kubectl "${kc[@]}" get helmcharts.helm.cattle.io -A --field-selector metadata.name=gateway-api-crd -o name 2>/dev/null)" ] +} + +# The Gateway API objects marked as gateway-crds' own, as kind/name. +gateway_api_held_by_gateway_crds() { + local objects existing object owner owner_namespace + + objects="$(chart_objects "$PANEL_DIR/charts/gateway-crds/templates/gateway-api.yaml")" + existing="$(marked_objects)" || return 1 + while IFS='|' read -r object owner owner_namespace; do + if [ "$owner|$owner_namespace" = "gateway-crds|traefik" ] && grep -qxF "$object" <<<"$objects"; then + echo "$object" + fi + done <<<"$existing" } helm_release_gone() { @@ -133,33 +254,43 @@ helm_release_recorded() { # --disable=traefik deletes the release but leaves the CRDs marked as its, and # Helm won't install over objects another release claims. Ones whose release # is gone are handed to this one; anything a live release, or no release, owns -# stops the install. Only a first install can collide: Helm upgrades what a -# release already holds whatever its marks say, and the Traefik chart's own -# crds/ strips them from the traefik.io CRDs the two charts share. +# stops the install. Once the release exists, Helm upgrades what it holds +# whatever the marks say, and the Traefik chart's own crds/ strips them from +# the traefik.io CRDs the two charts share, so only objects a removed release +# left are handed over: the Gateway API's, after k3s's gateway-api-crd chart +# that gateway-crds stepped aside for is turned off. A template left out (its +# file name) is one the install won't render. adopt_orphaned_chart_objects() { - local chart_dir="$1" release="$2" release_namespace="$3" - local objects existing object owner owner_namespace refused=false - local -a kc=() orphaned=() + local chart_dir="$1" release="$2" release_namespace="$3" left_out="${4:-}" + local objects existing object owner owner_namespace template recorded=false refused=false + local -a kc=() orphaned=() templates=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") if helm_release_recorded "$release" "$release_namespace"; then - return 0 + recorded=true fi - objects="$(chart_objects "$chart_dir")" - if ! existing="$(kubectl "${kc[@]}" get customresourcedefinitions,validatingadmissionpolicies,validatingadmissionpolicybindings \ - -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}|{.metadata.annotations.meta\.helm\.sh/release-name}|{.metadata.annotations.meta\.helm\.sh/release-namespace}{"\n"}{end}')"; then + for template in "$chart_dir"/templates/*.yaml; do + [ "$(basename "$template")" = "$left_out" ] || templates+=("$template") + done + objects="$(chart_objects "${templates[@]}")" + if ! existing="$(marked_objects)"; then err "could not read the cluster's CRDs before installing $release" return 1 fi while IFS='|' read -r object owner owner_namespace; do [ -n "$object" ] || continue - object="$(printf '%s' "$object" | tr '[:upper:]' '[:lower:]')" grep -qxF "$object" <<<"$objects" || continue if [ "$owner" = "$release" ] && [ "$owner_namespace" = "$release_namespace" ]; then continue fi + if [ "$recorded" = true ]; then + if [ -n "$owner" ] && helm_release_gone "$owner" "$owner_namespace"; then + orphaned+=("$object") + fi + continue + fi if [ -z "$owner" ]; then err "$object was created outside Helm, so $release can't install it; delete it, or label it app.kubernetes.io/managed-by=Helm and annotate it meta.helm.sh/release-name=$release meta.helm.sh/release-namespace=$release_namespace" refused=true diff --git a/utils/retire_ingress_nginx.sh b/utils/retire_ingress_nginx.sh index 1e42e78..18990d2 100644 --- a/utils/retire_ingress_nginx.sh +++ b/utils/retire_ingress_nginx.sh @@ -48,7 +48,7 @@ YAML # retires ingress-nginx before Tilt has applied them, and a dev cluster has no # visitors to keep serving. retire_ingress_nginx() { - local deadline moved="" http_port="" https_port="" + local deadline http_port="" https_port="" local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -70,22 +70,21 @@ retire_ingress_nginx() { -o jsonpath='{.spec.ports[?(@.name=="https")].port}' 2>/dev/null)" echo "Moving ports 80/443 from ingress-nginx to Traefik..." output_redirect kubectl "${kc[@]}" delete service ingress-nginx-controller -n ingress-nginx --wait=true - moved=1 - fi - deadline=$((SECONDS + ${TRAEFIK_ADDRESS_TIMEOUT:-180})) - until [ -n "$(kubectl "${kc[@]}" get service traefik -n traefik -o jsonpath='{.status.loadBalancer.ingress}' 2>/dev/null)" ]; do - if [ "$SECONDS" -ge "$deadline" ]; then - err "Traefik's LoadBalancer Service never got an address:" - kubectl "${kc[@]}" get service traefik -n traefik >&2 - kubectl "${kc[@]}" get pods -A -l svccontroller.k3s.cattle.io/svcname=traefik >&2 - if [ -n "$moved" ] && restore_ingress_nginx_service "${http_port:-80}" "${https_port:-443}"; then - err "ingress-nginx has its ports back and keeps serving the panel; re-run ./update.sh once Traefik's Service can get an address" + deadline=$((SECONDS + ${TRAEFIK_ADDRESS_TIMEOUT:-180})) + until [ -n "$(kubectl "${kc[@]}" get service traefik -n traefik -o jsonpath='{.status.loadBalancer.ingress}' 2>/dev/null)" ]; do + if [ "$SECONDS" -ge "$deadline" ]; then + err "Traefik's LoadBalancer Service never got an address:" + kubectl "${kc[@]}" get service traefik -n traefik >&2 + kubectl "${kc[@]}" get pods -A -l svccontroller.k3s.cattle.io/svcname=traefik >&2 + if restore_ingress_nginx_service "${http_port:-80}" "${https_port:-443}"; then + err "ingress-nginx has its ports back and keeps serving the panel; re-run ./update.sh once Traefik's Service can get an address" + fi + return 1 fi - return 1 - fi - sleep 2 - done + sleep 2 + done + fi # Orphaned, not cascaded: on an old install ingress-shim may still own the # 5stack-ssl Certificate through one of these. diff --git a/utils/setup_kustomize.sh b/utils/setup_kustomize.sh index 85d7b34..2f369ca 100644 --- a/utils/setup_kustomize.sh +++ b/utils/setup_kustomize.sh @@ -4,23 +4,57 @@ setup_kustomize() { # $PANEL_DIR, not $(dirname "$0"): when update.sh is sourced from # install.sh, $0 is still install.sh, and both depend on the caller's cwd. local dir="$PANEL_DIR" - if ! [ -f "$dir/kustomize" ] || ! [ -x "$dir/kustomize" ] - then - echo "kustomize not found. Installing..." - # Pin the installer to an immutable release tag rather than the moving - # `master` branch, and download-then-run instead of piping the network - # straight into bash. The installer itself checksum-verifies the - # kustomize binary it fetches for the requested version. - local version="5.8.3" - local script - script="$(mktemp)" - if curl -fsSL "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/kustomize/v${version}/hack/install_kustomize.sh" -o "$script"; then - bash "$script" "$version" "$dir" - else - echo "Failed to download kustomize installer" >&2 - rm -f "$script" + local version="5.8.3" os arch sum got work + if [ -f "$dir/kustomize" ] && [ -x "$dir/kustomize" ]; then + return 0 + fi + + echo "kustomize not found. Installing..." + case "$(uname -s)" in + Linux) os=linux ;; + Darwin) os=darwin ;; + *) + echo "kustomize $version has no release for $(uname -s)" >&2 + return 1 + ;; + esac + case "$(uname -m)" in + x86_64 | amd64) arch=amd64 ;; + aarch64 | arm64) arch=arm64 ;; + *) + echo "kustomize $version has no release for $(uname -m)" >&2 return 1 - fi - rm -f "$script" + ;; + esac + # The release's own archive, checked against the sums its checksums.txt + # published. Kustomize's install script finds the release through GitHub's + # API instead, which refuses after 60 anonymous requests an hour from one + # address, as CI runners share theirs. + case "${os}_${arch}" in + linux_amd64) sum=cb9e31198d3f63b44848bd0afc4c8efc56e6cfef9c93a4a39a211940e6decd61 ;; + linux_arm64) sum=9867b76482cfc6d25b546abc1d2b5a32bd137505aba640649b4b378f64f6d68f ;; + darwin_amd64) sum=d08fb85846c3d977f0e2a66e7ff45f1b5264c294685e5e726e1a8bdfe0da8dee ;; + darwin_arm64) sum=4cbb08e209d8fff47cd60bfb6407daaf5ee699f9b9295849a1065024fac2d1ef ;; + esac + + work="$(mktemp -d)" + if ! curl -fsSL -o "$work/kustomize.tar.gz" \ + "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${version}/kustomize_v${version}_${os}_${arch}.tar.gz"; then + echo "Failed to download kustomize $version" >&2 + rm -rf "$work" + return 1 + fi + got="$( (sha256sum "$work/kustomize.tar.gz" 2>/dev/null || shasum -a 256 "$work/kustomize.tar.gz") | cut -d' ' -f1)" + if [ "$got" != "$sum" ]; then + echo "The kustomize $version download doesn't match its published sha256 ($got)" >&2 + rm -rf "$work" + return 1 + fi + if ! tar -xzf "$work/kustomize.tar.gz" -C "$work" kustomize || ! mv "$work/kustomize" "$dir/kustomize"; then + echo "Could not unpack kustomize $version into $dir" >&2 + rm -rf "$work" + return 1 fi -} \ No newline at end of file + chmod +x "$dir/kustomize" + rm -rf "$work" +}