From 615e408d69ec7916b8170bd1b58c772e7ddad2cb Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 16:38:38 -0500 Subject: [PATCH 1/7] test: gateway tests reach the node on an address that isn't its InternalIP kube-proxy answers the node's InternalIP itself, so the tests never went through ServiceLB, which masquerades every other address. The new servicelb upgrade test and the unit tests cover Traefik binding the host's ports. --- .github/workflows/gateway.yml | 6 ++ tests/gateway/lib.sh | 17 +++- tests/gateway/servicelb_upgrade_test.sh | 110 ++++++++++++++++++++++++ tests/scripts/stub/kubectl | 9 +- tests/scripts/traefik_install_test.sh | 53 ++++++++++++ 5 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 tests/gateway/servicelb_upgrade_test.sh diff --git a/.github/workflows/gateway.yml b/.github/workflows/gateway.yml index 7cf8de2..aba9fd9 100644 --- a/.github/workflows/gateway.yml +++ b/.github/workflows/gateway.yml @@ -64,6 +64,12 @@ jobs: - uses: actions/checkout@v4 - run: bash tests/gateway/upgrade_test.sh + upgrade-from-servicelb: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash tests/gateway/servicelb_upgrade_test.sh + cert-manager: runs-on: ubuntu-latest steps: diff --git a/tests/gateway/lib.sh b/tests/gateway/lib.sh index 43fd7d5..2dd0d5f 100644 --- a/tests/gateway/lib.sh +++ b/tests/gateway/lib.sh @@ -2,7 +2,7 @@ # k3s in Docker, with the panel's routing layer installed by the same functions # update.sh calls, in front of stub backends named like the real Services. -# Requests come from a separate client container on the same Docker network, +# Requests come from a separate client container on a second Docker network, # so the client IP the backends see is a real peer's. One cluster at a time; # everything is named after $PREFIX and removed on exit. @@ -22,7 +22,7 @@ export KUBECONFIG PANEL_DIR DEBUG cleanup() { docker rm -f -v "$PREFIX-k3s" "$PREFIX-client" >/dev/null 2>&1 - docker network rm "$PREFIX-net" >/dev/null 2>&1 + docker network rm "$PREFIX-net" "$PREFIX-public" >/dev/null 2>&1 rm -rf "$WORK_DIR" } @@ -70,13 +70,22 @@ start_cluster() { sleep 2 done kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null || return 1 - NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-k3s")" docker run -d --name "$PREFIX-client" --network "$PREFIX-net" "$PYTHON_IMAGE" sleep infinity >/dev/null || return 1 client apk add --no-cache -q curl jq openssl >/dev/null || return 1 docker cp "$HERE/client.py" "$PREFIX-client:/client.py" >/dev/null + + # Clients reach the node on an address k3s doesn't know as its InternalIP, + # as a server's public one is when k3s runs on a private or VPN address. + # kube-proxy answers the InternalIP itself, which hid that ServiceLB + # masquerades everything else. + docker network create "$PREFIX-public" >/dev/null || return 1 + docker network connect "$PREFIX-public" "$PREFIX-k3s" || return 1 + docker network connect "$PREFIX-public" "$PREFIX-client" || return 1 + NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-k3s")" + [ "$NODE_IP" != "$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" ] || return 1 # shellcheck disable=SC2034 - CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-client")" + CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-client")" } set_env() { diff --git a/tests/gateway/servicelb_upgrade_test.sh b/tests/gateway/servicelb_upgrade_test.sh new file mode 100644 index 0000000..556a60d --- /dev/null +++ b/tests/gateway/servicelb_upgrade_test.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# An install whose Traefik still sits behind k3s ServiceLB, as the panel ran it +# before Traefik bound the host's ports itself, taken through update.sh's +# install_traefik. + +# shellcheck source=tests/gateway/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +trap cleanup EXIT + +API="api.$DOMAIN" +PORTS_ENV="$PANEL_DIR/overlays/config/traefik-config.env" + +install_traefik_behind_servicelb() { + local crd + install_helm_chart gateway-crds "$PANEL_DIR/charts/gateway-crds" "" >/dev/null || return 1 + for crd in gatewayclasses.gateway.networking.k8s.io gateways.gateway.networking.k8s.io \ + httproutes.gateway.networking.k8s.io middlewares.traefik.io; do + kubectl wait --for=condition=Established "crd/$crd" --timeout=120s >/dev/null || return 1 + done + install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat <<'EOF' +traefik: + service: + spec: {type: LoadBalancer, externalTrafficPolicy: Local} + updateStrategy: + rollingUpdate: {maxUnavailable: 0, maxSurge: 1} +EOF +)" >/dev/null || return 1 + kubectl -n traefik rollout status deployment/traefik --timeout=300s >/dev/null || return 1 + for _ in $(seq 1 90); do + [ -n "$(kubectl -n traefik get service traefik -o jsonpath='{.status.loadBalancer.ingress}')" ] && return 0 + sleep 2 + done + return 1 +} + +traefik_pod() { + kubectl -n traefik get pods -l app.kubernetes.io/name=traefik -o jsonpath='{.items[*].metadata.uid}' +} + +test_behind_servicelb_every_request_comes_from_its_pod() { + wait_for 200 https "$API" /me + case "$(json '.headers["x-forwarded-for"]')" in + 10.42.*) ;; + *) fail "expected a ServiceLB pod address, got $(json '.headers["x-forwarded-for"]')" ;; + esac +} + +test_the_move_costs_seconds() { + local longest + longest="$(jq -r .longest_outage <<<"$POLL")" + echo " outage while Traefik moved: ${longest}s ($(jq -r '"\(.failed) of \(.requests)"' <<<"$POLL") requests failed)" + awk -v t="$longest" 'BEGIN { exit !(t < 30) }' || fail "$POLL" +} + +test_servicelb_no_longer_fronts_traefik() { + [ "$(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')" = ClusterIP ] || fail "Traefik's Service is still $(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')" + [ -z "$(kubectl -n kube-system get daemonset -l svccontroller.k3s.cattle.io/svcname=traefik -o name)" ] || fail "svclb-traefik is still running" +} + +test_the_client_ip_reaches_the_backend() { + call https "$API" /me + expect_code 200 + expect_json '.headers["x-forwarded-for"]' "$CLIENT_IP" + expect_json '.headers["x-real-ip"]' "$CLIENT_IP" + call http "$API" /me + expect_code 308 +} + +test_running_the_update_again_changes_nothing() { + local before + before="$(traefik_pod)" + install_traefik "$PORTS_ENV" >/dev/null || fail "install_traefik failed" + move_traefik_to_host_ports "$PORTS_ENV" >/dev/null || fail "move_traefik_to_host_ports failed" + [ "$(traefik_pod)" = "$before" ] || fail "Traefik was restarted" + call https "$API" /me + expect_code 200 +} + +start_cluster || exit 1 +make_panel || exit 1 +install_traefik_behind_servicelb || exit 1 +deploy_panel_stubs || exit 1 +routing_layer https || exit 1 +tls_secret 5stack-ssl "$DOMAIN" "$API" "demos.$DOMAIN" "ws.$DOMAIN" "tv.$DOMAIN" "search.$DOMAIN" "console.$DOMAIN" || exit 1 +tls_secret 5stack-mediamtx-ssl "hls.$DOMAIN" || exit 1 + +run_test test_behind_servicelb_every_request_comes_from_its_pod + +client rm -f /tmp/stop +docker exec -d "$PREFIX-client" sh -c "python3 /client.py poll $NODE_IP $API /me /tmp/stop >/tmp/poll.json" +sleep 2 +install_traefik "$PORTS_ENV" || exit 1 +sleep 5 +client touch /tmp/stop +for _ in $(seq 1 10); do + client test -s /tmp/poll.json && break + sleep 1 +done +POLL="$(client cat /tmp/poll.json)" + +for name in \ + test_the_move_costs_seconds \ + test_servicelb_no_longer_fronts_traefik \ + test_the_client_ip_reaches_the_backend \ + test_running_the_update_again_changes_nothing; do + run_test "$name" +done + +summary diff --git a/tests/scripts/stub/kubectl b/tests/scripts/stub/kubectl index 76e019d..450f482 100755 --- a/tests/scripts/stub/kubectl +++ b/tests/scripts/stub/kubectl @@ -3,6 +3,7 @@ echo "$*" >>"$WORK_DIR/calls" args=" $* " +file="" pod="" container="" previous=false @@ -14,13 +15,19 @@ while [ $# -gt 0 ]; do shift ;; --previous) previous=true ;; + -f) + file="$2" + shift + ;; esac shift done case "$args" in - *" get crd helmcharts.helm.cattle.io "* | *" apply --server-side "*) ;; + *" get crd helmcharts.helm.cattle.io "*) ;; + *" apply --server-side "*) if [ -n "$file" ]; then cat "$file" >>"$WORK_DIR/server-side-applied"; fi ;; *" apply -f - "*) cat >"$WORK_DIR/applied" ;; + *" get service traefik "*'{.spec.type}'*) if [ -f "$WORK_DIR/traefik-type" ]; then cat "$WORK_DIR/traefik-type"; fi ;; *" get service traefik "*) if [ -f "$WORK_DIR/traefik-address" ]; then cat "$WORK_DIR/traefik-address"; fi ;; *" get helmchart "*) echo 1 ;; *" get gateway traefik-gateway "*) [ -f "$WORK_DIR/shared-gateway" ] || exit 1 ;; diff --git a/tests/scripts/traefik_install_test.sh b/tests/scripts/traefik_install_test.sh index a0c28b7..dd8d9f0 100644 --- a/tests/scripts/traefik_install_test.sh +++ b/tests/scripts/traefik_install_test.sh @@ -263,4 +263,57 @@ test_dev_retires_without_waiting_for_routes() { assert_not_contains "$(cat "$WORK_DIR/calls")" "wait httproute/api" } +test_retiring_with_no_ingress_nginx_waits_for_no_address() { + run_retire || fail "retire_ingress_nginx failed: $(cat "$WORK_DIR/output")" + assert_not_contains "$(cat "$WORK_DIR/calls")" "loadBalancer.ingress" +} + +run_install() { + ( + export PANEL_DIR="$TEST_ROOT" + # shellcheck source=utils/colors.sh + source "$TEST_ROOT/utils/colors.sh" + # shellcheck source=utils/output_redirect.sh + source "$TEST_ROOT/utils/output_redirect.sh" + # shellcheck source=utils/update_env_var.sh + source "$TEST_ROOT/utils/update_env_var.sh" + # shellcheck source=utils/install_traefik.sh + source "$TEST_ROOT/utils/install_traefik.sh" + "$@" "$WORK_DIR/traefik-config.env" + ) >"$WORK_DIR/output" 2>&1 || fail "$1 exited $?: $(cat "$WORK_DIR/output")" +} + +gateway_values() { + sed -n '/^ name: gateway$/,$p' "$WORK_DIR/server-side-applied" 2>/dev/null | sed -n '/valuesContent/,$p' +} + +test_traefik_binds_the_host_ports_itself() { + ports_file HTTP_PORT=8080 HTTPS_PORT=8443 + run_install install_traefik + assert_contains "$(gateway_values)" "web: {exposedPort: 8080, hostPort: 8080}" + assert_contains "$(gateway_values)" "websecure: {exposedPort: 8443, hostPort: 8443}" + assert_not_contains "$(gateway_values)" "LoadBalancer" +} + +test_traefik_waits_behind_servicelb_while_ingress_nginx_holds_the_ports() { + nginx_ports 80 443 + ports_file HTTP_PORT=80 HTTPS_PORT=443 + run_install install_traefik + assert_contains "$(gateway_values)" "spec: {type: LoadBalancer, externalTrafficPolicy: Local}" + assert_not_contains "$(gateway_values)" "hostPort" +} + +test_traefik_left_behind_servicelb_moves_onto_the_host_ports() { + echo LoadBalancer >"$WORK_DIR/traefik-type" + ports_file HTTP_PORT=80 HTTPS_PORT=443 + run_install move_traefik_to_host_ports + assert_contains "$(gateway_values)" "web: {exposedPort: 80, hostPort: 80}" +} + +test_traefik_already_on_the_host_ports_is_not_reinstalled() { + echo ClusterIP >"$WORK_DIR/traefik-type" + run_install move_traefik_to_host_ports + [ ! -f "$WORK_DIR/server-side-applied" ] || fail "the gateway chart was applied again" +} + run_tests From 54fe62a881ac50fb9f971ad562d5017e7aee17a7 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 16:38:38 -0500 Subject: [PATCH 2/7] fix: Traefik binds the host's ports itself, so backends see the client's address k3s ServiceLB masquerades, so behind it every request came from its pod whenever traffic arrived on an address other than the node's InternalIP. --- charts/gateway/values.yaml | 11 ++++++++-- dev.sh | 1 + tests/gateway/upgrade_test.sh | 5 ++++- update.sh | 1 + utils/install_traefik.sh | 38 ++++++++++++++++++++++++++--------- utils/retire_ingress_nginx.sh | 29 +++++++++++++------------- 6 files changed, 58 insertions(+), 27 deletions(-) diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index ccb8c6f..7cf3817 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -80,10 +80,17 @@ traefik: readTimeout: 0 writeTimeout: 0 + # install_traefik binds the host's ports straight to the pod (hostPort): + # k3s ServiceLB masquerades, so behind it every request came from its pod. service: spec: - type: LoadBalancer - externalTrafficPolicy: Local + type: ClusterIP + + # Old and new pods can't hold the same host port at once. + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 resources: requests: diff --git a/dev.sh b/dev.sh index d31fce0..cb50854 100755 --- a/dev.sh +++ b/dev.sh @@ -58,6 +58,7 @@ kubectl config use-context k3d-5stack-dev install_traefik "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1 retire_ingress_nginx --immediately || exit 1 +move_traefik_to_host_ports "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1 echo "Labeling node..." kubectl label node $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') 5stack-api=true 5stack-hasura=true 5stack-minio=true 5stack-rustfs=true 5stack-timescaledb=true 5stack-redis=true 5stack-typesense=true 5stack-web=true 5stack-dev-server=true 5stack-mediamtx=true diff --git a/tests/gateway/upgrade_test.sh b/tests/gateway/upgrade_test.sh index 0ab0f7e..b28ba0a 100644 --- a/tests/gateway/upgrade_test.sh +++ b/tests/gateway/upgrade_test.sh @@ -3,7 +3,8 @@ # An install that is still on ingress-nginx -- the controller, its admission # webhook, the panel's Ingresses as they were, and another app's nginx-class # Ingress with forward-auth -- taken through update.sh's routing steps: -# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx. +# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx, +# move_traefik_to_host_ports. # shellcheck source=tests/gateway/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" @@ -200,6 +201,7 @@ test_running_the_update_again_changes_nothing() { install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "install_traefik failed" routing_layer https || fail "the overlay did not apply" retire_ingress_nginx || fail "retire_ingress_nginx failed" + move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "move_traefik_to_host_ports failed" call https "$API" /me expect_code 200 expect_other_app_routes_with_its_auth @@ -222,6 +224,7 @@ sleep 2 install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1 routing_layer https || exit 1 retire_ingress_nginx || exit 1 +move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1 sleep 5 client touch /tmp/stop for _ in $(seq 1 10); do diff --git a/update.sh b/update.sh index 9e71aad..b42dde5 100755 --- a/update.sh +++ b/update.sh @@ -117,6 +117,7 @@ if [ "$REVERSE_PROXY" != true ]; then fi retire_ingress_nginx || die "Traefik did not take over ports 80/443" +move_traefik_to_host_ports || die "Traefik did not take over the host's ports; it is still serving behind ServiceLB" if [ "$VAULT_MANAGER" = true ]; then step "Syncing Vault secrets" diff --git a/utils/install_traefik.sh b/utils/install_traefik.sh index c8b22e3..94b8e31 100644 --- a/utils/install_traefik.sh +++ b/utils/install_traefik.sh @@ -39,7 +39,7 @@ carry_over_ingress_nginx_ports() { install_traefik() { local ports_env="${1:-$PANEL_DIR/overlays/config/traefik-config.env}" - local minor http_port https_port crd + local minor http_port https_port crd binding local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -73,16 +73,26 @@ install_traefik() { fi done - # The Service is externalTrafficPolicy: Local, so only the node running - # Traefik answers on these ports -- and DNS points at the server node, not - # at a game node that joined later. + # While ingress-nginx's ServiceLB pods hold the ports, Traefik waits behind + # its own until retire_ingress_nginx hands them over, so a stalled handover + # can still give them back; move_traefik_to_host_ports finishes the move. + if kubectl "${kc[@]}" -n ingress-nginx get service ingress-nginx-controller >/dev/null 2>&1; then + binding=" ports: + web: {exposedPort: $http_port} + websecure: {exposedPort: $https_port} + service: + spec: {type: LoadBalancer, externalTrafficPolicy: Local}" + else + binding=" ports: + web: {exposedPort: $http_port, hostPort: $http_port} + websecure: {exposedPort: $https_port, hostPort: $https_port}" + fi + + # Only the node running Traefik listens on these ports, and DNS points at + # the server node, not at a game node that joined later. install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat </dev/null)" != LoadBalancer ]; then + return 0 + fi + install_traefik "$@" +} + # The cluster-scoped objects a chart's templates define, as kind/name. chart_objects() { awk '/^kind: / { kind = $2; next } diff --git a/utils/retire_ingress_nginx.sh b/utils/retire_ingress_nginx.sh index 1e42e78..18990d2 100644 --- a/utils/retire_ingress_nginx.sh +++ b/utils/retire_ingress_nginx.sh @@ -48,7 +48,7 @@ YAML # retires ingress-nginx before Tilt has applied them, and a dev cluster has no # visitors to keep serving. retire_ingress_nginx() { - local deadline moved="" http_port="" https_port="" + local deadline http_port="" https_port="" local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -70,22 +70,21 @@ retire_ingress_nginx() { -o jsonpath='{.spec.ports[?(@.name=="https")].port}' 2>/dev/null)" echo "Moving ports 80/443 from ingress-nginx to Traefik..." output_redirect kubectl "${kc[@]}" delete service ingress-nginx-controller -n ingress-nginx --wait=true - moved=1 - fi - deadline=$((SECONDS + ${TRAEFIK_ADDRESS_TIMEOUT:-180})) - until [ -n "$(kubectl "${kc[@]}" get service traefik -n traefik -o jsonpath='{.status.loadBalancer.ingress}' 2>/dev/null)" ]; do - if [ "$SECONDS" -ge "$deadline" ]; then - err "Traefik's LoadBalancer Service never got an address:" - kubectl "${kc[@]}" get service traefik -n traefik >&2 - kubectl "${kc[@]}" get pods -A -l svccontroller.k3s.cattle.io/svcname=traefik >&2 - if [ -n "$moved" ] && restore_ingress_nginx_service "${http_port:-80}" "${https_port:-443}"; then - err "ingress-nginx has its ports back and keeps serving the panel; re-run ./update.sh once Traefik's Service can get an address" + deadline=$((SECONDS + ${TRAEFIK_ADDRESS_TIMEOUT:-180})) + until [ -n "$(kubectl "${kc[@]}" get service traefik -n traefik -o jsonpath='{.status.loadBalancer.ingress}' 2>/dev/null)" ]; do + if [ "$SECONDS" -ge "$deadline" ]; then + err "Traefik's LoadBalancer Service never got an address:" + kubectl "${kc[@]}" get service traefik -n traefik >&2 + kubectl "${kc[@]}" get pods -A -l svccontroller.k3s.cattle.io/svcname=traefik >&2 + if restore_ingress_nginx_service "${http_port:-80}" "${https_port:-443}"; then + err "ingress-nginx has its ports back and keeps serving the panel; re-run ./update.sh once Traefik's Service can get an address" + fi + return 1 fi - return 1 - fi - sleep 2 - done + sleep 2 + done + fi # Orphaned, not cascaded: on an old install ingress-shim may still own the # 5stack-ssl Certificate through one of these. From c177a9b826c629c5793dc49c61dc9cf5bc05df8a Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 16:58:42 -0500 Subject: [PATCH 3/7] fix: pull a new Traefik image onto its node before the rollout replaces the pod With host ports the old pod stops before its replacement exists, so a slow or refused pull would leave nothing serving. Failed kubectl lookups now stop the install instead of being read as answers. --- base/gateway/gateway.yaml | 2 +- tests/gateway/servicelb_upgrade_test.sh | 18 ++++++-- tests/scripts/stub/kubectl | 18 +++++++- tests/scripts/traefik_install_test.sh | 59 ++++++++++++++++++++++++- update.sh | 2 +- utils/install_traefik.sh | 42 ++++++++++++++++-- 6 files changed, 130 insertions(+), 11 deletions(-) diff --git a/base/gateway/gateway.yaml b/base/gateway/gateway.yaml index 0816337..ea15725 100644 --- a/base/gateway/gateway.yaml +++ b/base/gateway/gateway.yaml @@ -2,7 +2,7 @@ # inherit the hostname from. These are the plain-HTTP listeners of a # REVERSE_PROXY install; overlays/http/gateway-https-patch.yaml turns them into # HTTPS ones. Ports are Traefik's entrypoints (charts/gateway), not the node's: -# HTTP_PORT and HTTPS_PORT only move the LoadBalancer Service. The listeners for +# HTTP_PORT and HTTPS_PORT only move Traefik's host ports. The listeners for # every other host are on the chart's shared Gateway, traefik/traefik-gateway: # web (HTTP) and websecure (HTTPS on a self-signed certificate). apiVersion: gateway.networking.k8s.io/v1 diff --git a/tests/gateway/servicelb_upgrade_test.sh b/tests/gateway/servicelb_upgrade_test.sh index 556a60d..6f1d6c6 100644 --- a/tests/gateway/servicelb_upgrade_test.sh +++ b/tests/gateway/servicelb_upgrade_test.sh @@ -1,8 +1,8 @@ #!/bin/bash -# An install whose Traefik still sits behind k3s ServiceLB, as the panel ran it -# before Traefik bound the host's ports itself, taken through update.sh's -# install_traefik. +# An install whose Traefik still sits behind k3s ServiceLB on an older image, as +# the panel ran it before Traefik bound the host's ports itself, taken through +# update.sh's install_traefik. # shellcheck source=tests/gateway/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" @@ -20,6 +20,9 @@ install_traefik_behind_servicelb() { done install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat <<'EOF' traefik: + versionOverride: v3.7.15 + image: + digest: sha256:0f6150202bc1fed72a7bd2107460eaa510e5b8a905f3fdb5b84f2c8619c33368 service: spec: {type: LoadBalancer, externalTrafficPolicy: Local} updateStrategy: @@ -58,6 +61,14 @@ test_servicelb_no_longer_fronts_traefik() { [ -z "$(kubectl -n kube-system get daemonset -l svccontroller.k3s.cattle.io/svcname=traefik -o name)" ] || fail "svclb-traefik is still running" } +test_traefik_runs_the_new_image_pulled_beforehand() { + local want + want="docker.io/traefik@$(sed -n 's/^ *digest: *\(sha256:[0-9a-f]*\).*/\1/p' "$PANEL_DIR/charts/gateway/values.yaml")" + [ "$(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')" = "$want" ] || + fail "Traefik runs $(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')" + kubectl -n traefik wait --for=delete pod/traefik-pull --timeout=60s >/dev/null 2>&1 || fail "the pull pod was left behind" +} + test_the_client_ip_reaches_the_backend() { call https "$API" /me expect_code 200 @@ -102,6 +113,7 @@ POLL="$(client cat /tmp/poll.json)" for name in \ test_the_move_costs_seconds \ test_servicelb_no_longer_fronts_traefik \ + test_traefik_runs_the_new_image_pulled_beforehand \ test_the_client_ip_reaches_the_backend \ test_running_the_update_again_changes_nothing; do run_test "$name" diff --git a/tests/scripts/stub/kubectl b/tests/scripts/stub/kubectl index 450f482..4859f10 100755 --- a/tests/scripts/stub/kubectl +++ b/tests/scripts/stub/kubectl @@ -27,7 +27,16 @@ case "$args" in *" get crd helmcharts.helm.cattle.io "*) ;; *" apply --server-side "*) if [ -n "$file" ]; then cat "$file" >>"$WORK_DIR/server-side-applied"; fi ;; *" apply -f - "*) cat >"$WORK_DIR/applied" ;; - *" get service traefik "*'{.spec.type}'*) if [ -f "$WORK_DIR/traefik-type" ]; then cat "$WORK_DIR/traefik-type"; fi ;; + *" get service traefik "*'{.spec.type}'*) + if [ -f "$WORK_DIR/traefik-service-error" ]; then + echo "The connection to the server was refused" >&2 + exit 1 + fi + if [ -f "$WORK_DIR/traefik-type" ]; then cat "$WORK_DIR/traefik-type"; fi + ;; + *" get deployment traefik "*) if [ -f "$WORK_DIR/traefik-image" ]; then cat "$WORK_DIR/traefik-image"; fi ;; + *" get pods -l app.kubernetes.io/name=traefik "*) if [ -f "$WORK_DIR/traefik-node" ]; then cat "$WORK_DIR/traefik-node"; fi ;; + *" wait pod/traefik-pull "*) [ ! -f "$WORK_DIR/pull-fails" ] || exit 1 ;; *" get service traefik "*) if [ -f "$WORK_DIR/traefik-address" ]; then cat "$WORK_DIR/traefik-address"; fi ;; *" get helmchart "*) echo 1 ;; *" get gateway traefik-gateway "*) [ -f "$WORK_DIR/shared-gateway" ] || exit 1 ;; @@ -83,7 +92,14 @@ case "$args" in *'name=="https"'*) port_file="$WORK_DIR/nginx-https-port" ;; *) port_file="$WORK_DIR/nginx-http-port" ;; esac + if [ -f "$WORK_DIR/nginx-error" ]; then + echo "The connection to the server was refused" >&2 + exit 1 + fi if [ ! -f "$port_file" ]; then + case "$args" in + *" --ignore-not-found "*) exit 0 ;; + esac echo 'Error from server (NotFound): services "ingress-nginx-controller" not found' >&2 exit 1 fi diff --git a/tests/scripts/traefik_install_test.sh b/tests/scripts/traefik_install_test.sh index dd8d9f0..123b349 100644 --- a/tests/scripts/traefik_install_test.sh +++ b/tests/scripts/traefik_install_test.sh @@ -268,7 +268,7 @@ test_retiring_with_no_ingress_nginx_waits_for_no_address() { assert_not_contains "$(cat "$WORK_DIR/calls")" "loadBalancer.ingress" } -run_install() { +try_install() { ( export PANEL_DIR="$TEST_ROOT" # shellcheck source=utils/colors.sh @@ -280,7 +280,15 @@ run_install() { # shellcheck source=utils/install_traefik.sh source "$TEST_ROOT/utils/install_traefik.sh" "$@" "$WORK_DIR/traefik-config.env" - ) >"$WORK_DIR/output" 2>&1 || fail "$1 exited $?: $(cat "$WORK_DIR/output")" + ) >"$WORK_DIR/output" 2>&1 +} + +run_install() { + try_install "$@" || fail "$1 exited $?: $(cat "$WORK_DIR/output")" +} + +chart_image() { + printf 'docker.io/traefik@%s' "$(sed -n 's/^ *digest: *\(sha256:[0-9a-f]*\).*/\1/p' "$TEST_ROOT/charts/gateway/values.yaml")" } gateway_values() { @@ -316,4 +324,51 @@ test_traefik_already_on_the_host_ports_is_not_reinstalled() { [ ! -f "$WORK_DIR/server-side-applied" ] || fail "the gateway chart was applied again" } +test_a_failed_service_lookup_fails_the_move() { + touch "$WORK_DIR/traefik-service-error" + if try_install move_traefik_to_host_ports; then + fail "move_traefik_to_host_ports succeeded without knowing where Traefik is" + fi +} + +test_a_failed_ingress_nginx_lookup_stops_the_install() { + touch "$WORK_DIR/nginx-error" + if try_install install_traefik; then + fail "install_traefik succeeded without knowing whether ingress-nginx holds the ports" + fi + assert_eq "$(gateway_values)" "" +} + +test_a_new_image_is_pulled_onto_traefiks_node_before_the_upgrade() { + echo docker.io/traefik@sha256:0000 >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + run_install install_traefik + assert_contains "$(grep ' run traefik-pull ' "$WORK_DIR/calls")" "--image=$(chart_image) --restart=Never --overrides={\"spec\":{\"nodeName\":\"node-a\"" + [ "$(grep -n ' run traefik-pull ' "$WORK_DIR/calls" | cut -d: -f1)" -lt "$(grep -n ' apply --server-side ' "$WORK_DIR/calls" | tail -n 1 | cut -d: -f1)" ] || + fail "the image was pulled after the upgrade" +} + +test_a_refused_pull_leaves_traefik_alone() { + echo docker.io/traefik@sha256:0000 >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + touch "$WORK_DIR/pull-fails" + if try_install install_traefik; then + fail "install_traefik succeeded although the image never arrived" + fi + assert_eq "$(gateway_values)" "" + assert_contains "$(cat "$WORK_DIR/output")" "keeps running its current image" +} + +test_traefik_already_on_the_image_pulls_nothing() { + chart_image >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + run_install install_traefik + assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" +} + +test_a_first_install_pulls_nothing() { + run_install install_traefik + assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" +} + run_tests diff --git a/update.sh b/update.sh index b42dde5..f963857 100755 --- a/update.sh +++ b/update.sh @@ -117,7 +117,7 @@ if [ "$REVERSE_PROXY" != true ]; then fi retire_ingress_nginx || die "Traefik did not take over ports 80/443" -move_traefik_to_host_ports || die "Traefik did not take over the host's ports; it is still serving behind ServiceLB" +move_traefik_to_host_ports || die "Traefik did not take over the host's ports" if [ "$VAULT_MANAGER" = true ]; then step "Syncing Vault secrets" diff --git a/utils/install_traefik.sh b/utils/install_traefik.sh index 94b8e31..23d8145 100644 --- a/utils/install_traefik.sh +++ b/utils/install_traefik.sh @@ -39,7 +39,7 @@ carry_over_ingress_nginx_ports() { install_traefik() { local ports_env="${1:-$PANEL_DIR/overlays/config/traefik-config.env}" - local minor http_port https_port crd binding + local minor http_port https_port crd binding nginx local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -76,7 +76,8 @@ install_traefik() { # While ingress-nginx's ServiceLB pods hold the ports, Traefik waits behind # its own until retire_ingress_nginx hands them over, so a stalled handover # can still give them back; move_traefik_to_host_ports finishes the move. - if kubectl "${kc[@]}" -n ingress-nginx get service ingress-nginx-controller >/dev/null 2>&1; then + nginx="$(kubectl "${kc[@]}" -n ingress-nginx get service ingress-nginx-controller --ignore-not-found -o name)" || return 1 + if [ -n "$nginx" ]; then binding=" ports: web: {exposedPort: $http_port} websecure: {exposedPort: $https_port} @@ -88,6 +89,8 @@ install_traefik() { websecure: {exposedPort: $https_port, hostPort: $https_port}" fi + pull_traefik_image || return 1 + # Only the node running Traefik listens on these ports, and DNS points at # the server node, not at a game node that joined later. install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat </dev/null)" = "$image" ]; then + return 0 + fi + node="$(kubectl "${kc[@]}" -n traefik get pods -l app.kubernetes.io/name=traefik -o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null)" + if [ -z "$node" ]; then + return 0 + fi + + echo "Pulling $image onto $node..." + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --ignore-not-found --wait=true + if ! output_redirect kubectl "${kc[@]}" -n traefik run traefik-pull --image="$image" --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"$node\",\"tolerations\":[{\"operator\":\"Exists\"}]}}" \ + --command -- traefik version || + ! output_redirect kubectl "${kc[@]}" -n traefik wait pod/traefik-pull --timeout=300s \ + --for=jsonpath='{.status.phase}'=Succeeded; then + err "could not pull $image onto $node; Traefik keeps running its current image" + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --ignore-not-found --wait=false + return 1 + fi + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --wait=false +} + move_traefik_to_host_ports() { + local type local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") - if [ "$(kubectl "${kc[@]}" -n traefik get service traefik -o jsonpath='{.spec.type}' 2>/dev/null)" != LoadBalancer ]; then + type="$(kubectl "${kc[@]}" -n traefik get service traefik -o jsonpath='{.spec.type}')" || return 1 + if [ "$type" != LoadBalancer ]; then return 0 fi install_traefik "$@" From 8f1fbcf2e16a57402c1f20668d8abb83d38d8f71 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 17:18:27 -0500 Subject: [PATCH 4/7] feature: on k3s 1.37 and later the panel leaves the Gateway API CRDs to k3s's own chart, and hands over the ones it installed before the upgrade --- .github/workflows/gateway.yml | 6 + charts/gateway-crds/Chart.yaml | 4 +- .../gateway-crds/templates/gateway-api.yaml | 7 +- charts/gateway-crds/values.yaml | 4 + tests/gateway/k3s_upgrade_test.sh | 102 ++++++++++++++++ tests/gateway/lib.sh | 58 +++++++--- tests/gateway/upgrade_test.sh | 6 +- tests/scripts/traefik_install_test.sh | 72 +++++++++++- utils/install_traefik.sh | 109 +++++++++++++++--- 9 files changed, 329 insertions(+), 39 deletions(-) create mode 100644 charts/gateway-crds/values.yaml create mode 100644 tests/gateway/k3s_upgrade_test.sh diff --git a/.github/workflows/gateway.yml b/.github/workflows/gateway.yml index aba9fd9..d621532 100644 --- a/.github/workflows/gateway.yml +++ b/.github/workflows/gateway.yml @@ -70,6 +70,12 @@ jobs: - uses: actions/checkout@v4 - run: bash tests/gateway/servicelb_upgrade_test.sh + upgrade-k3s-to-its-own-gateway-api: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash tests/gateway/k3s_upgrade_test.sh + cert-manager: runs-on: ubuntu-latest steps: diff --git a/charts/gateway-crds/Chart.yaml b/charts/gateway-crds/Chart.yaml index 984e129..40044eb 100644 --- a/charts/gateway-crds/Chart.yaml +++ b/charts/gateway-crds/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: gateway-crds -description: The Gateway API (v1.6.3, standard channel) and Traefik v3.7 CRDs that the gateway chart needs. Install it first. +description: The Gateway API (v1.6.3, standard channel) and Traefik v3.7 CRDs that the gateway chart needs. Install it first. Set gatewayApi=false where the cluster brings its own Gateway API CRDs, as k3s does from 1.37. type: application -version: 0.1.0 +version: 0.2.0 appVersion: v1.6.3 kubeVersion: ">=1.31.0-0" diff --git a/charts/gateway-crds/templates/gateway-api.yaml b/charts/gateway-crds/templates/gateway-api.yaml index 37d14c1..ce936d5 100644 --- a/charts/gateway-crds/templates/gateway-api.yaml +++ b/charts/gateway-crds/templates/gateway-api.yaml @@ -1,6 +1,8 @@ +{{- if .Values.gatewayApi }} # Gateway API v1.6.3, standard channel: https://github.com/kubernetes-sigs/gateway-api/releases/download/v1.6.3/standard-install.yaml # (sha256 356c2b8e286a964dc20995435659ad1095386329974793af10bbfe3eff977fe5), with helm.sh/resource-policy: keep -# added to each CRD so an uninstall never takes every HTTPRoute on the cluster with it. +# added to every object here, so neither an uninstall nor gatewayApi: false deletes the +# CRDs, and every HTTPRoute on the cluster with them, or the safe-upgrades policy. # Copyright The Kubernetes Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -20078,6 +20080,7 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicy metadata: annotations: + helm.sh/resource-policy: keep gateway.networking.k8s.io/bundle-version: v1.6.3 gateway.networking.k8s.io/channel: standard name: "safe-upgrades.gateway.networking.k8s.io" @@ -20115,6 +20118,7 @@ apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingAdmissionPolicyBinding metadata: annotations: + helm.sh/resource-policy: keep gateway.networking.k8s.io/bundle-version: v1.6.3 gateway.networking.k8s.io/channel: standard name: safe-upgrades.gateway.networking.k8s.io @@ -20127,3 +20131,4 @@ spec: apiVersions: ["v1"] resources: ["customresourcedefinitions"] operations: ["CREATE", "UPDATE"] +{{- end }} diff --git a/charts/gateway-crds/values.yaml b/charts/gateway-crds/values.yaml new file mode 100644 index 0000000..c6106ab --- /dev/null +++ b/charts/gateway-crds/values.yaml @@ -0,0 +1,4 @@ +# The Gateway API's CRDs and its safe-upgrades admission policy. Turn it off +# where something else installs them, as k3s does from 1.37 with its +# gateway-api-crd chart. Traefik's CRDs are installed either way. +gatewayApi: true diff --git a/tests/gateway/k3s_upgrade_test.sh b/tests/gateway/k3s_upgrade_test.sh new file mode 100644 index 0000000..338b938 --- /dev/null +++ b/tests/gateway/k3s_upgrade_test.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# A panel installed while k3s left the Gateway API CRDs to it, then k3s +# upgraded in place as the panel's k3s updates do it: the same datastore and +# flags on a new binary. From 1.37 k3s's own gateway-api-crd chart installs +# beside gateway-crds' copies, and the next update.sh hands them over to it, +# safe-upgrades policy and all. + +# shellcheck source=tests/gateway/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +trap cleanup EXIT + +API="api.$DOMAIN" + +gateway_api_objects() { + chart_objects "$PANEL_DIR/charts/gateway-crds/templates/gateway-api.yaml" +} + +# release/namespace from an object's Helm marks, or "missing". +owner() { + kubectl get "$1" -o jsonpath='{.metadata.annotations.meta\.helm\.sh/release-name}/{.metadata.annotations.meta\.helm\.sh/release-namespace}' 2>/dev/null || + echo missing +} + +expect_gateway_api_owned_by() { + local object + for object in $(gateway_api_objects); do + [ "$(owner "$object")" = "$1" ] || fail "$object belongs to $(owner "$object"), not $1" + done +} + +update_panel() { + install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" && routing_layer http +} + +# k3s runs its chart's install Job a little after it starts. +k3s_chart_installed() { + for _ in $(seq 1 60); do + [ "$(kubectl -n kube-system get job helm-install-gateway-api-crd -o jsonpath='{.status.succeeded}' 2>/dev/null)" = 1 ] && + return 0 + sleep 3 + done + return 1 +} + +crds_generation() { + kubectl -n kube-system get helmchart gateway-crds -o jsonpath='{.metadata.generation}' +} + +test_gateway_crds_holds_the_gateway_api_before_k3s_does() { + expect_gateway_api_owned_by gateway-crds/traefik + wait_for 200 http "$API" /me +} + +test_the_panel_serves_through_the_k3s_upgrade() { + k3s_chart_installed || fail "$K3S_IMAGE never installed its gateway-api-crd chart" + wait_for 200 http "$API" /me +} + +test_update_hands_the_gateway_api_to_k3s() { + update_panel || fail "update.sh's routing steps failed after the k3s upgrade" + expect_gateway_api_owned_by gateway-api-crd/kube-system + case "$(kubectl -n kube-system get helmchart gateway-crds -o jsonpath='{.spec.valuesContent}')" in + *"gatewayApi: false"*) ;; + *) fail "gateway-crds still installs the Gateway API" ;; + esac + kubectl wait --for=condition=Established crd/middlewares.traefik.io --timeout=10s >/dev/null || + fail "gateway-crds dropped Traefik's CRDs with the Gateway API's" + wait_for 200 http "$API" /me +} + +test_a_restart_keeps_the_handover() { + k3s_chart_installed || fail "k3s didn't run its gateway-api-crd chart again" + expect_gateway_api_owned_by gateway-api-crd/kube-system + wait_for 200 http "$API" /me +} + +test_another_update_changes_nothing() { + local before + before="$(crds_generation)" + update_panel || fail "update.sh's routing steps failed on a second run" + [ "$(crds_generation)" = "$before" ] || fail "a second update changed the gateway-crds HelmChart" + expect_gateway_api_owned_by gateway-api-crd/kube-system + wait_for 200 http "$API" /me +} + +start_cluster "$K3S_BEFORE_GATEWAY_API" || exit 1 +make_panel || exit 1 +update_panel >/dev/null || exit 1 +deploy_panel_stubs || exit 1 +run_test test_gateway_crds_holds_the_gateway_api_before_k3s_does + +run_k3s "$K3S_IMAGE" || exit 1 +run_test test_the_panel_serves_through_the_k3s_upgrade +run_test test_update_hands_the_gateway_api_to_k3s + +# A reboot, or the next k3s patch. +run_k3s "$K3S_IMAGE" || exit 1 +run_test test_a_restart_keeps_the_handover +run_test test_another_update_changes_nothing + +summary diff --git a/tests/gateway/lib.sh b/tests/gateway/lib.sh index 2dd0d5f..ac01114 100644 --- a/tests/gateway/lib.sh +++ b/tests/gateway/lib.sh @@ -9,12 +9,19 @@ HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_ROOT="${TEST_ROOT:-$(cd "$HERE/../.." && pwd)}" PREFIX="${PREFIX:-gateway-test}" -K3S_IMAGE=rancher/k3s:v1.36.5-k3s1@sha256:057cfd5a9d51b89f1f0ee139c28d5d5fdd5aa7ddeac2742a0ba90c6dbc0329bb +K3S_IMAGE=rancher/k3s:v1.37.2-k3s1@sha256:21ea6e0fac4e3930c2ce8aa40ddd41a8b18cb3e9a0222c030dc005e7e88c4332 +# The last k3s that leaves the Gateway API CRDs to whoever installs Traefik; +# from 1.37 its gateway-api-crd chart installs them. Not named *_IMAGE, so +# the image-updates workflow leaves it here while K3S_IMAGE moves on. +# shellcheck disable=SC2034 # for the tests that source this +K3S_BEFORE_GATEWAY_API=rancher/k3s:v1.36.5-k3s1@sha256:057cfd5a9d51b89f1f0ee139c28d5d5fdd5aa7ddeac2742a0ba90c6dbc0329bb PYTHON_IMAGE=python:3.13.18-alpine3.24@sha256:bbc0290146ebe0a8b9a8ed07d3f2c7bdeb4d917fc93300a4e47490c8b5a11b48 DOMAIN=5stack.test WORK_DIR="$(mktemp -d "${TMPDIR:-/tmp}/gateway-test.XXXXXX")" PANEL_DIR="$WORK_DIR/panel" KUBECONFIG="$WORK_DIR/kubeconfig" +# k3s keeps its datastore here, so run_k3s can bring the same cluster back. +K3S_STATE="$PREFIX-k3s-state" DEBUG=false TESTS_PASSED=0 TESTS_FAILED=0 @@ -23,6 +30,7 @@ export KUBECONFIG PANEL_DIR DEBUG cleanup() { docker rm -f -v "$PREFIX-k3s" "$PREFIX-client" >/dev/null 2>&1 docker network rm "$PREFIX-net" "$PREFIX-public" >/dev/null 2>&1 + docker volume rm "$K3S_STATE" >/dev/null 2>&1 rm -rf "$WORK_DIR" } @@ -51,13 +59,38 @@ client() { docker exec "$PREFIX-client" "$@" } +# A cluster on k3s $1 (K3S_IMAGE by default) with the client container. start_cluster() { - local port cleanup mkdir -p "$WORK_DIR" docker network create "$PREFIX-net" >/dev/null || return 1 + # Clients reach the node on an address k3s doesn't know as its InternalIP, + # as a server's public one is when k3s runs on a private or VPN address. + # kube-proxy answers the InternalIP itself, which hid that ServiceLB + # masquerades everything else. + docker network create "$PREFIX-public" >/dev/null || return 1 + run_k3s "${1:-$K3S_IMAGE}" || return 1 + + docker run -d --name "$PREFIX-client" --network "$PREFIX-net" "$PYTHON_IMAGE" sleep infinity >/dev/null || return 1 + client apk add --no-cache -q curl jq openssl >/dev/null || return 1 + docker cp "$HERE/client.py" "$PREFIX-client:/client.py" >/dev/null + docker network connect "$PREFIX-public" "$PREFIX-client" || return 1 + # shellcheck disable=SC2034 + CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-client")" +} + +# (Re)starts the k3s server on the cluster's state volume. On another image +# that is the in-place upgrade the panel's k3s updates make with +# rancher/k3s-upgrade: the same datastore and flags on a new binary. Waits for +# the new kubelet itself, since the Node the old one left still says Ready. +run_k3s() { + local image="$1" version port + version="${image%%@*}" + version="${version##*:}" + version="${version/-k3s/+k3s}" + docker rm -f "$PREFIX-k3s" >/dev/null 2>&1 docker run -d --name "$PREFIX-k3s" --hostname "$PREFIX-k3s" --network "$PREFIX-net" --privileged \ - --tmpfs /run --tmpfs /var/run -p 127.0.0.1::6443 "$K3S_IMAGE" server \ + --tmpfs /run --tmpfs /var/run -v "$K3S_STATE:/var/lib/rancher/k3s" -p 127.0.0.1::6443 "$image" server \ --disable=traefik --disable=metrics-server --disable=local-storage --tls-san 127.0.0.1 >/dev/null || return 1 port="$(docker port "$PREFIX-k3s" 6443/tcp | head -n 1 | sed 's/.*://')" for _ in $(seq 1 90); do @@ -66,26 +99,19 @@ start_cluster() { done docker exec "$PREFIX-k3s" cat /etc/rancher/k3s/k3s.yaml | sed "s#https://127.0.0.1:6443#https://127.0.0.1:$port#" >"$KUBECONFIG" for _ in $(seq 1 90); do - kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null 2>&1 && break + if [ "$(kubectl get node "$PREFIX-k3s" -o jsonpath='{.status.nodeInfo.kubeletVersion}' 2>/dev/null)" = "$version" ] && + [ "$(kubectl get --raw "/api/v1/nodes/$PREFIX-k3s/proxy/healthz" 2>/dev/null)" = ok ] && + kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null 2>&1; then + break + fi sleep 2 done kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null || return 1 + [ "$(kubectl get node "$PREFIX-k3s" -o jsonpath='{.status.nodeInfo.kubeletVersion}')" = "$version" ] || return 1 - docker run -d --name "$PREFIX-client" --network "$PREFIX-net" "$PYTHON_IMAGE" sleep infinity >/dev/null || return 1 - client apk add --no-cache -q curl jq openssl >/dev/null || return 1 - docker cp "$HERE/client.py" "$PREFIX-client:/client.py" >/dev/null - - # Clients reach the node on an address k3s doesn't know as its InternalIP, - # as a server's public one is when k3s runs on a private or VPN address. - # kube-proxy answers the InternalIP itself, which hid that ServiceLB - # masquerades everything else. - docker network create "$PREFIX-public" >/dev/null || return 1 docker network connect "$PREFIX-public" "$PREFIX-k3s" || return 1 - docker network connect "$PREFIX-public" "$PREFIX-client" || return 1 NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-k3s")" [ "$NODE_IP" != "$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" ] || return 1 - # shellcheck disable=SC2034 - CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-client")" } set_env() { diff --git a/tests/gateway/upgrade_test.sh b/tests/gateway/upgrade_test.sh index b28ba0a..1dca1bd 100644 --- a/tests/gateway/upgrade_test.sh +++ b/tests/gateway/upgrade_test.sh @@ -39,7 +39,7 @@ install_ingress_nginx() { # traefik-crd release, which is gone. leave_k3s_traefik_crds() { local -a crds - kubectl apply --server-side -f "$PANEL_DIR/charts/gateway-crds/templates/gateway-api.yaml" >/dev/null || return 1 + grep -v "^{{" "$PANEL_DIR/charts/gateway-crds/templates/gateway-api.yaml" | kubectl apply --server-side -f - >/dev/null || return 1 kubectl delete validatingadmissionpolicybinding,validatingadmissionpolicy safe-upgrades.gateway.networking.k8s.io >/dev/null || return 1 mapfile -t crds < <(kubectl get crd -o name | grep '\.gateway\.networking\.k8s\.io$') kubectl annotate --overwrite "${crds[@]}" meta.helm.sh/release-name=traefik-crd meta.helm.sh/release-namespace=kube-system >/dev/null || return 1 @@ -207,7 +207,9 @@ test_running_the_update_again_changes_nothing() { expect_other_app_routes_with_its_auth } -start_cluster || exit 1 +# k3s's Traefik only held the Gateway API CRDs before 1.37, when k3s began +# installing them as a chart of their own. +start_cluster "$K3S_BEFORE_GATEWAY_API" || exit 1 make_panel || exit 1 install_ingress_nginx || exit 1 deploy_panel_stubs || exit 1 diff --git a/tests/scripts/traefik_install_test.sh b/tests/scripts/traefik_install_test.sh index 123b349..379f038 100644 --- a/tests/scripts/traefik_install_test.sh +++ b/tests/scripts/traefik_install_test.sh @@ -143,10 +143,23 @@ hand_over() { source "$TEST_ROOT/utils/output_redirect.sh" # shellcheck source=utils/install_traefik.sh source "$TEST_ROOT/utils/install_traefik.sh" - adopt_orphaned_chart_objects "$TEST_ROOT/charts/gateway-crds" gateway-crds traefik + adopt_orphaned_chart_objects "$TEST_ROOT/charts/gateway-crds" gateway-crds traefik "$@" ) >"$WORK_DIR/output" 2>&1 } +# Runs one of install_traefik.sh's functions against the stub. +traefik_fn() { + ( + # shellcheck source=utils/colors.sh + source "$TEST_ROOT/utils/colors.sh" + # shellcheck source=utils/output_redirect.sh + source "$TEST_ROOT/utils/output_redirect.sh" + # shellcheck source=utils/install_traefik.sh + source "$TEST_ROOT/utils/install_traefik.sh" + PANEL_DIR="$TEST_ROOT" "$@" + ) +} + handovers() { grep -E ' (annotate|label) ' "$WORK_DIR/calls" 2>/dev/null || true } @@ -219,6 +232,63 @@ test_a_cluster_with_nothing_left_over_hands_nothing_over() { assert_eq "$(handovers)" "" } +K3S_GATEWAY_API_CHART="CustomResourceDefinition/httproutes.gateway.networking.k8s.io|gateway-api-crd|kube-system" + +test_k3s_installing_the_gateway_api_is_seen_from_its_chart() { + if traefik_fn k3s_installs_gateway_api; then + fail "a cluster without k3s's gateway-api-crd chart was taken for one with it" + fi + echo helmchart.helm.cattle.io/gateway-api-crd >"$WORK_DIR/helmcharts-gateway-api-crd" + traefik_fn k3s_installs_gateway_api || fail "k3s's gateway-api-crd chart went unseen" +} + +# From k3s 1.37 its gateway-api-crd release holds the Gateway API CRDs, which +# gateway-crds then leaves out rather than refusing to install. +test_a_left_out_template_is_not_checked() { + cluster_objects "$K3S_GATEWAY_API_CHART" + echo helmchart.helm.cattle.io/gateway-api-crd >"$WORK_DIR/helmcharts-gateway-api-crd" + if hand_over; then + fail "gateway-crds would install over the CRDs of k3s's gateway-api-crd release" + fi + assert_contains "$(cat "$WORK_DIR/output")" "belongs to the Helm release gateway-api-crd in kube-system" + rm "$WORK_DIR/calls" + hand_over gateway-api.yaml || fail "adopt_orphaned_chart_objects failed: $(cat "$WORK_DIR/output")" + assert_eq "$(handovers)" "" +} + +# k3s's gateway-api-crd turned off after gateway-crds stepped aside leaves the +# CRDs marked as that release's; gateway-crds takes them back on its upgrade. +test_once_installed_what_a_removed_release_left_is_handed_over() { + cluster_objects "$K3S_GATEWAY_API_CHART" \ + "CustomResourceDefinition/ingressroutes.traefik.io||" \ + "CustomResourceDefinition/gateways.gateway.networking.k8s.io|gateway-crds|traefik" + echo secret/sh.helm.release.v1.gateway-crds.v2 >"$WORK_DIR/releases-gateway-crds" + hand_over || fail "adopt_orphaned_chart_objects failed: $(cat "$WORK_DIR/output")" + assert_contains "$(handovers)" "annotate --overwrite customresourcedefinition/httproutes.gateway.networking.k8s.io meta.helm.sh/release-name=gateway-crds meta.helm.sh/release-namespace=traefik" + assert_not_contains "$(handovers)" ingressroutes + assert_not_contains "$(handovers)" gateways +} + +test_once_installed_a_live_release_is_left_alone() { + cluster_objects "$K3S_GATEWAY_API_CHART" + echo secret/sh.helm.release.v1.gateway-crds.v2 >"$WORK_DIR/releases-gateway-crds" + echo helmchart.helm.cattle.io/gateway-api-crd >"$WORK_DIR/helmcharts-gateway-api-crd" + hand_over || fail "adopt_orphaned_chart_objects failed: $(cat "$WORK_DIR/output")" + assert_eq "$(handovers)" "" +} + +test_only_gateway_api_objects_gateway_crds_holds_are_handed_to_k3s() { + cluster_objects "CustomResourceDefinition/httproutes.gateway.networking.k8s.io|gateway-crds|traefik" \ + "ValidatingAdmissionPolicy/safe-upgrades.gateway.networking.k8s.io|gateway-crds|traefik" \ + "ValidatingAdmissionPolicyBinding/safe-upgrades.gateway.networking.k8s.io|gateway-crds|traefik" \ + "CustomResourceDefinition/ingressroutes.traefik.io|gateway-crds|traefik" \ + "CustomResourceDefinition/gateways.gateway.networking.k8s.io|gateway-api-crd|kube-system" \ + "CustomResourceDefinition/grpcroutes.gateway.networking.k8s.io||" + assert_eq "$(traefik_fn gateway_api_held_by_gateway_crds)" "customresourcedefinition/httproutes.gateway.networking.k8s.io +validatingadmissionpolicy/safe-upgrades.gateway.networking.k8s.io +validatingadmissionpolicybinding/safe-upgrades.gateway.networking.k8s.io" +} + test_an_unchanged_chart_that_installed_is_left_alone() { run_install_chart assert_eq "$(calls)" "" diff --git a/utils/install_traefik.sh b/utils/install_traefik.sh index 23d8145..7d848a9 100644 --- a/utils/install_traefik.sh +++ b/utils/install_traefik.sh @@ -39,8 +39,8 @@ carry_over_ingress_nginx_ports() { install_traefik() { local ports_env="${1:-$PANEL_DIR/overlays/config/traefik-config.env}" - local minor http_port https_port crd binding nginx - local -a kc=() + local minor http_port https_port crd binding nginx crds_values left_out held object + local -a kc=() handed=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") # Gateway API v1.6's CRDs validate with CEL's isIP(), which the API server @@ -62,8 +62,37 @@ install_traefik() { return 1 fi - adopt_orphaned_chart_objects "$PANEL_DIR/charts/gateway-crds" gateway-crds traefik || return 1 - install_helm_chart gateway-crds "$PANEL_DIR/charts/gateway-crds" "" || return 1 + # Where k3s installs the Gateway API itself, gateway-crds brings only + # Traefik's CRDs. One that still holds k3s's objects, from before k3s was + # upgraded to 1.37, hands them over: Helm deletes what a release drops + # unless the live object is marked keep, which before gateway-crds 0.2.0 + # the safe-upgrades policy and its binding weren't. + crds_values="" + left_out="" + handed=() + if k3s_installs_gateway_api; then + crds_values="gatewayApi: false" + left_out=gateway-api.yaml + held="$(gateway_api_held_by_gateway_crds)" || { + err "could not read the cluster's Gateway API CRDs before installing gateway-crds" + return 1 + } + while IFS= read -r object; do + [ -n "$object" ] && handed+=("$object") + done <<<"$held" + if [ "${#handed[@]}" -gt 0 ]; then + output_redirect kubectl "${kc[@]}" annotate --overwrite "${handed[@]}" helm.sh/resource-policy=keep || return 1 + fi + fi + + adopt_orphaned_chart_objects "$PANEL_DIR/charts/gateway-crds" gateway-crds traefik "$left_out" || return 1 + install_helm_chart gateway-crds "$PANEL_DIR/charts/gateway-crds" "$crds_values" || return 1 + + if [ "${#handed[@]}" -gt 0 ]; then + warn "k3s installs the Gateway API CRDs now; handing ${#handed[@]} of gateway-crds' objects to its gateway-api-crd release" + output_redirect kubectl "${kc[@]}" annotate --overwrite "${handed[@]}" \ + meta.helm.sh/release-name=gateway-api-crd meta.helm.sh/release-namespace=kube-system || return 1 + fi for crd in gatewayclasses.gateway.networking.k8s.io gateways.gateway.networking.k8s.io \ httproutes.gateway.networking.k8s.io middlewares.traefik.io; do @@ -160,10 +189,46 @@ move_traefik_to_host_ports() { install_traefik "$@" } -# The cluster-scoped objects a chart's templates define, as kind/name. +# The cluster-scoped objects the given chart templates define, as kind/name. chart_objects() { awk '/^kind: / { kind = $2; next } - kind && /^ name: / { name = $2; gsub(/"/, "", name); print tolower(kind) "/" name; kind = "" }' "$1"/templates/*.yaml + kind && /^ name: / { name = $2; gsub(/"/, "", name); print tolower(kind) "/" name; kind = "" }' "$@" +} + +# The cluster's CRDs, admission policies and bindings with Helm's marks, as +# kind/name|release|release namespace. +marked_objects() { + local listed + local -a kc=() + [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") + + listed="$(kubectl "${kc[@]}" get customresourcedefinitions,validatingadmissionpolicies,validatingadmissionpolicybindings \ + -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}|{.metadata.annotations.meta\.helm\.sh/release-name}|{.metadata.annotations.meta\.helm\.sh/release-namespace}{"\n"}{end}')" || return 1 + awk -F'|' -v OFS='|' 'NF { $1 = tolower($1); print }' <<<"$listed" +} + +# From 1.37 k3s installs the Gateway API CRDs and the safe-upgrades policy +# itself, as its gateway-api-crd chart, whatever --disable=traefik says, and +# takes them over from any release that holds them. Turned off with +# --disable=gateway-api-crd, the chart isn't there. +k3s_installs_gateway_api() { + local -a kc=() + [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") + + [ -n "$(kubectl "${kc[@]}" get helmcharts.helm.cattle.io -A --field-selector metadata.name=gateway-api-crd -o name 2>/dev/null)" ] +} + +# The Gateway API objects marked as gateway-crds' own, as kind/name. +gateway_api_held_by_gateway_crds() { + local objects existing object owner owner_namespace + + objects="$(chart_objects "$PANEL_DIR/charts/gateway-crds/templates/gateway-api.yaml")" + existing="$(marked_objects)" || return 1 + while IFS='|' read -r object owner owner_namespace; do + if [ "$owner|$owner_namespace" = "gateway-crds|traefik" ] && grep -qxF "$object" <<<"$objects"; then + echo "$object" + fi + done <<<"$existing" } helm_release_gone() { @@ -189,33 +254,43 @@ helm_release_recorded() { # --disable=traefik deletes the release but leaves the CRDs marked as its, and # Helm won't install over objects another release claims. Ones whose release # is gone are handed to this one; anything a live release, or no release, owns -# stops the install. Only a first install can collide: Helm upgrades what a -# release already holds whatever its marks say, and the Traefik chart's own -# crds/ strips them from the traefik.io CRDs the two charts share. +# stops the install. Once the release exists, Helm upgrades what it holds +# whatever the marks say, and the Traefik chart's own crds/ strips them from +# the traefik.io CRDs the two charts share, so only objects a removed release +# left are handed over: the Gateway API's, after k3s's gateway-api-crd chart +# that gateway-crds stepped aside for is turned off. A template left out (its +# file name) is one the install won't render. adopt_orphaned_chart_objects() { - local chart_dir="$1" release="$2" release_namespace="$3" - local objects existing object owner owner_namespace refused=false - local -a kc=() orphaned=() + local chart_dir="$1" release="$2" release_namespace="$3" left_out="${4:-}" + local objects existing object owner owner_namespace template recorded=false refused=false + local -a kc=() orphaned=() templates=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") if helm_release_recorded "$release" "$release_namespace"; then - return 0 + recorded=true fi - objects="$(chart_objects "$chart_dir")" - if ! existing="$(kubectl "${kc[@]}" get customresourcedefinitions,validatingadmissionpolicies,validatingadmissionpolicybindings \ - -o jsonpath='{range .items[*]}{.kind}/{.metadata.name}|{.metadata.annotations.meta\.helm\.sh/release-name}|{.metadata.annotations.meta\.helm\.sh/release-namespace}{"\n"}{end}')"; then + for template in "$chart_dir"/templates/*.yaml; do + [ "$(basename "$template")" = "$left_out" ] || templates+=("$template") + done + objects="$(chart_objects "${templates[@]}")" + if ! existing="$(marked_objects)"; then err "could not read the cluster's CRDs before installing $release" return 1 fi while IFS='|' read -r object owner owner_namespace; do [ -n "$object" ] || continue - object="$(printf '%s' "$object" | tr '[:upper:]' '[:lower:]')" grep -qxF "$object" <<<"$objects" || continue if [ "$owner" = "$release" ] && [ "$owner_namespace" = "$release_namespace" ]; then continue fi + if [ "$recorded" = true ]; then + if [ -n "$owner" ] && helm_release_gone "$owner" "$owner_namespace"; then + orphaned+=("$object") + fi + continue + fi if [ -z "$owner" ]; then err "$object was created outside Helm, so $release can't install it; delete it, or label it app.kubernetes.io/managed-by=Helm and annotate it meta.helm.sh/release-name=$release meta.helm.sh/release-namespace=$release_namespace" refused=true From 0861dddb977b1c66a7dd90e62eeb43e7e579d94a Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 17:35:25 -0500 Subject: [PATCH 5/7] fix: gateway tests pick their k3s through K3S_IMAGE, so start_cluster keeps taking no arguments --- tests/gateway/k3s_upgrade_test.sh | 2 +- tests/gateway/lib.sh | 4 ++-- tests/gateway/upgrade_test.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/gateway/k3s_upgrade_test.sh b/tests/gateway/k3s_upgrade_test.sh index 338b938..6baf77e 100644 --- a/tests/gateway/k3s_upgrade_test.sh +++ b/tests/gateway/k3s_upgrade_test.sh @@ -84,7 +84,7 @@ test_another_update_changes_nothing() { wait_for 200 http "$API" /me } -start_cluster "$K3S_BEFORE_GATEWAY_API" || exit 1 +K3S_IMAGE="$K3S_BEFORE_GATEWAY_API" start_cluster || exit 1 make_panel || exit 1 update_panel >/dev/null || exit 1 deploy_panel_stubs || exit 1 diff --git a/tests/gateway/lib.sh b/tests/gateway/lib.sh index ac01114..15dc9c2 100644 --- a/tests/gateway/lib.sh +++ b/tests/gateway/lib.sh @@ -59,7 +59,7 @@ client() { docker exec "$PREFIX-client" "$@" } -# A cluster on k3s $1 (K3S_IMAGE by default) with the client container. +# A cluster on K3S_IMAGE, with the client container. start_cluster() { cleanup mkdir -p "$WORK_DIR" @@ -69,7 +69,7 @@ start_cluster() { # kube-proxy answers the InternalIP itself, which hid that ServiceLB # masquerades everything else. docker network create "$PREFIX-public" >/dev/null || return 1 - run_k3s "${1:-$K3S_IMAGE}" || return 1 + run_k3s "$K3S_IMAGE" || return 1 docker run -d --name "$PREFIX-client" --network "$PREFIX-net" "$PYTHON_IMAGE" sleep infinity >/dev/null || return 1 client apk add --no-cache -q curl jq openssl >/dev/null || return 1 diff --git a/tests/gateway/upgrade_test.sh b/tests/gateway/upgrade_test.sh index 1dca1bd..8854126 100644 --- a/tests/gateway/upgrade_test.sh +++ b/tests/gateway/upgrade_test.sh @@ -209,7 +209,7 @@ test_running_the_update_again_changes_nothing() { # k3s's Traefik only held the Gateway API CRDs before 1.37, when k3s began # installing them as a chart of their own. -start_cluster "$K3S_BEFORE_GATEWAY_API" || exit 1 +K3S_IMAGE="$K3S_BEFORE_GATEWAY_API" start_cluster || exit 1 make_panel || exit 1 install_ingress_nginx || exit 1 deploy_panel_stubs || exit 1 From 22618bd49145a4b3d184b517a787870fe00fd690 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 17:44:27 -0500 Subject: [PATCH 6/7] feature: minor and patch image releases merge themselves once their checks pass, and each major release opens a PR to review --- .github/scripts/image_updates.sh | 184 +++++++++++++++++++++++++ .github/workflows/image-updates.yml | 203 ++++++++++++++++++++++++++++ .github/workflows/panel-scripts.yml | 5 +- tests/scripts/image_updates_test.sh | 181 +++++++++++++++++++++++++ tests/scripts/stub/crane | 13 ++ 5 files changed, 585 insertions(+), 1 deletion(-) create mode 100755 .github/scripts/image_updates.sh create mode 100644 .github/workflows/image-updates.yml create mode 100644 tests/scripts/image_updates_test.sh create mode 100755 tests/scripts/stub/crane diff --git a/.github/scripts/image_updates.sh b/.github/scripts/image_updates.sh new file mode 100755 index 0000000..32627f6 --- /dev/null +++ b/.github/scripts/image_updates.sh @@ -0,0 +1,184 @@ +#!/bin/bash +# Finds newer releases of the third-party images the panel pins, for +# .github/workflows/image-updates.yml. Run it from inside the repository. +# +# image_updates.sh plan one line per update: +# image_updates.sh apply OLD NEW rewrites every pin of OLD to NEW +# image_updates.sh title the commit title for the plan lines on stdin +# +# A pin is a tracked `image: ` line or a shell `_IMAGE=` +# assignment whose tag is a version. An update keeps the tag's shape: its `v`, +# its number of version parts and the suffix after the version (`-alpine`, +# `-pg18`, `-k3s1`) stay the same, so a tag only moves to another release of +# the same build, and prereleases (`-rc1`, `.rc22`) never match. A pinned +# digest is re-pinned to the new tag's. An update is major when the first +# version part changes, or below 1.0 the second. +set -euo pipefail + +# Pins that only move by hand. +IGNORED_PATHS=( + # The upstream cert-manager release file, held on 1.17: from 1.18 the + # HTTP-01 solver's Ingress is rejected by the older ingress-nginx some + # panels still run, and renewals quietly stop. + ':!overlays/cert-manager-crds/cert-manager.yaml' + # The system-upgrade-controller's release assets, which + # tests/scripts/system_upgrade_test.sh checks against their sha256. + ':!base/system-upgrade/' + # The old releases the TimescaleDB upgrade tests start from. + ':!tests/timescaledb/' + # This script's own test fixtures. + ':!tests/scripts/image_updates_test.sh' + # Workflows, which the app's token can't push. + ':!.github/' +) + +die() { + echo "image_updates.sh: $*" >&2 + exit 1 +} + +cd "$(git rev-parse --show-toplevel)" + +# Every distinct ref pinned in the tracked files. +pins() { + git grep -I -h -o -E \ + -e 'image:[[:space:]]*"?[^[:space:]"]+' \ + -e '^[[:space:]]*[A-Z0-9_]+_IMAGE=[^[:space:]"]+' \ + -- "${IGNORED_PATHS[@]}" | + sed -E 's/^.*(image:[[:space:]]*"?|_IMAGE=)//' | + sort -u +} + +# The newest tags in the list on stdin that keep the shape of tag $1, as +# "minor " and "major " lines. +newer_tags() { + awk -v current="$1" ' + # Splits a tag into its v, version and suffix. 0 for a tag that is not + # a plain version, like latest, 19beta1-alpine or 31.0.rc22-arm64. + function parse(tag, out) { + out["v"] = "" + if (tag ~ /^v[0-9]/) { + out["v"] = "v" + tag = substr(tag, 2) + } + if (!match(tag, /^[0-9]+(\.[0-9]+)*/)) return 0 + out["version"] = substr(tag, 1, RLENGTH) + out["suffix"] = substr(tag, RLENGTH + 1) + return out["suffix"] == "" || out["suffix"] ~ /^-/ + } + # 1 when version a is newer than version b, which has as many parts. + function newer(a, b, x, y, n, i) { + n = split(a, x, ".") + split(b, y, ".") + for (i = 1; i <= n; i++) { + if (x[i] + 0 != y[i] + 0) return x[i] + 0 > y[i] + 0 + } + return 0 + } + BEGIN { + if (!parse(current, cur)) exit + parts = split(cur["version"], c, ".") + } + { + if (!parse($0, cand)) next + if (cand["v"] != cur["v"] || cand["suffix"] != cur["suffix"]) next + if (split(cand["version"], n, ".") != parts) next + if (!newer(cand["version"], cur["version"])) next + kind = "minor" + if (n[1] + 0 != c[1] + 0 || (c[1] + 0 == 0 && parts > 1 && n[2] + 0 != c[2] + 0)) kind = "major" + if (!(kind in best) || newer(cand["version"], best[kind])) { + best[kind] = cand["version"] + best_tag[kind] = $0 + } + } + END { + if ("minor" in best_tag) print "minor", best_tag["minor"] + if ("major" in best_tag) print "major", best_tag["major"] + } + ' +} + +plan() { + local ref name tag digest kind new_tag new_digest tags + pins | while IFS= read -r ref; do + digest="" + if [[ $ref == *@* ]]; then + digest="${ref#*@}" + fi + name="${ref%%@*}" + tag="${name##*:}" + # name@digest, a registry port with no tag after it, or a tag like + # latest that no release moves + if [[ $name != *:* || ! $tag =~ ^v?[0-9]+(\.[0-9]+)*(-.*)?$ ]]; then + continue + fi + name="${name%:*}" + if ! tags="$(crane ls "$name" 2>&1 &2 + continue + fi + newer_tags "$tag" <<<"$tags" | while read -r kind new_tag; do + if [ -z "$digest" ]; then + echo "$kind $ref $name:$new_tag" + elif new_digest="$(crane digest "$name:$new_tag" 2>&1 &2 + fi + done + done +} + +apply() { + local old="$1" new="$2" files=() before + while IFS= read -r file; do + files+=("$file") + done < <(git grep -I -l -F -e "$old" -- "${IGNORED_PATHS[@]}") + [ ${#files[@]} -gt 0 ] || die "$old is not pinned anywhere" + before="$(cat -- "${files[@]}" | git hash-object --stdin)" + OLD="$old" NEW="$new" perl -pi -e \ + 's/(image:[ \t]*"?|_IMAGE=)\Q$ENV{OLD}\E(?=["\s]|$)/$1$ENV{NEW}/g' -- "${files[@]}" + if [ "$(cat -- "${files[@]}" | git hash-object --stdin)" = "$before" ]; then + die "$old is mentioned, but not pinned, in ${files[*]}" + fi +} + +# "redis 8.12.0-alpine" for docker.io/library/redis:8.12.0-alpine@sha256:... +label() { + local ref="${1%%@*}" name + name="${ref%:*}" + case "${name%%/*}" in + *.* | *:* | localhost) [[ $name == */* ]] && name="${name#*/}" ;; + esac + name="${name#library/}" + echo "$name ${ref##*:}" +} + +title() { + local kind old new labels=() list + while read -r kind old new; do + [ -n "$new" ] || continue + labels+=("$(label "$new")") + done + [ ${#labels[@]} -gt 0 ] || die "no updates on stdin" + list="${labels[0]}" + if [ ${#labels[@]} -gt 1 ]; then + list="$(printf '%s, ' "${labels[@]:0:${#labels[@]}-1}")" + list="${list%, } and ${labels[${#labels[@]}-1]}" + fi + if [ ${#list} -gt 200 ]; then + list="${#labels[@]} images" + fi + echo "chore: $list" +} + +case "${1:-}" in + plan) plan ;; + apply) + [ $# -eq 3 ] || die "usage: image_updates.sh apply OLD NEW" + apply "$2" "$3" + ;; + title) title ;; + *) die "usage: image_updates.sh plan | apply OLD NEW | title" ;; +esac diff --git a/.github/workflows/image-updates.yml b/.github/workflows/image-updates.yml new file mode 100644 index 0000000..d164791 --- /dev/null +++ b/.github/workflows/image-updates.yml @@ -0,0 +1,203 @@ +name: Image updates + +# Moves the third-party images the panel pins to their newest releases; see +# .github/scripts/image_updates.sh for what counts as one and which pins only +# move by hand. Minor and patch releases go into one PR that merges itself +# once its checks pass. Each major release gets a PR of its own, assigned to +# REVIEWERS to check that the panel still works on it. Closing a major PR skips +# that release; a newer major release opens a new PR. +on: + schedule: + - cron: "0 13 * * 1" + workflow_dispatch: + +permissions: {} + +concurrency: + group: image-updates + +env: + REVIEWERS: lukepolo + CRANE_VERSION: v0.22.1 + CRANE_SHA256: 0ab7a1d6932a213aed964ce97666c3077fe691c8606413674a8b3e0b9ec4cda0 + GH_REPO: ${{ github.repository }} + +defaults: + run: + shell: bash + +jobs: + update: + if: github.repository == '5stackgg/5stack-panel' + runs-on: ubuntu-latest + timeout-minutes: 45 + # GITHUB_TOKEN only reads checks and keeps the labels. Branches and PRs + # come from the app's token, because ones from GITHUB_TOKEN start no checks + # and no release. + permissions: + contents: read + checks: read + statuses: read + pull-requests: read + issues: write + steps: + - name: Get a token from the 5Stack Ops app + id: app + uses: actions/create-github-app-token@v3 + with: + client-id: ${{ vars.OPS_APP_CLIENT_ID }} + private-key: ${{ secrets.OPS_APP_KEY }} + permission-contents: write + permission-pull-requests: write + + - uses: actions/checkout@v4 + with: + token: ${{ steps.app.outputs.token }} + fetch-depth: 0 + + - name: Install crane + run: | + curl -fsSLo "$RUNNER_TEMP/crane.tar.gz" \ + "https://github.com/google/go-containerregistry/releases/download/$CRANE_VERSION/go-containerregistry_Linux_x86_64.tar.gz" + echo "$CRANE_SHA256 $RUNNER_TEMP/crane.tar.gz" | sha256sum -c - + tar -xzf "$RUNNER_TEMP/crane.tar.gz" -C "$RUNNER_TEMP" crane + echo "$RUNNER_TEMP" >>"$GITHUB_PATH" + + # The steps below switch to branches made from main, so they run a copy + # of this commit's script rather than whichever one the branch has. + - name: Keep this run's script + run: | + cp .github/scripts/image_updates.sh "$RUNNER_TEMP/image_updates.sh" + echo "SCRIPT=$RUNNER_TEMP/image_updates.sh" >>"$GITHUB_ENV" + + - name: Commit as the app + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + APP_SLUG: ${{ steps.app.outputs.app-slug }} + run: | + id="$(gh api "/users/$APP_SLUG%5Bbot%5D" --jq .id)" + git config user.name "${APP_SLUG}[bot]" + git config user.email "$id+${APP_SLUG}[bot]@users.noreply.github.com" + + - name: Keep the labels + env: + GH_TOKEN: ${{ github.token }} + run: | + gh label create image-update --force --color 0e8a16 \ + --description "Minor and patch image releases, merged once the checks pass" + gh label create image-update:major --force --color d93f0b \ + --description "A major image release to check before merging" + + - name: Merge the minor and patch releases + id: minor + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + CHECKS_TOKEN: ${{ github.token }} + run: | + branch=image-updates/minor + "$SCRIPT" plan >"$RUNNER_TEMP/plan" + cat "$RUNNER_TEMP/plan" + awk '$1 == "minor"' "$RUNNER_TEMP/plan" >"$RUNNER_TEMP/minor" + pr="$(gh pr list --head "$branch" --state open --json number --jq '.[0].number // empty')" + if [ ! -s "$RUNNER_TEMP/minor" ]; then + if [ -n "$pr" ]; then + gh pr close "$pr" --delete-branch --comment "main already has these releases." + fi + exit 0 + fi + + git switch -q -C "$branch" origin/main + while read -r _ old new <&3; do + "$SCRIPT" apply "$old" "$new" + done 3<"$RUNNER_TEMP/minor" + title="$("$SCRIPT" title <"$RUNNER_TEMP/minor")" + git commit -q -am "$title" + git push -q --force origin "$branch" + head="$(git rev-parse HEAD)" + if [ -z "$pr" ]; then + gh pr create --base main --head "$branch" --title "$title" --body "" --label image-update + pr="$(gh pr view "$branch" --json number --jq .number)" + else + gh pr edit "$pr" --title "$title" + fi + + # Let the checks the push started register before waiting on them. + sleep 90 + if ! GH_TOKEN="$CHECKS_TOKEN" gh pr checks "$pr" --watch --fail-fast --interval 30; then + gh pr edit "$pr" --add-assignee "$REVIEWERS" + echo "::error::#$pr's checks failed, so it was left open for $REVIEWERS" + exit 1 + fi + if ! gh pr merge "$pr" --squash --match-head-commit "$head"; then + gh pr edit "$pr" --add-assignee "$REVIEWERS" + echo "::error::#$pr passed its checks but could not be merged, so it was left open for $REVIEWERS" + exit 1 + fi + + - name: Open a PR for each major release + # Also after the minor releases' PR fails, but not when the setup did. + if: ${{ !cancelled() && steps.minor.outcome != 'skipped' }} + env: + GH_TOKEN: ${{ steps.app.outputs.token }} + run: | + # true when origin's $1 changes main the way HEAD does + same_change() { + [ "$(git diff "origin/main...origin/$1" | git patch-id --stable)" = \ + "$(git diff origin/main HEAD | git patch-id --stable)" ] + } + + # Planned again on main, which may now have the minor releases. + git fetch -q origin main + git switch -q --discard-changes --detach origin/main + "$SCRIPT" plan >"$RUNNER_TEMP/plan" + awk '$1 == "major"' "$RUNNER_TEMP/plan" >"$RUNNER_TEMP/major" + cat "$RUNNER_TEMP/major" + + while read -r kind old new <&3; do + name="${new%%@*}" + tag="${name##*:}" + name="${name%:*}" + old_tag="${old%%@*}" + old_tag="${old_tag##*:}" + slug="$(printf '%s' "$name" | tr '/:' '--')" + branch="image-updates/major/$slug/$tag" + state="$(gh pr list --head "$branch" --state all --json state \ + --jq 'map(.state) | if any(. == "OPEN") then "open" elif length > 0 then "closed" else "" end')" + # Merged, or closed to skip this release. + if [ "$state" = closed ]; then + continue + fi + + git switch -q --discard-changes -C "$branch" origin/main + "$SCRIPT" apply "$old" "$new" + title="$("$SCRIPT" title <<<"$kind $old $new")" + git commit -q -am "$title" + + if [ "$state" = open ]; then + # main changed this pin since the PR was opened, as a minor + # release of the same image does: rebuild it so it still merges. + if ! same_change "$branch"; then + git push -q --force origin "$branch" + fi + continue + fi + + git push -q --force origin "$branch" + label="${title#chore: }" + gh pr create --base main --head "$branch" --title "$title" \ + --label image-update:major --assignee "$REVIEWERS" --reviewer "$REVIEWERS" \ + --body "$(printf '%s\n' \ + "$label is a new major release, up from $old_tag, so it waits for a review instead of merging itself. Before merging, check that the panel still works on it:" \ + "" \ + "- [ ] Read its release notes from $old_tag up to $tag for breaking changes and migrations." \ + "- [ ] Try it on a test panel. This branch is published to the \`branch/$branch\` channel: \`sudo ./service.sh --channel branch/$branch\`." \ + "" \ + "Closing this PR skips $tag. A newer major release opens a new PR.")" + pr="$(gh pr view "$branch" --json number --jq .number)" + + gh pr list --state open --label image-update:major --json number,headRefName \ + --jq ".[] | select(.headRefName | startswith(\"image-updates/major/$slug/\")) | select(.headRefName != \"$branch\") | .number" | + while read -r superseded; do + gh pr close "$superseded" --delete-branch --comment "Superseded by #$pr." + done + done 3<"$RUNNER_TEMP/major" diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml index fb594cb..0908be9 100644 --- a/.github/workflows/panel-scripts.yml +++ b/.github/workflows/panel-scripts.yml @@ -25,6 +25,7 @@ on: - "base/backups/**" - "base/game-server-node-connector/**" - "overlays/nvidia/**" + - ".github/scripts/**" - .github/workflows/panel-scripts.yml pull_request: paths: @@ -49,6 +50,7 @@ on: - "base/backups/**" - "base/game-server-node-connector/**" - "overlays/nvidia/**" + - ".github/scripts/**" - .github/workflows/panel-scripts.yml jobs: @@ -56,7 +58,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: shellcheck -x service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh tests/scripts/*.sh tests/scripts/stub/kubectl + - run: shellcheck -x service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh .github/scripts/image_updates.sh tests/scripts/*.sh tests/scripts/stub/kubectl tests/scripts/stub/crane - run: bash tests/scripts/setup_env_test.sh - run: bash tests/scripts/apply_overlay_test.sh - run: bash tests/scripts/debug_test.sh @@ -72,3 +74,4 @@ jobs: - run: bash tests/scripts/postgres_backup_test.sh - run: bash tests/scripts/choose_node_test.sh - run: bash tests/scripts/nvidia_fallback_test.sh + - run: bash tests/scripts/image_updates_test.sh diff --git a/tests/scripts/image_updates_test.sh b/tests/scripts/image_updates_test.sh new file mode 100644 index 0000000..37cac2d --- /dev/null +++ b/tests/scripts/image_updates_test.sh @@ -0,0 +1,181 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +STUB_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")/stub" && pwd)" +SCRIPT="$TEST_ROOT/.github/scripts/image_updates.sh" +OLD_DIGEST="sha256:$(printf "%064d" 1)" +NEW_DIGEST="sha256:$(printf "%064d" 2)" + +# pin FILE LINE...: a tracked file of the test repository. +pin() { + mkdir -p "$(dirname "$WORK_DIR/repo/$1")" + printf '%s\n' "${@:2}" >"$WORK_DIR/repo/$1" +} + +# tags REPOSITORY TAG...: what the registry lists for REPOSITORY. +tags() { + mkdir -p "$WORK_DIR/tags" + printf '%s\n' "${@:2}" >"$WORK_DIR/tags/$(printf '%s' "$1" | tr / _)" +} + +digest() { + echo "$1 $2" >>"$WORK_DIR/digests" +} + +image_updates() { + [ -d "$WORK_DIR/repo/.git" ] || git init -q "$WORK_DIR/repo" + ( + cd "$WORK_DIR/repo" || exit 1 + git add -A + PATH="$STUB_BIN:$PATH" bash "$SCRIPT" "$@" + ) +} + +test_the_newest_minor_and_the_newest_major_are_planned() { + pin base/typesense.yaml " image: typesense/typesense:30.2" + tags typesense/typesense 29.1 30.0 30.2 30.3 30.10 31.0 32.1 + assert_eq "$(image_updates plan)" "minor typesense/typesense:30.2 typesense/typesense:30.10 +major typesense/typesense:30.2 typesense/typesense:32.1" +} + +test_an_update_keeps_the_suffix_and_the_number_of_version_parts() { + pin base/redis.yaml " image: redis:8.10.3-alpine" + tags redis 8.10.3-alpine 8.10.4-alpine3.23 8.11-alpine 8.12.0-alpine 8.12.1 9.0.0-rc1-alpine 8.12.1-alpine-arm64 + assert_eq "$(image_updates plan)" "minor redis:8.10.3-alpine redis:8.12.0-alpine" +} + +test_an_update_keeps_the_v() { + pin base/hasura.yaml " - image: hasura/graphql-engine:v2.51.3-ce.cli-migrations-v3" + tags hasura/graphql-engine v2.51.4-ce.cli-migrations-v3 2.51.5-ce.cli-migrations-v3 v2.51.9 v2.52.0-beta.1-ce.cli-migrations-v3 + assert_eq "$(image_updates plan)" "minor hasura/graphql-engine:v2.51.3-ce.cli-migrations-v3 hasura/graphql-engine:v2.51.4-ce.cli-migrations-v3" +} + +test_prereleases_and_tags_that_are_not_versions_are_skipped() { + pin base/typesense.yaml " image: typesense/typesense:30.2" + tags typesense/typesense latest main 31.0.rc22-arm64 30.3rc1 30.2 + assert_eq "$(image_updates plan)" "" +} + +# Below 1.0 a minor release may break things, as a major does from 1.0 on. +test_below_1_0_a_new_second_part_is_major() { + pin base/rustfs.yaml " image: rustfs/rc:v0.1.36" + tags rustfs/rc v0.1.35 v0.1.37 v0.2.0 v1.0.0 + assert_eq "$(image_updates plan)" "minor rustfs/rc:v0.1.36 rustfs/rc:v0.1.37 +major rustfs/rc:v0.1.36 rustfs/rc:v1.0.0" +} + +test_a_one_part_version_only_moves_to_a_major() { + pin base/backup.yaml " image: postgres:18-alpine" + tags postgres 17-alpine 18-alpine 18.1-alpine 19-alpine 19beta1-alpine 19-alpine3.22 + assert_eq "$(image_updates plan)" "major postgres:18-alpine postgres:19-alpine" +} + +test_a_pinned_digest_is_repinned_to_the_new_tags() { + pin base/redis.yaml " image: redis:8.10.3-alpine@$OLD_DIGEST" + tags redis 8.10.4-alpine + digest redis:8.10.4-alpine "$NEW_DIGEST" + assert_eq "$(image_updates plan)" "minor redis:8.10.3-alpine@$OLD_DIGEST redis:8.10.4-alpine@$NEW_DIGEST" +} + +test_a_digest_that_is_not_a_sha256_is_not_pinned() { + pin base/redis.yaml " image: redis:8.10.3-alpine@$OLD_DIGEST" + tags redis 8.10.4-alpine + digest redis:8.10.4-alpine "sha256:\$(reboot)" + assert_eq "$(image_updates plan 2>"$WORK_DIR/stderr")" "" + assert_contains "$(cat "$WORK_DIR/stderr")" "::warning::could not read the digest of redis:8.10.4-alpine" +} + +test_shell_and_quoted_pins_are_planned() { + pin tests/lib.sh "K3S_IMAGE=rancher/k3s:v1.36.5-k3s1" + pin overlays/x.yaml ' image: "coturn/coturn:4.18.0"' + tags rancher/k3s v1.36.5-rc1-k3s1 v1.36.6-k3s1 v1.36.5-k3s2 + tags coturn/coturn 4.18.1 4.18.0-r0-debian-amd64 + assert_eq "$(image_updates plan)" "minor coturn/coturn:4.18.0 coturn/coturn:4.18.1 +minor rancher/k3s:v1.36.5-k3s1 rancher/k3s:v1.36.6-k3s1" +} + +test_refs_without_a_version_tag_never_reach_the_registry() { + pin base/api.yaml " - image: ghcr.io/5stackgg/api:latest" + # shellcheck disable=SC2016 # a pin of an unexpanded variable + pin tests/t.sh "SEED_IMAGE=timescale/timescaledb@sha256:abc" 'image: $PEBBLE_IMAGE' + assert_eq "$(image_updates plan)" "" + [ ! -e "$WORK_DIR/crane-calls" ] || fail "crane was asked: $(cat "$WORK_DIR/crane-calls")" +} + +test_held_pins_are_not_planned() { + pin overlays/cert-manager-crds/cert-manager.yaml ' image: "quay.io/jetstack/cert-manager-controller:v1.17.4"' + pin base/system-upgrade/system-upgrade-controller.yaml " image: rancher/system-upgrade-controller:v0.20.2" + pin tests/timescaledb/major_upgrade_test.sh "OLD_IMAGE=timescale/timescaledb:2.30.2-pg17" + pin tests/scripts/image_updates_test.sh "K3S_IMAGE=rancher/k3s:v1.36.5-k3s1" + pin .github/workflows/x.yml " image: rancher/k3s:v1.36.5-k3s1" + tags rancher/k3s v1.36.6-k3s1 + tags quay.io/jetstack/cert-manager-controller v1.17.4 v1.21.2 + tags rancher/system-upgrade-controller v0.20.2 v0.20.3 + tags timescale/timescaledb 2.30.2-pg17 2.31.0-pg17 + assert_eq "$(image_updates plan)" "" +} + +test_an_unreadable_repository_skips_only_its_own_pins() { + pin base/a.yaml " image: busybox:1.38" " image: coturn/coturn:4.18.0" + tags busybox 1.38 1.39 + assert_eq "$(image_updates plan 2>"$WORK_DIR/stderr")" "minor busybox:1.38 busybox:1.39" + assert_contains "$(cat "$WORK_DIR/stderr")" "::warning::could not list the tags of coturn/coturn" +} + +test_apply_rewrites_every_pin_and_nothing_else() { + pin base/a.yaml \ + " image: busybox:1.38" \ + ' - image: "busybox:1.38"' \ + " image: busybox:1.38.1" \ + " image: mybusybox:1.38" \ + " image: docker.io/busybox:1.38" + pin tests/a.sh \ + "TEST_IMAGE=busybox:1.38" \ + "OTHER=busybox:1.38" \ + 'echo "pulling image busybox:1.38"' + pin tests/timescaledb/a.sh "OLD_IMAGE=busybox:1.38" + image_updates apply busybox:1.38 busybox:1.39 || fail "apply failed" + assert_eq "$(cat "$WORK_DIR/repo/base/a.yaml")" " image: busybox:1.39 + - image: \"busybox:1.39\" + image: busybox:1.38.1 + image: mybusybox:1.38 + image: docker.io/busybox:1.38" + assert_eq "$(cat "$WORK_DIR/repo/tests/a.sh")" "TEST_IMAGE=busybox:1.39 +OTHER=busybox:1.38 +echo \"pulling image busybox:1.38\"" + assert_eq "$(cat "$WORK_DIR/repo/tests/timescaledb/a.sh")" "OLD_IMAGE=busybox:1.38" +} + +test_apply_moves_a_digest_pin_whole() { + pin base/a.yaml " image: redis:8.10.3-alpine@$OLD_DIGEST" + image_updates apply "redis:8.10.3-alpine@$OLD_DIGEST" "redis:8.10.4-alpine@$NEW_DIGEST" || fail "apply failed" + assert_eq "$(cat "$WORK_DIR/repo/base/a.yaml")" " image: redis:8.10.4-alpine@$NEW_DIGEST" +} + +test_apply_fails_when_the_ref_is_not_pinned() { + pin tests/a.sh 'echo "pulling image busybox:1.38"' + if image_updates apply busybox:1.38 busybox:1.39 2>"$WORK_DIR/stderr"; then + fail "apply succeeded" + fi + assert_contains "$(cat "$WORK_DIR/stderr")" "busybox:1.38 is mentioned, but not pinned, in tests/a.sh" + if image_updates apply busybox:1.37 busybox:1.39 2>"$WORK_DIR/stderr"; then + fail "apply succeeded" + fi + assert_contains "$(cat "$WORK_DIR/stderr")" "busybox:1.37 is not pinned anywhere" +} + +test_the_title_names_each_image_and_its_new_tag() { + pin README.md "readme" + assert_eq "$(image_updates title <<<"minor redis:8.10.3-alpine redis:8.12.0-alpine@sha256:abc")" \ + "chore: redis 8.12.0-alpine" + assert_eq "$(printf '%s\n' \ + "minor redis:8.10.3-alpine redis:8.12.0-alpine@sha256:abc" \ + "minor ghcr.io/letsencrypt/pebble:2.10.1 ghcr.io/letsencrypt/pebble:2.10.2" \ + "major docker.io/library/postgres:18-alpine docker.io/library/postgres:19-alpine" | + image_updates title)" \ + "chore: redis 8.12.0-alpine, letsencrypt/pebble 2.10.2 and postgres 19-alpine" +} + +run_tests diff --git a/tests/scripts/stub/crane b/tests/scripts/stub/crane new file mode 100755 index 0000000..3ba173f --- /dev/null +++ b/tests/scripts/stub/crane @@ -0,0 +1,13 @@ +#!/bin/bash + +# crane for tests/scripts/image_updates_test.sh: the tags of a repository come +# from $WORK_DIR/tags/, and digests from +# " " lines in $WORK_DIR/digests. + +echo "$*" >>"$WORK_DIR/crane-calls" + +case "$1" in + ls) cat "$WORK_DIR/tags/$(printf '%s' "$2" | tr / _)" ;; + digest) awk -v ref="$2" '$1 == ref { print $2; found = 1 } END { exit !found }' "$WORK_DIR/digests" 2>/dev/null ;; + *) exit 2 ;; +esac From 5ac59bf33c3e980408bbce1fafe185dc881308fb Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 17:53:25 -0500 Subject: [PATCH 7/7] fix: kustomize is downloaded straight from its release and checked against its sha256, instead of through GitHub's rate-limited API --- utils/setup_kustomize.sh | 70 +++++++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 18 deletions(-) diff --git a/utils/setup_kustomize.sh b/utils/setup_kustomize.sh index 85d7b34..2f369ca 100644 --- a/utils/setup_kustomize.sh +++ b/utils/setup_kustomize.sh @@ -4,23 +4,57 @@ setup_kustomize() { # $PANEL_DIR, not $(dirname "$0"): when update.sh is sourced from # install.sh, $0 is still install.sh, and both depend on the caller's cwd. local dir="$PANEL_DIR" - if ! [ -f "$dir/kustomize" ] || ! [ -x "$dir/kustomize" ] - then - echo "kustomize not found. Installing..." - # Pin the installer to an immutable release tag rather than the moving - # `master` branch, and download-then-run instead of piping the network - # straight into bash. The installer itself checksum-verifies the - # kustomize binary it fetches for the requested version. - local version="5.8.3" - local script - script="$(mktemp)" - if curl -fsSL "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/kustomize/v${version}/hack/install_kustomize.sh" -o "$script"; then - bash "$script" "$version" "$dir" - else - echo "Failed to download kustomize installer" >&2 - rm -f "$script" + local version="5.8.3" os arch sum got work + if [ -f "$dir/kustomize" ] && [ -x "$dir/kustomize" ]; then + return 0 + fi + + echo "kustomize not found. Installing..." + case "$(uname -s)" in + Linux) os=linux ;; + Darwin) os=darwin ;; + *) + echo "kustomize $version has no release for $(uname -s)" >&2 + return 1 + ;; + esac + case "$(uname -m)" in + x86_64 | amd64) arch=amd64 ;; + aarch64 | arm64) arch=arm64 ;; + *) + echo "kustomize $version has no release for $(uname -m)" >&2 return 1 - fi - rm -f "$script" + ;; + esac + # The release's own archive, checked against the sums its checksums.txt + # published. Kustomize's install script finds the release through GitHub's + # API instead, which refuses after 60 anonymous requests an hour from one + # address, as CI runners share theirs. + case "${os}_${arch}" in + linux_amd64) sum=cb9e31198d3f63b44848bd0afc4c8efc56e6cfef9c93a4a39a211940e6decd61 ;; + linux_arm64) sum=9867b76482cfc6d25b546abc1d2b5a32bd137505aba640649b4b378f64f6d68f ;; + darwin_amd64) sum=d08fb85846c3d977f0e2a66e7ff45f1b5264c294685e5e726e1a8bdfe0da8dee ;; + darwin_arm64) sum=4cbb08e209d8fff47cd60bfb6407daaf5ee699f9b9295849a1065024fac2d1ef ;; + esac + + work="$(mktemp -d)" + if ! curl -fsSL -o "$work/kustomize.tar.gz" \ + "https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize%2Fv${version}/kustomize_v${version}_${os}_${arch}.tar.gz"; then + echo "Failed to download kustomize $version" >&2 + rm -rf "$work" + return 1 + fi + got="$( (sha256sum "$work/kustomize.tar.gz" 2>/dev/null || shasum -a 256 "$work/kustomize.tar.gz") | cut -d' ' -f1)" + if [ "$got" != "$sum" ]; then + echo "The kustomize $version download doesn't match its published sha256 ($got)" >&2 + rm -rf "$work" + return 1 + fi + if ! tar -xzf "$work/kustomize.tar.gz" -C "$work" kustomize || ! mv "$work/kustomize" "$dir/kustomize"; then + echo "Could not unpack kustomize $version into $dir" >&2 + rm -rf "$work" + return 1 fi -} \ No newline at end of file + chmod +x "$dir/kustomize" + rm -rf "$work" +}