From 615e408d69ec7916b8170bd1b58c772e7ddad2cb Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 16:38:38 -0500 Subject: [PATCH 1/3] test: gateway tests reach the node on an address that isn't its InternalIP kube-proxy answers the node's InternalIP itself, so the tests never went through ServiceLB, which masquerades every other address. The new servicelb upgrade test and the unit tests cover Traefik binding the host's ports. --- .github/workflows/gateway.yml | 6 ++ tests/gateway/lib.sh | 17 +++- tests/gateway/servicelb_upgrade_test.sh | 110 ++++++++++++++++++++++++ tests/scripts/stub/kubectl | 9 +- tests/scripts/traefik_install_test.sh | 53 ++++++++++++ 5 files changed, 190 insertions(+), 5 deletions(-) create mode 100644 tests/gateway/servicelb_upgrade_test.sh diff --git a/.github/workflows/gateway.yml b/.github/workflows/gateway.yml index 7cf8de2..aba9fd9 100644 --- a/.github/workflows/gateway.yml +++ b/.github/workflows/gateway.yml @@ -64,6 +64,12 @@ jobs: - uses: actions/checkout@v4 - run: bash tests/gateway/upgrade_test.sh + upgrade-from-servicelb: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash tests/gateway/servicelb_upgrade_test.sh + cert-manager: runs-on: ubuntu-latest steps: diff --git a/tests/gateway/lib.sh b/tests/gateway/lib.sh index 43fd7d5..2dd0d5f 100644 --- a/tests/gateway/lib.sh +++ b/tests/gateway/lib.sh @@ -2,7 +2,7 @@ # k3s in Docker, with the panel's routing layer installed by the same functions # update.sh calls, in front of stub backends named like the real Services. -# Requests come from a separate client container on the same Docker network, +# Requests come from a separate client container on a second Docker network, # so the client IP the backends see is a real peer's. One cluster at a time; # everything is named after $PREFIX and removed on exit. @@ -22,7 +22,7 @@ export KUBECONFIG PANEL_DIR DEBUG cleanup() { docker rm -f -v "$PREFIX-k3s" "$PREFIX-client" >/dev/null 2>&1 - docker network rm "$PREFIX-net" >/dev/null 2>&1 + docker network rm "$PREFIX-net" "$PREFIX-public" >/dev/null 2>&1 rm -rf "$WORK_DIR" } @@ -70,13 +70,22 @@ start_cluster() { sleep 2 done kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null || return 1 - NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-k3s")" docker run -d --name "$PREFIX-client" --network "$PREFIX-net" "$PYTHON_IMAGE" sleep infinity >/dev/null || return 1 client apk add --no-cache -q curl jq openssl >/dev/null || return 1 docker cp "$HERE/client.py" "$PREFIX-client:/client.py" >/dev/null + + # Clients reach the node on an address k3s doesn't know as its InternalIP, + # as a server's public one is when k3s runs on a private or VPN address. + # kube-proxy answers the InternalIP itself, which hid that ServiceLB + # masquerades everything else. + docker network create "$PREFIX-public" >/dev/null || return 1 + docker network connect "$PREFIX-public" "$PREFIX-k3s" || return 1 + docker network connect "$PREFIX-public" "$PREFIX-client" || return 1 + NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-k3s")" + [ "$NODE_IP" != "$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" ] || return 1 # shellcheck disable=SC2034 - CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-client")" + CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-client")" } set_env() { diff --git a/tests/gateway/servicelb_upgrade_test.sh b/tests/gateway/servicelb_upgrade_test.sh new file mode 100644 index 0000000..556a60d --- /dev/null +++ b/tests/gateway/servicelb_upgrade_test.sh @@ -0,0 +1,110 @@ +#!/bin/bash + +# An install whose Traefik still sits behind k3s ServiceLB, as the panel ran it +# before Traefik bound the host's ports itself, taken through update.sh's +# install_traefik. + +# shellcheck source=tests/gateway/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +trap cleanup EXIT + +API="api.$DOMAIN" +PORTS_ENV="$PANEL_DIR/overlays/config/traefik-config.env" + +install_traefik_behind_servicelb() { + local crd + install_helm_chart gateway-crds "$PANEL_DIR/charts/gateway-crds" "" >/dev/null || return 1 + for crd in gatewayclasses.gateway.networking.k8s.io gateways.gateway.networking.k8s.io \ + httproutes.gateway.networking.k8s.io middlewares.traefik.io; do + kubectl wait --for=condition=Established "crd/$crd" --timeout=120s >/dev/null || return 1 + done + install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat <<'EOF' +traefik: + service: + spec: {type: LoadBalancer, externalTrafficPolicy: Local} + updateStrategy: + rollingUpdate: {maxUnavailable: 0, maxSurge: 1} +EOF +)" >/dev/null || return 1 + kubectl -n traefik rollout status deployment/traefik --timeout=300s >/dev/null || return 1 + for _ in $(seq 1 90); do + [ -n "$(kubectl -n traefik get service traefik -o jsonpath='{.status.loadBalancer.ingress}')" ] && return 0 + sleep 2 + done + return 1 +} + +traefik_pod() { + kubectl -n traefik get pods -l app.kubernetes.io/name=traefik -o jsonpath='{.items[*].metadata.uid}' +} + +test_behind_servicelb_every_request_comes_from_its_pod() { + wait_for 200 https "$API" /me + case "$(json '.headers["x-forwarded-for"]')" in + 10.42.*) ;; + *) fail "expected a ServiceLB pod address, got $(json '.headers["x-forwarded-for"]')" ;; + esac +} + +test_the_move_costs_seconds() { + local longest + longest="$(jq -r .longest_outage <<<"$POLL")" + echo " outage while Traefik moved: ${longest}s ($(jq -r '"\(.failed) of \(.requests)"' <<<"$POLL") requests failed)" + awk -v t="$longest" 'BEGIN { exit !(t < 30) }' || fail "$POLL" +} + +test_servicelb_no_longer_fronts_traefik() { + [ "$(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')" = ClusterIP ] || fail "Traefik's Service is still $(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')" + [ -z "$(kubectl -n kube-system get daemonset -l svccontroller.k3s.cattle.io/svcname=traefik -o name)" ] || fail "svclb-traefik is still running" +} + +test_the_client_ip_reaches_the_backend() { + call https "$API" /me + expect_code 200 + expect_json '.headers["x-forwarded-for"]' "$CLIENT_IP" + expect_json '.headers["x-real-ip"]' "$CLIENT_IP" + call http "$API" /me + expect_code 308 +} + +test_running_the_update_again_changes_nothing() { + local before + before="$(traefik_pod)" + install_traefik "$PORTS_ENV" >/dev/null || fail "install_traefik failed" + move_traefik_to_host_ports "$PORTS_ENV" >/dev/null || fail "move_traefik_to_host_ports failed" + [ "$(traefik_pod)" = "$before" ] || fail "Traefik was restarted" + call https "$API" /me + expect_code 200 +} + +start_cluster || exit 1 +make_panel || exit 1 +install_traefik_behind_servicelb || exit 1 +deploy_panel_stubs || exit 1 +routing_layer https || exit 1 +tls_secret 5stack-ssl "$DOMAIN" "$API" "demos.$DOMAIN" "ws.$DOMAIN" "tv.$DOMAIN" "search.$DOMAIN" "console.$DOMAIN" || exit 1 +tls_secret 5stack-mediamtx-ssl "hls.$DOMAIN" || exit 1 + +run_test test_behind_servicelb_every_request_comes_from_its_pod + +client rm -f /tmp/stop +docker exec -d "$PREFIX-client" sh -c "python3 /client.py poll $NODE_IP $API /me /tmp/stop >/tmp/poll.json" +sleep 2 +install_traefik "$PORTS_ENV" || exit 1 +sleep 5 +client touch /tmp/stop +for _ in $(seq 1 10); do + client test -s /tmp/poll.json && break + sleep 1 +done +POLL="$(client cat /tmp/poll.json)" + +for name in \ + test_the_move_costs_seconds \ + test_servicelb_no_longer_fronts_traefik \ + test_the_client_ip_reaches_the_backend \ + test_running_the_update_again_changes_nothing; do + run_test "$name" +done + +summary diff --git a/tests/scripts/stub/kubectl b/tests/scripts/stub/kubectl index 76e019d..450f482 100755 --- a/tests/scripts/stub/kubectl +++ b/tests/scripts/stub/kubectl @@ -3,6 +3,7 @@ echo "$*" >>"$WORK_DIR/calls" args=" $* " +file="" pod="" container="" previous=false @@ -14,13 +15,19 @@ while [ $# -gt 0 ]; do shift ;; --previous) previous=true ;; + -f) + file="$2" + shift + ;; esac shift done case "$args" in - *" get crd helmcharts.helm.cattle.io "* | *" apply --server-side "*) ;; + *" get crd helmcharts.helm.cattle.io "*) ;; + *" apply --server-side "*) if [ -n "$file" ]; then cat "$file" >>"$WORK_DIR/server-side-applied"; fi ;; *" apply -f - "*) cat >"$WORK_DIR/applied" ;; + *" get service traefik "*'{.spec.type}'*) if [ -f "$WORK_DIR/traefik-type" ]; then cat "$WORK_DIR/traefik-type"; fi ;; *" get service traefik "*) if [ -f "$WORK_DIR/traefik-address" ]; then cat "$WORK_DIR/traefik-address"; fi ;; *" get helmchart "*) echo 1 ;; *" get gateway traefik-gateway "*) [ -f "$WORK_DIR/shared-gateway" ] || exit 1 ;; diff --git a/tests/scripts/traefik_install_test.sh b/tests/scripts/traefik_install_test.sh index a0c28b7..dd8d9f0 100644 --- a/tests/scripts/traefik_install_test.sh +++ b/tests/scripts/traefik_install_test.sh @@ -263,4 +263,57 @@ test_dev_retires_without_waiting_for_routes() { assert_not_contains "$(cat "$WORK_DIR/calls")" "wait httproute/api" } +test_retiring_with_no_ingress_nginx_waits_for_no_address() { + run_retire || fail "retire_ingress_nginx failed: $(cat "$WORK_DIR/output")" + assert_not_contains "$(cat "$WORK_DIR/calls")" "loadBalancer.ingress" +} + +run_install() { + ( + export PANEL_DIR="$TEST_ROOT" + # shellcheck source=utils/colors.sh + source "$TEST_ROOT/utils/colors.sh" + # shellcheck source=utils/output_redirect.sh + source "$TEST_ROOT/utils/output_redirect.sh" + # shellcheck source=utils/update_env_var.sh + source "$TEST_ROOT/utils/update_env_var.sh" + # shellcheck source=utils/install_traefik.sh + source "$TEST_ROOT/utils/install_traefik.sh" + "$@" "$WORK_DIR/traefik-config.env" + ) >"$WORK_DIR/output" 2>&1 || fail "$1 exited $?: $(cat "$WORK_DIR/output")" +} + +gateway_values() { + sed -n '/^ name: gateway$/,$p' "$WORK_DIR/server-side-applied" 2>/dev/null | sed -n '/valuesContent/,$p' +} + +test_traefik_binds_the_host_ports_itself() { + ports_file HTTP_PORT=8080 HTTPS_PORT=8443 + run_install install_traefik + assert_contains "$(gateway_values)" "web: {exposedPort: 8080, hostPort: 8080}" + assert_contains "$(gateway_values)" "websecure: {exposedPort: 8443, hostPort: 8443}" + assert_not_contains "$(gateway_values)" "LoadBalancer" +} + +test_traefik_waits_behind_servicelb_while_ingress_nginx_holds_the_ports() { + nginx_ports 80 443 + ports_file HTTP_PORT=80 HTTPS_PORT=443 + run_install install_traefik + assert_contains "$(gateway_values)" "spec: {type: LoadBalancer, externalTrafficPolicy: Local}" + assert_not_contains "$(gateway_values)" "hostPort" +} + +test_traefik_left_behind_servicelb_moves_onto_the_host_ports() { + echo LoadBalancer >"$WORK_DIR/traefik-type" + ports_file HTTP_PORT=80 HTTPS_PORT=443 + run_install move_traefik_to_host_ports + assert_contains "$(gateway_values)" "web: {exposedPort: 80, hostPort: 80}" +} + +test_traefik_already_on_the_host_ports_is_not_reinstalled() { + echo ClusterIP >"$WORK_DIR/traefik-type" + run_install move_traefik_to_host_ports + [ ! -f "$WORK_DIR/server-side-applied" ] || fail "the gateway chart was applied again" +} + run_tests From 54fe62a881ac50fb9f971ad562d5017e7aee17a7 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 16:38:38 -0500 Subject: [PATCH 2/3] fix: Traefik binds the host's ports itself, so backends see the client's address k3s ServiceLB masquerades, so behind it every request came from its pod whenever traffic arrived on an address other than the node's InternalIP. --- charts/gateway/values.yaml | 11 ++++++++-- dev.sh | 1 + tests/gateway/upgrade_test.sh | 5 ++++- update.sh | 1 + utils/install_traefik.sh | 38 ++++++++++++++++++++++++++--------- utils/retire_ingress_nginx.sh | 29 +++++++++++++------------- 6 files changed, 58 insertions(+), 27 deletions(-) diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index ccb8c6f..7cf3817 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -80,10 +80,17 @@ traefik: readTimeout: 0 writeTimeout: 0 + # install_traefik binds the host's ports straight to the pod (hostPort): + # k3s ServiceLB masquerades, so behind it every request came from its pod. service: spec: - type: LoadBalancer - externalTrafficPolicy: Local + type: ClusterIP + + # Old and new pods can't hold the same host port at once. + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 resources: requests: diff --git a/dev.sh b/dev.sh index d31fce0..cb50854 100755 --- a/dev.sh +++ b/dev.sh @@ -58,6 +58,7 @@ kubectl config use-context k3d-5stack-dev install_traefik "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1 retire_ingress_nginx --immediately || exit 1 +move_traefik_to_host_ports "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1 echo "Labeling node..." kubectl label node $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') 5stack-api=true 5stack-hasura=true 5stack-minio=true 5stack-rustfs=true 5stack-timescaledb=true 5stack-redis=true 5stack-typesense=true 5stack-web=true 5stack-dev-server=true 5stack-mediamtx=true diff --git a/tests/gateway/upgrade_test.sh b/tests/gateway/upgrade_test.sh index 0ab0f7e..b28ba0a 100644 --- a/tests/gateway/upgrade_test.sh +++ b/tests/gateway/upgrade_test.sh @@ -3,7 +3,8 @@ # An install that is still on ingress-nginx -- the controller, its admission # webhook, the panel's Ingresses as they were, and another app's nginx-class # Ingress with forward-auth -- taken through update.sh's routing steps: -# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx. +# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx, +# move_traefik_to_host_ports. # shellcheck source=tests/gateway/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" @@ -200,6 +201,7 @@ test_running_the_update_again_changes_nothing() { install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "install_traefik failed" routing_layer https || fail "the overlay did not apply" retire_ingress_nginx || fail "retire_ingress_nginx failed" + move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "move_traefik_to_host_ports failed" call https "$API" /me expect_code 200 expect_other_app_routes_with_its_auth @@ -222,6 +224,7 @@ sleep 2 install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1 routing_layer https || exit 1 retire_ingress_nginx || exit 1 +move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1 sleep 5 client touch /tmp/stop for _ in $(seq 1 10); do diff --git a/update.sh b/update.sh index 9e71aad..b42dde5 100755 --- a/update.sh +++ b/update.sh @@ -117,6 +117,7 @@ if [ "$REVERSE_PROXY" != true ]; then fi retire_ingress_nginx || die "Traefik did not take over ports 80/443" +move_traefik_to_host_ports || die "Traefik did not take over the host's ports; it is still serving behind ServiceLB" if [ "$VAULT_MANAGER" = true ]; then step "Syncing Vault secrets" diff --git a/utils/install_traefik.sh b/utils/install_traefik.sh index c8b22e3..94b8e31 100644 --- a/utils/install_traefik.sh +++ b/utils/install_traefik.sh @@ -39,7 +39,7 @@ carry_over_ingress_nginx_ports() { install_traefik() { local ports_env="${1:-$PANEL_DIR/overlays/config/traefik-config.env}" - local minor http_port https_port crd + local minor http_port https_port crd binding local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -73,16 +73,26 @@ install_traefik() { fi done - # The Service is externalTrafficPolicy: Local, so only the node running - # Traefik answers on these ports -- and DNS points at the server node, not - # at a game node that joined later. + # While ingress-nginx's ServiceLB pods hold the ports, Traefik waits behind + # its own until retire_ingress_nginx hands them over, so a stalled handover + # can still give them back; move_traefik_to_host_ports finishes the move. + if kubectl "${kc[@]}" -n ingress-nginx get service ingress-nginx-controller >/dev/null 2>&1; then + binding=" ports: + web: {exposedPort: $http_port} + websecure: {exposedPort: $https_port} + service: + spec: {type: LoadBalancer, externalTrafficPolicy: Local}" + else + binding=" ports: + web: {exposedPort: $http_port, hostPort: $http_port} + websecure: {exposedPort: $https_port, hostPort: $https_port}" + fi + + # Only the node running Traefik listens on these ports, and DNS points at + # the server node, not at a game node that joined later. install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat </dev/null)" != LoadBalancer ]; then + return 0 + fi + install_traefik "$@" +} + # The cluster-scoped objects a chart's templates define, as kind/name. chart_objects() { awk '/^kind: / { kind = $2; next } diff --git a/utils/retire_ingress_nginx.sh b/utils/retire_ingress_nginx.sh index 1e42e78..18990d2 100644 --- a/utils/retire_ingress_nginx.sh +++ b/utils/retire_ingress_nginx.sh @@ -48,7 +48,7 @@ YAML # retires ingress-nginx before Tilt has applied them, and a dev cluster has no # visitors to keep serving. retire_ingress_nginx() { - local deadline moved="" http_port="" https_port="" + local deadline http_port="" https_port="" local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -70,22 +70,21 @@ retire_ingress_nginx() { -o jsonpath='{.spec.ports[?(@.name=="https")].port}' 2>/dev/null)" echo "Moving ports 80/443 from ingress-nginx to Traefik..." output_redirect kubectl "${kc[@]}" delete service ingress-nginx-controller -n ingress-nginx --wait=true - moved=1 - fi - deadline=$((SECONDS + ${TRAEFIK_ADDRESS_TIMEOUT:-180})) - until [ -n "$(kubectl "${kc[@]}" get service traefik -n traefik -o jsonpath='{.status.loadBalancer.ingress}' 2>/dev/null)" ]; do - if [ "$SECONDS" -ge "$deadline" ]; then - err "Traefik's LoadBalancer Service never got an address:" - kubectl "${kc[@]}" get service traefik -n traefik >&2 - kubectl "${kc[@]}" get pods -A -l svccontroller.k3s.cattle.io/svcname=traefik >&2 - if [ -n "$moved" ] && restore_ingress_nginx_service "${http_port:-80}" "${https_port:-443}"; then - err "ingress-nginx has its ports back and keeps serving the panel; re-run ./update.sh once Traefik's Service can get an address" + deadline=$((SECONDS + ${TRAEFIK_ADDRESS_TIMEOUT:-180})) + until [ -n "$(kubectl "${kc[@]}" get service traefik -n traefik -o jsonpath='{.status.loadBalancer.ingress}' 2>/dev/null)" ]; do + if [ "$SECONDS" -ge "$deadline" ]; then + err "Traefik's LoadBalancer Service never got an address:" + kubectl "${kc[@]}" get service traefik -n traefik >&2 + kubectl "${kc[@]}" get pods -A -l svccontroller.k3s.cattle.io/svcname=traefik >&2 + if restore_ingress_nginx_service "${http_port:-80}" "${https_port:-443}"; then + err "ingress-nginx has its ports back and keeps serving the panel; re-run ./update.sh once Traefik's Service can get an address" + fi + return 1 fi - return 1 - fi - sleep 2 - done + sleep 2 + done + fi # Orphaned, not cascaded: on an old install ingress-shim may still own the # 5stack-ssl Certificate through one of these. From c177a9b826c629c5793dc49c61dc9cf5bc05df8a Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Tue, 3 Nov 2026 16:58:42 -0500 Subject: [PATCH 3/3] fix: pull a new Traefik image onto its node before the rollout replaces the pod With host ports the old pod stops before its replacement exists, so a slow or refused pull would leave nothing serving. Failed kubectl lookups now stop the install instead of being read as answers. --- base/gateway/gateway.yaml | 2 +- tests/gateway/servicelb_upgrade_test.sh | 18 ++++++-- tests/scripts/stub/kubectl | 18 +++++++- tests/scripts/traefik_install_test.sh | 59 ++++++++++++++++++++++++- update.sh | 2 +- utils/install_traefik.sh | 42 ++++++++++++++++-- 6 files changed, 130 insertions(+), 11 deletions(-) diff --git a/base/gateway/gateway.yaml b/base/gateway/gateway.yaml index 0816337..ea15725 100644 --- a/base/gateway/gateway.yaml +++ b/base/gateway/gateway.yaml @@ -2,7 +2,7 @@ # inherit the hostname from. These are the plain-HTTP listeners of a # REVERSE_PROXY install; overlays/http/gateway-https-patch.yaml turns them into # HTTPS ones. Ports are Traefik's entrypoints (charts/gateway), not the node's: -# HTTP_PORT and HTTPS_PORT only move the LoadBalancer Service. The listeners for +# HTTP_PORT and HTTPS_PORT only move Traefik's host ports. The listeners for # every other host are on the chart's shared Gateway, traefik/traefik-gateway: # web (HTTP) and websecure (HTTPS on a self-signed certificate). apiVersion: gateway.networking.k8s.io/v1 diff --git a/tests/gateway/servicelb_upgrade_test.sh b/tests/gateway/servicelb_upgrade_test.sh index 556a60d..6f1d6c6 100644 --- a/tests/gateway/servicelb_upgrade_test.sh +++ b/tests/gateway/servicelb_upgrade_test.sh @@ -1,8 +1,8 @@ #!/bin/bash -# An install whose Traefik still sits behind k3s ServiceLB, as the panel ran it -# before Traefik bound the host's ports itself, taken through update.sh's -# install_traefik. +# An install whose Traefik still sits behind k3s ServiceLB on an older image, as +# the panel ran it before Traefik bound the host's ports itself, taken through +# update.sh's install_traefik. # shellcheck source=tests/gateway/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" @@ -20,6 +20,9 @@ install_traefik_behind_servicelb() { done install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat <<'EOF' traefik: + versionOverride: v3.7.15 + image: + digest: sha256:0f6150202bc1fed72a7bd2107460eaa510e5b8a905f3fdb5b84f2c8619c33368 service: spec: {type: LoadBalancer, externalTrafficPolicy: Local} updateStrategy: @@ -58,6 +61,14 @@ test_servicelb_no_longer_fronts_traefik() { [ -z "$(kubectl -n kube-system get daemonset -l svccontroller.k3s.cattle.io/svcname=traefik -o name)" ] || fail "svclb-traefik is still running" } +test_traefik_runs_the_new_image_pulled_beforehand() { + local want + want="docker.io/traefik@$(sed -n 's/^ *digest: *\(sha256:[0-9a-f]*\).*/\1/p' "$PANEL_DIR/charts/gateway/values.yaml")" + [ "$(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')" = "$want" ] || + fail "Traefik runs $(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')" + kubectl -n traefik wait --for=delete pod/traefik-pull --timeout=60s >/dev/null 2>&1 || fail "the pull pod was left behind" +} + test_the_client_ip_reaches_the_backend() { call https "$API" /me expect_code 200 @@ -102,6 +113,7 @@ POLL="$(client cat /tmp/poll.json)" for name in \ test_the_move_costs_seconds \ test_servicelb_no_longer_fronts_traefik \ + test_traefik_runs_the_new_image_pulled_beforehand \ test_the_client_ip_reaches_the_backend \ test_running_the_update_again_changes_nothing; do run_test "$name" diff --git a/tests/scripts/stub/kubectl b/tests/scripts/stub/kubectl index 450f482..4859f10 100755 --- a/tests/scripts/stub/kubectl +++ b/tests/scripts/stub/kubectl @@ -27,7 +27,16 @@ case "$args" in *" get crd helmcharts.helm.cattle.io "*) ;; *" apply --server-side "*) if [ -n "$file" ]; then cat "$file" >>"$WORK_DIR/server-side-applied"; fi ;; *" apply -f - "*) cat >"$WORK_DIR/applied" ;; - *" get service traefik "*'{.spec.type}'*) if [ -f "$WORK_DIR/traefik-type" ]; then cat "$WORK_DIR/traefik-type"; fi ;; + *" get service traefik "*'{.spec.type}'*) + if [ -f "$WORK_DIR/traefik-service-error" ]; then + echo "The connection to the server was refused" >&2 + exit 1 + fi + if [ -f "$WORK_DIR/traefik-type" ]; then cat "$WORK_DIR/traefik-type"; fi + ;; + *" get deployment traefik "*) if [ -f "$WORK_DIR/traefik-image" ]; then cat "$WORK_DIR/traefik-image"; fi ;; + *" get pods -l app.kubernetes.io/name=traefik "*) if [ -f "$WORK_DIR/traefik-node" ]; then cat "$WORK_DIR/traefik-node"; fi ;; + *" wait pod/traefik-pull "*) [ ! -f "$WORK_DIR/pull-fails" ] || exit 1 ;; *" get service traefik "*) if [ -f "$WORK_DIR/traefik-address" ]; then cat "$WORK_DIR/traefik-address"; fi ;; *" get helmchart "*) echo 1 ;; *" get gateway traefik-gateway "*) [ -f "$WORK_DIR/shared-gateway" ] || exit 1 ;; @@ -83,7 +92,14 @@ case "$args" in *'name=="https"'*) port_file="$WORK_DIR/nginx-https-port" ;; *) port_file="$WORK_DIR/nginx-http-port" ;; esac + if [ -f "$WORK_DIR/nginx-error" ]; then + echo "The connection to the server was refused" >&2 + exit 1 + fi if [ ! -f "$port_file" ]; then + case "$args" in + *" --ignore-not-found "*) exit 0 ;; + esac echo 'Error from server (NotFound): services "ingress-nginx-controller" not found' >&2 exit 1 fi diff --git a/tests/scripts/traefik_install_test.sh b/tests/scripts/traefik_install_test.sh index dd8d9f0..123b349 100644 --- a/tests/scripts/traefik_install_test.sh +++ b/tests/scripts/traefik_install_test.sh @@ -268,7 +268,7 @@ test_retiring_with_no_ingress_nginx_waits_for_no_address() { assert_not_contains "$(cat "$WORK_DIR/calls")" "loadBalancer.ingress" } -run_install() { +try_install() { ( export PANEL_DIR="$TEST_ROOT" # shellcheck source=utils/colors.sh @@ -280,7 +280,15 @@ run_install() { # shellcheck source=utils/install_traefik.sh source "$TEST_ROOT/utils/install_traefik.sh" "$@" "$WORK_DIR/traefik-config.env" - ) >"$WORK_DIR/output" 2>&1 || fail "$1 exited $?: $(cat "$WORK_DIR/output")" + ) >"$WORK_DIR/output" 2>&1 +} + +run_install() { + try_install "$@" || fail "$1 exited $?: $(cat "$WORK_DIR/output")" +} + +chart_image() { + printf 'docker.io/traefik@%s' "$(sed -n 's/^ *digest: *\(sha256:[0-9a-f]*\).*/\1/p' "$TEST_ROOT/charts/gateway/values.yaml")" } gateway_values() { @@ -316,4 +324,51 @@ test_traefik_already_on_the_host_ports_is_not_reinstalled() { [ ! -f "$WORK_DIR/server-side-applied" ] || fail "the gateway chart was applied again" } +test_a_failed_service_lookup_fails_the_move() { + touch "$WORK_DIR/traefik-service-error" + if try_install move_traefik_to_host_ports; then + fail "move_traefik_to_host_ports succeeded without knowing where Traefik is" + fi +} + +test_a_failed_ingress_nginx_lookup_stops_the_install() { + touch "$WORK_DIR/nginx-error" + if try_install install_traefik; then + fail "install_traefik succeeded without knowing whether ingress-nginx holds the ports" + fi + assert_eq "$(gateway_values)" "" +} + +test_a_new_image_is_pulled_onto_traefiks_node_before_the_upgrade() { + echo docker.io/traefik@sha256:0000 >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + run_install install_traefik + assert_contains "$(grep ' run traefik-pull ' "$WORK_DIR/calls")" "--image=$(chart_image) --restart=Never --overrides={\"spec\":{\"nodeName\":\"node-a\"" + [ "$(grep -n ' run traefik-pull ' "$WORK_DIR/calls" | cut -d: -f1)" -lt "$(grep -n ' apply --server-side ' "$WORK_DIR/calls" | tail -n 1 | cut -d: -f1)" ] || + fail "the image was pulled after the upgrade" +} + +test_a_refused_pull_leaves_traefik_alone() { + echo docker.io/traefik@sha256:0000 >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + touch "$WORK_DIR/pull-fails" + if try_install install_traefik; then + fail "install_traefik succeeded although the image never arrived" + fi + assert_eq "$(gateway_values)" "" + assert_contains "$(cat "$WORK_DIR/output")" "keeps running its current image" +} + +test_traefik_already_on_the_image_pulls_nothing() { + chart_image >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + run_install install_traefik + assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" +} + +test_a_first_install_pulls_nothing() { + run_install install_traefik + assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" +} + run_tests diff --git a/update.sh b/update.sh index b42dde5..f963857 100755 --- a/update.sh +++ b/update.sh @@ -117,7 +117,7 @@ if [ "$REVERSE_PROXY" != true ]; then fi retire_ingress_nginx || die "Traefik did not take over ports 80/443" -move_traefik_to_host_ports || die "Traefik did not take over the host's ports; it is still serving behind ServiceLB" +move_traefik_to_host_ports || die "Traefik did not take over the host's ports" if [ "$VAULT_MANAGER" = true ]; then step "Syncing Vault secrets" diff --git a/utils/install_traefik.sh b/utils/install_traefik.sh index 94b8e31..23d8145 100644 --- a/utils/install_traefik.sh +++ b/utils/install_traefik.sh @@ -39,7 +39,7 @@ carry_over_ingress_nginx_ports() { install_traefik() { local ports_env="${1:-$PANEL_DIR/overlays/config/traefik-config.env}" - local minor http_port https_port crd binding + local minor http_port https_port crd binding nginx local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -76,7 +76,8 @@ install_traefik() { # While ingress-nginx's ServiceLB pods hold the ports, Traefik waits behind # its own until retire_ingress_nginx hands them over, so a stalled handover # can still give them back; move_traefik_to_host_ports finishes the move. - if kubectl "${kc[@]}" -n ingress-nginx get service ingress-nginx-controller >/dev/null 2>&1; then + nginx="$(kubectl "${kc[@]}" -n ingress-nginx get service ingress-nginx-controller --ignore-not-found -o name)" || return 1 + if [ -n "$nginx" ]; then binding=" ports: web: {exposedPort: $http_port} websecure: {exposedPort: $https_port} @@ -88,6 +89,8 @@ install_traefik() { websecure: {exposedPort: $https_port, hostPort: $https_port}" fi + pull_traefik_image || return 1 + # Only the node running Traefik listens on these ports, and DNS points at # the server node, not at a game node that joined later. install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat </dev/null)" = "$image" ]; then + return 0 + fi + node="$(kubectl "${kc[@]}" -n traefik get pods -l app.kubernetes.io/name=traefik -o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null)" + if [ -z "$node" ]; then + return 0 + fi + + echo "Pulling $image onto $node..." + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --ignore-not-found --wait=true + if ! output_redirect kubectl "${kc[@]}" -n traefik run traefik-pull --image="$image" --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"$node\",\"tolerations\":[{\"operator\":\"Exists\"}]}}" \ + --command -- traefik version || + ! output_redirect kubectl "${kc[@]}" -n traefik wait pod/traefik-pull --timeout=300s \ + --for=jsonpath='{.status.phase}'=Succeeded; then + err "could not pull $image onto $node; Traefik keeps running its current image" + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --ignore-not-found --wait=false + return 1 + fi + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --wait=false +} + move_traefik_to_host_ports() { + local type local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") - if [ "$(kubectl "${kc[@]}" -n traefik get service traefik -o jsonpath='{.spec.type}' 2>/dev/null)" != LoadBalancer ]; then + type="$(kubectl "${kc[@]}" -n traefik get service traefik -o jsonpath='{.spec.type}')" || return 1 + if [ "$type" != LoadBalancer ]; then return 0 fi install_traefik "$@"