fix: Traefik binds the host's ports, so backends see real client addresses

.github/workflows/gateway.yml

@@ -64,6 +64,12 @@ jobs:
       - uses: actions/checkout@v4
       - run: bash tests/gateway/upgrade_test.sh
 
+  upgrade-from-servicelb:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: bash tests/gateway/servicelb_upgrade_test.sh
+
   cert-manager:
     runs-on: ubuntu-latest
     steps:

base/gateway/gateway.yaml

@@ -2,7 +2,7 @@
 # inherit the hostname from. These are the plain-HTTP listeners of a
 # REVERSE_PROXY install; overlays/http/gateway-https-patch.yaml turns them into
 # HTTPS ones. Ports are Traefik's entrypoints (charts/gateway), not the node's:
-# HTTP_PORT and HTTPS_PORT only move the LoadBalancer Service. The listeners for
+# HTTP_PORT and HTTPS_PORT only move Traefik's host ports. The listeners for
 # every other host are on the chart's shared Gateway, traefik/traefik-gateway:
 # web (HTTP) and websecure (HTTPS on a self-signed certificate).
 apiVersion: gateway.networking.k8s.io/v1

charts/gateway/values.yaml

@@ -80,10 +80,17 @@ traefik:
           readTimeout: 0
           writeTimeout: 0
 
+  # install_traefik binds the host's ports straight to the pod (hostPort):
+  # k3s ServiceLB masquerades, so behind it every request came from its pod.
   service:
     spec:
-      type: LoadBalancer
-      externalTrafficPolicy: Local
+      type: ClusterIP
+
+  # Old and new pods can't hold the same host port at once.
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0
 
   resources:
     requests:

dev.sh

@@ -58,6 +58,7 @@ kubectl config use-context k3d-5stack-dev
 
 install_traefik "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1
 retire_ingress_nginx --immediately || exit 1
+move_traefik_to_host_ports "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1
 
 echo "Labeling node..."
 kubectl label node $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') 5stack-api=true 5stack-hasura=true 5stack-minio=true 5stack-rustfs=true 5stack-timescaledb=true 5stack-redis=true 5stack-typesense=true 5stack-web=true 5stack-dev-server=true 5stack-mediamtx=true

tests/gateway/lib.sh

@@ -2,7 +2,7 @@
 
 # k3s in Docker, with the panel's routing layer installed by the same functions
 # update.sh calls, in front of stub backends named like the real Services.
-# Requests come from a separate client container on the same Docker network,
+# Requests come from a separate client container on a second Docker network,
 # so the client IP the backends see is a real peer's. One cluster at a time;
 # everything is named after $PREFIX and removed on exit.
 
@@ -22,7 +22,7 @@ export KUBECONFIG PANEL_DIR DEBUG
 
 cleanup() {
     docker rm -f -v "$PREFIX-k3s" "$PREFIX-client" >/dev/null 2>&1
-    docker network rm "$PREFIX-net" >/dev/null 2>&1
+    docker network rm "$PREFIX-net" "$PREFIX-public" >/dev/null 2>&1
     rm -rf "$WORK_DIR"
 }
 
@@ -70,13 +70,22 @@ start_cluster() {
         sleep 2
     done
     kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null || return 1
-    NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-k3s")"
 
     docker run -d --name "$PREFIX-client" --network "$PREFIX-net" "$PYTHON_IMAGE" sleep infinity >/dev/null || return 1
     client apk add --no-cache -q curl jq openssl >/dev/null || return 1
     docker cp "$HERE/client.py" "$PREFIX-client:/client.py" >/dev/null
+
+    # Clients reach the node on an address k3s doesn't know as its InternalIP,
+    # as a server's public one is when k3s runs on a private or VPN address.
+    # kube-proxy answers the InternalIP itself, which hid that ServiceLB
+    # masquerades everything else.
+    docker network create "$PREFIX-public" >/dev/null || return 1
+    docker network connect "$PREFIX-public" "$PREFIX-k3s" || return 1
+    docker network connect "$PREFIX-public" "$PREFIX-client" || return 1
+    NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-k3s")"
+    [ "$NODE_IP" != "$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" ] || return 1
     # shellcheck disable=SC2034
-    CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-client")"
+    CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-client")"
 }
 
 set_env() {

tests/gateway/servicelb_upgrade_test.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+# An install whose Traefik still sits behind k3s ServiceLB on an older image, as
+# the panel ran it before Traefik bound the host's ports itself, taken through
+# update.sh's install_traefik.
+
+# shellcheck source=tests/gateway/lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+trap cleanup EXIT
+
+API="api.$DOMAIN"
+PORTS_ENV="$PANEL_DIR/overlays/config/traefik-config.env"
+
+install_traefik_behind_servicelb() {
+    local crd
+    install_helm_chart gateway-crds "$PANEL_DIR/charts/gateway-crds" "" >/dev/null || return 1
+    for crd in gatewayclasses.gateway.networking.k8s.io gateways.gateway.networking.k8s.io \
+        httproutes.gateway.networking.k8s.io middlewares.traefik.io; do
+        kubectl wait --for=condition=Established "crd/$crd" --timeout=120s >/dev/null || return 1
+    done
+    install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat <<'EOF'
+traefik:
+  versionOverride: v3.7.15
+  image:
+    digest: sha256:0f6150202bc1fed72a7bd2107460eaa510e5b8a905f3fdb5b84f2c8619c33368
+  service:
+    spec: {type: LoadBalancer, externalTrafficPolicy: Local}
+  updateStrategy:
+    rollingUpdate: {maxUnavailable: 0, maxSurge: 1}
+EOF
+)" >/dev/null || return 1
+    kubectl -n traefik rollout status deployment/traefik --timeout=300s >/dev/null || return 1
+    for _ in $(seq 1 90); do
+        [ -n "$(kubectl -n traefik get service traefik -o jsonpath='{.status.loadBalancer.ingress}')" ] && return 0
+        sleep 2
+    done
+    return 1
+}
+
+traefik_pod() {
+    kubectl -n traefik get pods -l app.kubernetes.io/name=traefik -o jsonpath='{.items[*].metadata.uid}'
+}
+
+test_behind_servicelb_every_request_comes_from_its_pod() {
+    wait_for 200 https "$API" /me
+    case "$(json '.headers["x-forwarded-for"]')" in
+        10.42.*) ;;
+        *) fail "expected a ServiceLB pod address, got $(json '.headers["x-forwarded-for"]')" ;;
+    esac
+}
+
+test_the_move_costs_seconds() {
+    local longest
+    longest="$(jq -r .longest_outage <<<"$POLL")"
+    echo "      outage while Traefik moved: ${longest}s ($(jq -r '"\(.failed) of \(.requests)"' <<<"$POLL") requests failed)"
+    awk -v t="$longest" 'BEGIN { exit !(t < 30) }' || fail "$POLL"
+}
+
+test_servicelb_no_longer_fronts_traefik() {
+    [ "$(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')" = ClusterIP ] || fail "Traefik's Service is still $(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')"
+    [ -z "$(kubectl -n kube-system get daemonset -l svccontroller.k3s.cattle.io/svcname=traefik -o name)" ] || fail "svclb-traefik is still running"
+}
+
+test_traefik_runs_the_new_image_pulled_beforehand() {
+    local want
+    want="docker.io/traefik@$(sed -n 's/^ *digest: *\(sha256:[0-9a-f]*\).*/\1/p' "$PANEL_DIR/charts/gateway/values.yaml")"
+    [ "$(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')" = "$want" ] ||
+        fail "Traefik runs $(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')"
+    kubectl -n traefik wait --for=delete pod/traefik-pull --timeout=60s >/dev/null 2>&1 || fail "the pull pod was left behind"
+}
+
+test_the_client_ip_reaches_the_backend() {
+    call https "$API" /me
+    expect_code 200
+    expect_json '.headers["x-forwarded-for"]' "$CLIENT_IP"
+    expect_json '.headers["x-real-ip"]' "$CLIENT_IP"
+    call http "$API" /me
+    expect_code 308
+}
+
+test_running_the_update_again_changes_nothing() {
+    local before
+    before="$(traefik_pod)"
+    install_traefik "$PORTS_ENV" >/dev/null || fail "install_traefik failed"
+    move_traefik_to_host_ports "$PORTS_ENV" >/dev/null || fail "move_traefik_to_host_ports failed"
+    [ "$(traefik_pod)" = "$before" ] || fail "Traefik was restarted"
+    call https "$API" /me
+    expect_code 200
+}
+
+start_cluster || exit 1
+make_panel || exit 1
+install_traefik_behind_servicelb || exit 1
+deploy_panel_stubs || exit 1
+routing_layer https || exit 1
+tls_secret 5stack-ssl "$DOMAIN" "$API" "demos.$DOMAIN" "ws.$DOMAIN" "tv.$DOMAIN" "search.$DOMAIN" "console.$DOMAIN" || exit 1
+tls_secret 5stack-mediamtx-ssl "hls.$DOMAIN" || exit 1
+
+run_test test_behind_servicelb_every_request_comes_from_its_pod
+
+client rm -f /tmp/stop
+docker exec -d "$PREFIX-client" sh -c "python3 /client.py poll $NODE_IP $API /me /tmp/stop >/tmp/poll.json"
+sleep 2
+install_traefik "$PORTS_ENV" || exit 1
+sleep 5
+client touch /tmp/stop
+for _ in $(seq 1 10); do
+    client test -s /tmp/poll.json && break
+    sleep 1
+done
+POLL="$(client cat /tmp/poll.json)"
+
+for name in \
+    test_the_move_costs_seconds \
+    test_servicelb_no_longer_fronts_traefik \
+    test_traefik_runs_the_new_image_pulled_beforehand \
+    test_the_client_ip_reaches_the_backend \
+    test_running_the_update_again_changes_nothing; do
+    run_test "$name"
+done
+
+summary

tests/gateway/upgrade_test.sh

@@ -3,7 +3,8 @@
 # An install that is still on ingress-nginx -- the controller, its admission
 # webhook, the panel's Ingresses as they were, and another app's nginx-class
 # Ingress with forward-auth -- taken through update.sh's routing steps:
-# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx.
+# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx,
+# move_traefik_to_host_ports.
 
 # shellcheck source=tests/gateway/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
@@ -200,6 +201,7 @@ test_running_the_update_again_changes_nothing() {
     install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "install_traefik failed"
     routing_layer https || fail "the overlay did not apply"
     retire_ingress_nginx || fail "retire_ingress_nginx failed"
+    move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "move_traefik_to_host_ports failed"
     call https "$API" /me
     expect_code 200
     expect_other_app_routes_with_its_auth
@@ -222,6 +224,7 @@ sleep 2
 install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1
 routing_layer https || exit 1
 retire_ingress_nginx || exit 1
+move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1
 sleep 5
 client touch /tmp/stop
 for _ in $(seq 1 10); do

tests/scripts/stub/kubectl

@@ -3,6 +3,7 @@
 echo "$*" >>"$WORK_DIR/calls"
 
 args=" $* "
+file=""
 pod=""
 container=""
 previous=false
@@ -14,13 +15,28 @@ while [ $# -gt 0 ]; do
             shift
             ;;
         --previous) previous=true ;;
+        -f)
+            file="$2"
+            shift
+            ;;
     esac
     shift
 done
 
 case "$args" in
-    *" get crd helmcharts.helm.cattle.io "* | *" apply --server-side "*) ;;
+    *" get crd helmcharts.helm.cattle.io "*) ;;
+    *" apply --server-side "*) if [ -n "$file" ]; then cat "$file" >>"$WORK_DIR/server-side-applied"; fi ;;
     *" apply -f - "*) cat >"$WORK_DIR/applied" ;;
+    *" get service traefik "*'{.spec.type}'*)
+        if [ -f "$WORK_DIR/traefik-service-error" ]; then
+            echo "The connection to the server was refused" >&2
+            exit 1
+        fi
+        if [ -f "$WORK_DIR/traefik-type" ]; then cat "$WORK_DIR/traefik-type"; fi
+        ;;
+    *" get deployment traefik "*) if [ -f "$WORK_DIR/traefik-image" ]; then cat "$WORK_DIR/traefik-image"; fi ;;
+    *" get pods -l app.kubernetes.io/name=traefik "*) if [ -f "$WORK_DIR/traefik-node" ]; then cat "$WORK_DIR/traefik-node"; fi ;;
+    *" wait pod/traefik-pull "*) [ ! -f "$WORK_DIR/pull-fails" ] || exit 1 ;;
     *" get service traefik "*) if [ -f "$WORK_DIR/traefik-address" ]; then cat "$WORK_DIR/traefik-address"; fi ;;
     *" get helmchart "*) echo 1 ;;
     *" get gateway traefik-gateway "*) [ -f "$WORK_DIR/shared-gateway" ] || exit 1 ;;
@@ -76,7 +92,14 @@ case "$args" in
             *'name=="https"'*) port_file="$WORK_DIR/nginx-https-port" ;;
             *) port_file="$WORK_DIR/nginx-http-port" ;;
         esac
+        if [ -f "$WORK_DIR/nginx-error" ]; then
+            echo "The connection to the server was refused" >&2
+            exit 1
+        fi
         if [ ! -f "$port_file" ]; then
+            case "$args" in
+                *" --ignore-not-found "*) exit 0 ;;
+            esac
             echo 'Error from server (NotFound): services "ingress-nginx-controller" not found' >&2
             exit 1
         fi