diff --git a/.github/workflows/gateway.yml b/.github/workflows/gateway.yml index 7cf8de2..aba9fd9 100644 --- a/.github/workflows/gateway.yml +++ b/.github/workflows/gateway.yml @@ -64,6 +64,12 @@ jobs: - uses: actions/checkout@v4 - run: bash tests/gateway/upgrade_test.sh + upgrade-from-servicelb: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: bash tests/gateway/servicelb_upgrade_test.sh + cert-manager: runs-on: ubuntu-latest steps: diff --git a/base/gateway/gateway.yaml b/base/gateway/gateway.yaml index 0816337..ea15725 100644 --- a/base/gateway/gateway.yaml +++ b/base/gateway/gateway.yaml @@ -2,7 +2,7 @@ # inherit the hostname from. These are the plain-HTTP listeners of a # REVERSE_PROXY install; overlays/http/gateway-https-patch.yaml turns them into # HTTPS ones. Ports are Traefik's entrypoints (charts/gateway), not the node's: -# HTTP_PORT and HTTPS_PORT only move the LoadBalancer Service. The listeners for +# HTTP_PORT and HTTPS_PORT only move Traefik's host ports. The listeners for # every other host are on the chart's shared Gateway, traefik/traefik-gateway: # web (HTTP) and websecure (HTTPS on a self-signed certificate). apiVersion: gateway.networking.k8s.io/v1 diff --git a/charts/gateway/values.yaml b/charts/gateway/values.yaml index ccb8c6f..7cf3817 100644 --- a/charts/gateway/values.yaml +++ b/charts/gateway/values.yaml @@ -80,10 +80,17 @@ traefik: readTimeout: 0 writeTimeout: 0 + # install_traefik binds the host's ports straight to the pod (hostPort): + # k3s ServiceLB masquerades, so behind it every request came from its pod. service: spec: - type: LoadBalancer - externalTrafficPolicy: Local + type: ClusterIP + + # Old and new pods can't hold the same host port at once. + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + maxSurge: 0 resources: requests: diff --git a/dev.sh b/dev.sh index d31fce0..cb50854 100755 --- a/dev.sh +++ b/dev.sh @@ -58,6 +58,7 @@ kubectl config use-context k3d-5stack-dev install_traefik "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1 retire_ingress_nginx --immediately || exit 1 +move_traefik_to_host_ports "$PANEL_DIR/overlays/dev/config/traefik-config.env" || exit 1 echo "Labeling node..." kubectl label node $(kubectl get nodes -o jsonpath='{.items[0].metadata.name}') 5stack-api=true 5stack-hasura=true 5stack-minio=true 5stack-rustfs=true 5stack-timescaledb=true 5stack-redis=true 5stack-typesense=true 5stack-web=true 5stack-dev-server=true 5stack-mediamtx=true diff --git a/tests/gateway/lib.sh b/tests/gateway/lib.sh index 43fd7d5..2dd0d5f 100644 --- a/tests/gateway/lib.sh +++ b/tests/gateway/lib.sh @@ -2,7 +2,7 @@ # k3s in Docker, with the panel's routing layer installed by the same functions # update.sh calls, in front of stub backends named like the real Services. -# Requests come from a separate client container on the same Docker network, +# Requests come from a separate client container on a second Docker network, # so the client IP the backends see is a real peer's. One cluster at a time; # everything is named after $PREFIX and removed on exit. @@ -22,7 +22,7 @@ export KUBECONFIG PANEL_DIR DEBUG cleanup() { docker rm -f -v "$PREFIX-k3s" "$PREFIX-client" >/dev/null 2>&1 - docker network rm "$PREFIX-net" >/dev/null 2>&1 + docker network rm "$PREFIX-net" "$PREFIX-public" >/dev/null 2>&1 rm -rf "$WORK_DIR" } @@ -70,13 +70,22 @@ start_cluster() { sleep 2 done kubectl wait --for=condition=Ready node --all --timeout=5s >/dev/null || return 1 - NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-k3s")" docker run -d --name "$PREFIX-client" --network "$PREFIX-net" "$PYTHON_IMAGE" sleep infinity >/dev/null || return 1 client apk add --no-cache -q curl jq openssl >/dev/null || return 1 docker cp "$HERE/client.py" "$PREFIX-client:/client.py" >/dev/null + + # Clients reach the node on an address k3s doesn't know as its InternalIP, + # as a server's public one is when k3s runs on a private or VPN address. + # kube-proxy answers the InternalIP itself, which hid that ServiceLB + # masquerades everything else. + docker network create "$PREFIX-public" >/dev/null || return 1 + docker network connect "$PREFIX-public" "$PREFIX-k3s" || return 1 + docker network connect "$PREFIX-public" "$PREFIX-client" || return 1 + NODE_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-k3s")" + [ "$NODE_IP" != "$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" ] || return 1 # shellcheck disable=SC2034 - CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-net\").IPAddress}}" "$PREFIX-client")" + CLIENT_IP="$(docker inspect -f "{{(index .NetworkSettings.Networks \"$PREFIX-public\").IPAddress}}" "$PREFIX-client")" } set_env() { diff --git a/tests/gateway/servicelb_upgrade_test.sh b/tests/gateway/servicelb_upgrade_test.sh new file mode 100644 index 0000000..6f1d6c6 --- /dev/null +++ b/tests/gateway/servicelb_upgrade_test.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# An install whose Traefik still sits behind k3s ServiceLB on an older image, as +# the panel ran it before Traefik bound the host's ports itself, taken through +# update.sh's install_traefik. + +# shellcheck source=tests/gateway/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +trap cleanup EXIT + +API="api.$DOMAIN" +PORTS_ENV="$PANEL_DIR/overlays/config/traefik-config.env" + +install_traefik_behind_servicelb() { + local crd + install_helm_chart gateway-crds "$PANEL_DIR/charts/gateway-crds" "" >/dev/null || return 1 + for crd in gatewayclasses.gateway.networking.k8s.io gateways.gateway.networking.k8s.io \ + httproutes.gateway.networking.k8s.io middlewares.traefik.io; do + kubectl wait --for=condition=Established "crd/$crd" --timeout=120s >/dev/null || return 1 + done + install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat <<'EOF' +traefik: + versionOverride: v3.7.15 + image: + digest: sha256:0f6150202bc1fed72a7bd2107460eaa510e5b8a905f3fdb5b84f2c8619c33368 + service: + spec: {type: LoadBalancer, externalTrafficPolicy: Local} + updateStrategy: + rollingUpdate: {maxUnavailable: 0, maxSurge: 1} +EOF +)" >/dev/null || return 1 + kubectl -n traefik rollout status deployment/traefik --timeout=300s >/dev/null || return 1 + for _ in $(seq 1 90); do + [ -n "$(kubectl -n traefik get service traefik -o jsonpath='{.status.loadBalancer.ingress}')" ] && return 0 + sleep 2 + done + return 1 +} + +traefik_pod() { + kubectl -n traefik get pods -l app.kubernetes.io/name=traefik -o jsonpath='{.items[*].metadata.uid}' +} + +test_behind_servicelb_every_request_comes_from_its_pod() { + wait_for 200 https "$API" /me + case "$(json '.headers["x-forwarded-for"]')" in + 10.42.*) ;; + *) fail "expected a ServiceLB pod address, got $(json '.headers["x-forwarded-for"]')" ;; + esac +} + +test_the_move_costs_seconds() { + local longest + longest="$(jq -r .longest_outage <<<"$POLL")" + echo " outage while Traefik moved: ${longest}s ($(jq -r '"\(.failed) of \(.requests)"' <<<"$POLL") requests failed)" + awk -v t="$longest" 'BEGIN { exit !(t < 30) }' || fail "$POLL" +} + +test_servicelb_no_longer_fronts_traefik() { + [ "$(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')" = ClusterIP ] || fail "Traefik's Service is still $(kubectl -n traefik get service traefik -o jsonpath='{.spec.type}')" + [ -z "$(kubectl -n kube-system get daemonset -l svccontroller.k3s.cattle.io/svcname=traefik -o name)" ] || fail "svclb-traefik is still running" +} + +test_traefik_runs_the_new_image_pulled_beforehand() { + local want + want="docker.io/traefik@$(sed -n 's/^ *digest: *\(sha256:[0-9a-f]*\).*/\1/p' "$PANEL_DIR/charts/gateway/values.yaml")" + [ "$(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')" = "$want" ] || + fail "Traefik runs $(kubectl -n traefik get deployment traefik -o jsonpath='{.spec.template.spec.containers[0].image}')" + kubectl -n traefik wait --for=delete pod/traefik-pull --timeout=60s >/dev/null 2>&1 || fail "the pull pod was left behind" +} + +test_the_client_ip_reaches_the_backend() { + call https "$API" /me + expect_code 200 + expect_json '.headers["x-forwarded-for"]' "$CLIENT_IP" + expect_json '.headers["x-real-ip"]' "$CLIENT_IP" + call http "$API" /me + expect_code 308 +} + +test_running_the_update_again_changes_nothing() { + local before + before="$(traefik_pod)" + install_traefik "$PORTS_ENV" >/dev/null || fail "install_traefik failed" + move_traefik_to_host_ports "$PORTS_ENV" >/dev/null || fail "move_traefik_to_host_ports failed" + [ "$(traefik_pod)" = "$before" ] || fail "Traefik was restarted" + call https "$API" /me + expect_code 200 +} + +start_cluster || exit 1 +make_panel || exit 1 +install_traefik_behind_servicelb || exit 1 +deploy_panel_stubs || exit 1 +routing_layer https || exit 1 +tls_secret 5stack-ssl "$DOMAIN" "$API" "demos.$DOMAIN" "ws.$DOMAIN" "tv.$DOMAIN" "search.$DOMAIN" "console.$DOMAIN" || exit 1 +tls_secret 5stack-mediamtx-ssl "hls.$DOMAIN" || exit 1 + +run_test test_behind_servicelb_every_request_comes_from_its_pod + +client rm -f /tmp/stop +docker exec -d "$PREFIX-client" sh -c "python3 /client.py poll $NODE_IP $API /me /tmp/stop >/tmp/poll.json" +sleep 2 +install_traefik "$PORTS_ENV" || exit 1 +sleep 5 +client touch /tmp/stop +for _ in $(seq 1 10); do + client test -s /tmp/poll.json && break + sleep 1 +done +POLL="$(client cat /tmp/poll.json)" + +for name in \ + test_the_move_costs_seconds \ + test_servicelb_no_longer_fronts_traefik \ + test_traefik_runs_the_new_image_pulled_beforehand \ + test_the_client_ip_reaches_the_backend \ + test_running_the_update_again_changes_nothing; do + run_test "$name" +done + +summary diff --git a/tests/gateway/upgrade_test.sh b/tests/gateway/upgrade_test.sh index 0ab0f7e..b28ba0a 100644 --- a/tests/gateway/upgrade_test.sh +++ b/tests/gateway/upgrade_test.sh @@ -3,7 +3,8 @@ # An install that is still on ingress-nginx -- the controller, its admission # webhook, the panel's Ingresses as they were, and another app's nginx-class # Ingress with forward-auth -- taken through update.sh's routing steps: -# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx. +# install_traefik, the overlay's Gateway and routes, retire_ingress_nginx, +# move_traefik_to_host_ports. # shellcheck source=tests/gateway/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" @@ -200,6 +201,7 @@ test_running_the_update_again_changes_nothing() { install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "install_traefik failed" routing_layer https || fail "the overlay did not apply" retire_ingress_nginx || fail "retire_ingress_nginx failed" + move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" >/dev/null || fail "move_traefik_to_host_ports failed" call https "$API" /me expect_code 200 expect_other_app_routes_with_its_auth @@ -222,6 +224,7 @@ sleep 2 install_traefik "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1 routing_layer https || exit 1 retire_ingress_nginx || exit 1 +move_traefik_to_host_ports "$PANEL_DIR/overlays/config/traefik-config.env" || exit 1 sleep 5 client touch /tmp/stop for _ in $(seq 1 10); do diff --git a/tests/scripts/stub/kubectl b/tests/scripts/stub/kubectl index 76e019d..4859f10 100755 --- a/tests/scripts/stub/kubectl +++ b/tests/scripts/stub/kubectl @@ -3,6 +3,7 @@ echo "$*" >>"$WORK_DIR/calls" args=" $* " +file="" pod="" container="" previous=false @@ -14,13 +15,28 @@ while [ $# -gt 0 ]; do shift ;; --previous) previous=true ;; + -f) + file="$2" + shift + ;; esac shift done case "$args" in - *" get crd helmcharts.helm.cattle.io "* | *" apply --server-side "*) ;; + *" get crd helmcharts.helm.cattle.io "*) ;; + *" apply --server-side "*) if [ -n "$file" ]; then cat "$file" >>"$WORK_DIR/server-side-applied"; fi ;; *" apply -f - "*) cat >"$WORK_DIR/applied" ;; + *" get service traefik "*'{.spec.type}'*) + if [ -f "$WORK_DIR/traefik-service-error" ]; then + echo "The connection to the server was refused" >&2 + exit 1 + fi + if [ -f "$WORK_DIR/traefik-type" ]; then cat "$WORK_DIR/traefik-type"; fi + ;; + *" get deployment traefik "*) if [ -f "$WORK_DIR/traefik-image" ]; then cat "$WORK_DIR/traefik-image"; fi ;; + *" get pods -l app.kubernetes.io/name=traefik "*) if [ -f "$WORK_DIR/traefik-node" ]; then cat "$WORK_DIR/traefik-node"; fi ;; + *" wait pod/traefik-pull "*) [ ! -f "$WORK_DIR/pull-fails" ] || exit 1 ;; *" get service traefik "*) if [ -f "$WORK_DIR/traefik-address" ]; then cat "$WORK_DIR/traefik-address"; fi ;; *" get helmchart "*) echo 1 ;; *" get gateway traefik-gateway "*) [ -f "$WORK_DIR/shared-gateway" ] || exit 1 ;; @@ -76,7 +92,14 @@ case "$args" in *'name=="https"'*) port_file="$WORK_DIR/nginx-https-port" ;; *) port_file="$WORK_DIR/nginx-http-port" ;; esac + if [ -f "$WORK_DIR/nginx-error" ]; then + echo "The connection to the server was refused" >&2 + exit 1 + fi if [ ! -f "$port_file" ]; then + case "$args" in + *" --ignore-not-found "*) exit 0 ;; + esac echo 'Error from server (NotFound): services "ingress-nginx-controller" not found' >&2 exit 1 fi diff --git a/tests/scripts/traefik_install_test.sh b/tests/scripts/traefik_install_test.sh index a0c28b7..123b349 100644 --- a/tests/scripts/traefik_install_test.sh +++ b/tests/scripts/traefik_install_test.sh @@ -263,4 +263,112 @@ test_dev_retires_without_waiting_for_routes() { assert_not_contains "$(cat "$WORK_DIR/calls")" "wait httproute/api" } +test_retiring_with_no_ingress_nginx_waits_for_no_address() { + run_retire || fail "retire_ingress_nginx failed: $(cat "$WORK_DIR/output")" + assert_not_contains "$(cat "$WORK_DIR/calls")" "loadBalancer.ingress" +} + +try_install() { + ( + export PANEL_DIR="$TEST_ROOT" + # shellcheck source=utils/colors.sh + source "$TEST_ROOT/utils/colors.sh" + # shellcheck source=utils/output_redirect.sh + source "$TEST_ROOT/utils/output_redirect.sh" + # shellcheck source=utils/update_env_var.sh + source "$TEST_ROOT/utils/update_env_var.sh" + # shellcheck source=utils/install_traefik.sh + source "$TEST_ROOT/utils/install_traefik.sh" + "$@" "$WORK_DIR/traefik-config.env" + ) >"$WORK_DIR/output" 2>&1 +} + +run_install() { + try_install "$@" || fail "$1 exited $?: $(cat "$WORK_DIR/output")" +} + +chart_image() { + printf 'docker.io/traefik@%s' "$(sed -n 's/^ *digest: *\(sha256:[0-9a-f]*\).*/\1/p' "$TEST_ROOT/charts/gateway/values.yaml")" +} + +gateway_values() { + sed -n '/^ name: gateway$/,$p' "$WORK_DIR/server-side-applied" 2>/dev/null | sed -n '/valuesContent/,$p' +} + +test_traefik_binds_the_host_ports_itself() { + ports_file HTTP_PORT=8080 HTTPS_PORT=8443 + run_install install_traefik + assert_contains "$(gateway_values)" "web: {exposedPort: 8080, hostPort: 8080}" + assert_contains "$(gateway_values)" "websecure: {exposedPort: 8443, hostPort: 8443}" + assert_not_contains "$(gateway_values)" "LoadBalancer" +} + +test_traefik_waits_behind_servicelb_while_ingress_nginx_holds_the_ports() { + nginx_ports 80 443 + ports_file HTTP_PORT=80 HTTPS_PORT=443 + run_install install_traefik + assert_contains "$(gateway_values)" "spec: {type: LoadBalancer, externalTrafficPolicy: Local}" + assert_not_contains "$(gateway_values)" "hostPort" +} + +test_traefik_left_behind_servicelb_moves_onto_the_host_ports() { + echo LoadBalancer >"$WORK_DIR/traefik-type" + ports_file HTTP_PORT=80 HTTPS_PORT=443 + run_install move_traefik_to_host_ports + assert_contains "$(gateway_values)" "web: {exposedPort: 80, hostPort: 80}" +} + +test_traefik_already_on_the_host_ports_is_not_reinstalled() { + echo ClusterIP >"$WORK_DIR/traefik-type" + run_install move_traefik_to_host_ports + [ ! -f "$WORK_DIR/server-side-applied" ] || fail "the gateway chart was applied again" +} + +test_a_failed_service_lookup_fails_the_move() { + touch "$WORK_DIR/traefik-service-error" + if try_install move_traefik_to_host_ports; then + fail "move_traefik_to_host_ports succeeded without knowing where Traefik is" + fi +} + +test_a_failed_ingress_nginx_lookup_stops_the_install() { + touch "$WORK_DIR/nginx-error" + if try_install install_traefik; then + fail "install_traefik succeeded without knowing whether ingress-nginx holds the ports" + fi + assert_eq "$(gateway_values)" "" +} + +test_a_new_image_is_pulled_onto_traefiks_node_before_the_upgrade() { + echo docker.io/traefik@sha256:0000 >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + run_install install_traefik + assert_contains "$(grep ' run traefik-pull ' "$WORK_DIR/calls")" "--image=$(chart_image) --restart=Never --overrides={\"spec\":{\"nodeName\":\"node-a\"" + [ "$(grep -n ' run traefik-pull ' "$WORK_DIR/calls" | cut -d: -f1)" -lt "$(grep -n ' apply --server-side ' "$WORK_DIR/calls" | tail -n 1 | cut -d: -f1)" ] || + fail "the image was pulled after the upgrade" +} + +test_a_refused_pull_leaves_traefik_alone() { + echo docker.io/traefik@sha256:0000 >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + touch "$WORK_DIR/pull-fails" + if try_install install_traefik; then + fail "install_traefik succeeded although the image never arrived" + fi + assert_eq "$(gateway_values)" "" + assert_contains "$(cat "$WORK_DIR/output")" "keeps running its current image" +} + +test_traefik_already_on_the_image_pulls_nothing() { + chart_image >"$WORK_DIR/traefik-image" + echo node-a >"$WORK_DIR/traefik-node" + run_install install_traefik + assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" +} + +test_a_first_install_pulls_nothing() { + run_install install_traefik + assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" +} + run_tests diff --git a/update.sh b/update.sh index 9e71aad..f963857 100755 --- a/update.sh +++ b/update.sh @@ -117,6 +117,7 @@ if [ "$REVERSE_PROXY" != true ]; then fi retire_ingress_nginx || die "Traefik did not take over ports 80/443" +move_traefik_to_host_ports || die "Traefik did not take over the host's ports" if [ "$VAULT_MANAGER" = true ]; then step "Syncing Vault secrets" diff --git a/utils/install_traefik.sh b/utils/install_traefik.sh index c8b22e3..23d8145 100644 --- a/utils/install_traefik.sh +++ b/utils/install_traefik.sh @@ -39,7 +39,7 @@ carry_over_ingress_nginx_ports() { install_traefik() { local ports_env="${1:-$PANEL_DIR/overlays/config/traefik-config.env}" - local minor http_port https_port crd + local minor http_port https_port crd binding nginx local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -73,16 +73,29 @@ install_traefik() { fi done - # The Service is externalTrafficPolicy: Local, so only the node running - # Traefik answers on these ports -- and DNS points at the server node, not - # at a game node that joined later. + # While ingress-nginx's ServiceLB pods hold the ports, Traefik waits behind + # its own until retire_ingress_nginx hands them over, so a stalled handover + # can still give them back; move_traefik_to_host_ports finishes the move. + nginx="$(kubectl "${kc[@]}" -n ingress-nginx get service ingress-nginx-controller --ignore-not-found -o name)" || return 1 + if [ -n "$nginx" ]; then + binding=" ports: + web: {exposedPort: $http_port} + websecure: {exposedPort: $https_port} + service: + spec: {type: LoadBalancer, externalTrafficPolicy: Local}" + else + binding=" ports: + web: {exposedPort: $http_port, hostPort: $http_port} + websecure: {exposedPort: $https_port, hostPort: $https_port}" + fi + + pull_traefik_image || return 1 + + # Only the node running Traefik listens on these ports, and DNS points at + # the server node, not at a game node that joined later. install_helm_chart gateway "$PANEL_DIR/charts/gateway" "$(cat </dev/null)" = "$image" ]; then + return 0 + fi + node="$(kubectl "${kc[@]}" -n traefik get pods -l app.kubernetes.io/name=traefik -o jsonpath='{.items[0].spec.nodeName}' 2>/dev/null)" + if [ -z "$node" ]; then + return 0 + fi + + echo "Pulling $image onto $node..." + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --ignore-not-found --wait=true + if ! output_redirect kubectl "${kc[@]}" -n traefik run traefik-pull --image="$image" --restart=Never \ + --overrides="{\"spec\":{\"nodeName\":\"$node\",\"tolerations\":[{\"operator\":\"Exists\"}]}}" \ + --command -- traefik version || + ! output_redirect kubectl "${kc[@]}" -n traefik wait pod/traefik-pull --timeout=300s \ + --for=jsonpath='{.status.phase}'=Succeeded; then + err "could not pull $image onto $node; Traefik keeps running its current image" + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --ignore-not-found --wait=false + return 1 + fi + output_redirect kubectl "${kc[@]}" -n traefik delete pod traefik-pull --wait=false +} + +move_traefik_to_host_ports() { + local type + local -a kc=() + [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") + + type="$(kubectl "${kc[@]}" -n traefik get service traefik -o jsonpath='{.spec.type}')" || return 1 + if [ "$type" != LoadBalancer ]; then + return 0 + fi + install_traefik "$@" +} + # The cluster-scoped objects a chart's templates define, as kind/name. chart_objects() { awk '/^kind: / { kind = $2; next } diff --git a/utils/retire_ingress_nginx.sh b/utils/retire_ingress_nginx.sh index 1e42e78..18990d2 100644 --- a/utils/retire_ingress_nginx.sh +++ b/utils/retire_ingress_nginx.sh @@ -48,7 +48,7 @@ YAML # retires ingress-nginx before Tilt has applied them, and a dev cluster has no # visitors to keep serving. retire_ingress_nginx() { - local deadline moved="" http_port="" https_port="" + local deadline http_port="" https_port="" local -a kc=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -70,22 +70,21 @@ retire_ingress_nginx() { -o jsonpath='{.spec.ports[?(@.name=="https")].port}' 2>/dev/null)" echo "Moving ports 80/443 from ingress-nginx to Traefik..." output_redirect kubectl "${kc[@]}" delete service ingress-nginx-controller -n ingress-nginx --wait=true - moved=1 - fi - deadline=$((SECONDS + ${TRAEFIK_ADDRESS_TIMEOUT:-180})) - until [ -n "$(kubectl "${kc[@]}" get service traefik -n traefik -o jsonpath='{.status.loadBalancer.ingress}' 2>/dev/null)" ]; do - if [ "$SECONDS" -ge "$deadline" ]; then - err "Traefik's LoadBalancer Service never got an address:" - kubectl "${kc[@]}" get service traefik -n traefik >&2 - kubectl "${kc[@]}" get pods -A -l svccontroller.k3s.cattle.io/svcname=traefik >&2 - if [ -n "$moved" ] && restore_ingress_nginx_service "${http_port:-80}" "${https_port:-443}"; then - err "ingress-nginx has its ports back and keeps serving the panel; re-run ./update.sh once Traefik's Service can get an address" + deadline=$((SECONDS + ${TRAEFIK_ADDRESS_TIMEOUT:-180})) + until [ -n "$(kubectl "${kc[@]}" get service traefik -n traefik -o jsonpath='{.status.loadBalancer.ingress}' 2>/dev/null)" ]; do + if [ "$SECONDS" -ge "$deadline" ]; then + err "Traefik's LoadBalancer Service never got an address:" + kubectl "${kc[@]}" get service traefik -n traefik >&2 + kubectl "${kc[@]}" get pods -A -l svccontroller.k3s.cattle.io/svcname=traefik >&2 + if restore_ingress_nginx_service "${http_port:-80}" "${https_port:-443}"; then + err "ingress-nginx has its ports back and keeps serving the panel; re-run ./update.sh once Traefik's Service can get an address" + fi + return 1 fi - return 1 - fi - sleep 2 - done + sleep 2 + done + fi # Orphaned, not cascaded: on an old install ingress-shim may still own the # 5stack-ssl Certificate through one of these.