From 815e8afb938c8236a9891bca0b56212424bbd294 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Wed, 4 Nov 2026 13:57:25 -0500 Subject: [PATCH 1/4] fix: a fresh install on k3s 1.37 waits for the Gateway API CRDs k3s is still creating (#736) kubectl wait fails at once on a CRD that doesn't exist yet, so install_traefik raced k3s's gateway-api-crd release; it now waits for each CRD to appear before waiting for it to be established. (cherry picked from commit 343eaeea5d9b44e1a1d7065bc44e38cc52735907) --- tests/scripts/stub/kubectl | 11 +++++++++++ tests/scripts/traefik_install_test.sh | 7 +++++++ utils/install_traefik.sh | 12 +++++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/tests/scripts/stub/kubectl b/tests/scripts/stub/kubectl index 4859f10..3670464 100755 --- a/tests/scripts/stub/kubectl +++ b/tests/scripts/stub/kubectl @@ -25,6 +25,17 @@ done case "$args" in *" get crd helmcharts.helm.cattle.io "*) ;; + *" crd/"*) + if [ -s "$WORK_DIR/crds-appear-after" ]; then + left="$(cat "$WORK_DIR/crds-appear-after")" + echo $((left - 1)) >"$WORK_DIR/crds-appear-after" + if [ "$left" -gt 0 ]; then + crd="$(printf '%s' "$args" | sed -n 's#.* crd/\([^ ]*\).*#\1#p')" + echo "Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io \"$crd\" not found" >&2 + exit 1 + fi + fi + ;; *" apply --server-side "*) if [ -n "$file" ]; then cat "$file" >>"$WORK_DIR/server-side-applied"; fi ;; *" apply -f - "*) cat >"$WORK_DIR/applied" ;; *" get service traefik "*'{.spec.type}'*) diff --git a/tests/scripts/traefik_install_test.sh b/tests/scripts/traefik_install_test.sh index 379f038..b6dda4f 100644 --- a/tests/scripts/traefik_install_test.sh +++ b/tests/scripts/traefik_install_test.sh @@ -436,6 +436,13 @@ test_traefik_already_on_the_image_pulls_nothing() { assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" } +test_gateway_api_crds_k3s_has_yet_to_create_are_waited_for() { + echo helmchart.helm.cattle.io/gateway-api-crd >"$WORK_DIR/helmcharts-gateway-api-crd" + echo 1 >"$WORK_DIR/crds-appear-after" + run_install install_traefik + assert_contains "$(cat "$WORK_DIR/calls")" "wait --for=condition=Established crd/gatewayclasses.gateway.networking.k8s.io" +} + test_a_first_install_pulls_nothing() { run_install install_traefik assert_not_contains "$(cat "$WORK_DIR/calls")" "run traefik-pull" diff --git a/utils/install_traefik.sh b/utils/install_traefik.sh index 7d848a9..3ea8a96 100644 --- a/utils/install_traefik.sh +++ b/utils/install_traefik.sh @@ -39,7 +39,7 @@ carry_over_ingress_nginx_ports() { install_traefik() { local ports_env="${1:-$PANEL_DIR/overlays/config/traefik-config.env}" - local minor http_port https_port crd binding nginx crds_values left_out held object + local minor http_port https_port crd binding nginx crds_values left_out held object deadline local -a kc=() handed=() [ -n "${KUBECONFIG:-}" ] && kc=(--kubeconfig="$KUBECONFIG") @@ -94,8 +94,18 @@ install_traefik() { meta.helm.sh/release-name=gateway-api-crd meta.helm.sh/release-namespace=kube-system || return 1 fi + # k3s's gateway-api-crd release can still be on its way, and kubectl wait + # gives up at once on a CRD that doesn't exist yet. + deadline=$((SECONDS + 300)) for crd in gatewayclasses.gateway.networking.k8s.io gateways.gateway.networking.k8s.io \ httproutes.gateway.networking.k8s.io middlewares.traefik.io; do + until kubectl "${kc[@]}" get "crd/$crd" >/dev/null 2>&1; do + if [ "$SECONDS" -ge "$deadline" ]; then + err "CRD $crd was never created" + return 1 + fi + sleep 2 + done if ! output_redirect kubectl "${kc[@]}" wait --for=condition=Established "crd/$crd" --timeout=120s; then err "CRD $crd never became established" return 1 From f258ae29503e60a8466b8631a979a4c633f33c45 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Wed, 4 Nov 2026 14:25:45 -0500 Subject: [PATCH 2/4] fix: the TimescaleDB extension step says what it waits for, a fresh install skips it, and no kubectl exec can hang an update The step sat silent for up to five minutes while the first start pulled the image. A panel with no timescaledb volume yet has nothing to update, and every kubectl exec in the update path (the extension queries, the PG_VERSION reads, vacuumdb, the signed clip link check) now runs under a time limit. --- .../backblaze-proxy/signed-clip-links.test.ts | 2 +- tests/timescaledb/extension_test.sh | 35 ++++++++++++++++++- tests/timescaledb/major_upgrade_test.sh | 19 +++++++++- tests/timescaledb/stub/kubectl | 8 +++++ update.sh | 6 ++-- utils/colors.sh | 13 +++++++ utils/signed_clip_links.sh | 2 +- utils/update_timescaledb_extension.sh | 17 +++++++-- utils/upgrade_timescaledb_major.sh | 9 +++-- 9 files changed, 99 insertions(+), 12 deletions(-) diff --git a/cloudflare-workers/backblaze-proxy/signed-clip-links.test.ts b/cloudflare-workers/backblaze-proxy/signed-clip-links.test.ts index 7953371..3c9cddf 100644 --- a/cloudflare-workers/backblaze-proxy/signed-clip-links.test.ts +++ b/cloudflare-workers/backblaze-proxy/signed-clip-links.test.ts @@ -111,7 +111,7 @@ describe("backblaze-proxy.sh signed clip links check", () => { assert.match( call, new RegExp( - `^--kubeconfig=${KUBECONFIG} -n 5stack exec timescaledb-0 -c timescaledb -- sh -c `, + `^--kubeconfig=${KUBECONFIG} -n 5stack --request-timeout=120s exec timescaledb-0 -c timescaledb -- sh -c `, ), ); } diff --git a/tests/timescaledb/extension_test.sh b/tests/timescaledb/extension_test.sh index 7d16bf4..ca50508 100755 --- a/tests/timescaledb/extension_test.sh +++ b/tests/timescaledb/extension_test.sh @@ -27,11 +27,44 @@ test_extension_not_installed_returns_quietly() { load_scripts update_timescaledb_extension.sh stub default-version 2.30.2 output="$(update_timescaledb_extension 2>&1)" || fail "returned $?: $output" - assert_eq "$output" "" + assert_eq "$output" " Waiting for TimescaleDB to start (the first start pulls its image)" assert_not_called "ALTER EXTENSION" assert_not_called "rollout restart" } +test_the_wait_for_timescaledb_says_what_it_waits_for() { + load_scripts update_timescaledb_extension.sh + stub extversion 2.30.2 + stub default-version 2.30.2 + output="$(update_timescaledb_extension 2>&1)" || fail "returned $?: $output" + assert_eq "$(head -n 1 <<<"$output")" " Waiting for TimescaleDB to start (the first start pulls its image)" + assert_called "rollout status statefulset/timescaledb --timeout=300s" +} + +test_every_query_has_a_time_limit() { + load_scripts update_timescaledb_extension.sh + stub extversion 2.26.1 + stub default-version 2.30.2 + update_timescaledb_extension >"$STUB_DIR/out" 2>&1 || fail "returned $?: $(cat "$STUB_DIR/out")" + assert_eq "$(grep -c ' exec timescaledb-0 ' "$STUB_DIR/calls")" 4 + assert_eq "$(grep ' exec timescaledb-0 ' "$STUB_DIR/calls" | grep -vc -- '--request-timeout=120s exec timescaledb-0 ')" 0 +} + +test_a_query_that_hangs_is_cut_off_and_warned_about() { + load_scripts update_timescaledb_extension.sh + stub exec-hangs 30 + started="$(date +%s)" + status=0 + TIMESCALEDB_EXEC_TIMEOUT=1 update_timescaledb_extension >"$STUB_DIR/out" 2>&1 || status=$? + took=$(($(date +%s) - started)) + assert_eq "$status" 1 + if [ "$took" -ge 10 ]; then + fail "a hung query held the update for ${took}s" + fi + assert_contains "$(cat "$STUB_DIR/out")" "timescaledb did not answer; its extension was not updated" + assert_not_called "ALTER EXTENSION" +} + test_rollout_never_ready_warns_without_exiting() { load_scripts update_timescaledb_extension.sh stub_rc rollout 1 diff --git a/tests/timescaledb/major_upgrade_test.sh b/tests/timescaledb/major_upgrade_test.sh index 9aa6366..fdcca03 100755 --- a/tests/timescaledb/major_upgrade_test.sh +++ b/tests/timescaledb/major_upgrade_test.sh @@ -9,7 +9,7 @@ KUBECONFIG_FLAG=--kubeconfig=/home/ops/.kube/prod.yaml METRICS_WARNING='E1014 18:50:17.402803 77293 memcache.go:287] "Unhandled Error" err="couldn'"'"'t get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request"' setup_upgrade() { - load_scripts upgrade_timescaledb_major.sh + load_scripts update_timescaledb_extension.sh upgrade_timescaledb_major.sh export TIMESCALEDB_UPGRADE_TTY=/nonexistent/tty PANEL_DIR="$STUB_DIR/panel" manifest "$TARGET" @@ -88,6 +88,22 @@ test_fresh_install_without_a_volume_is_a_no_op() { assert_not_called "apply " } +test_a_panel_without_a_volume_is_a_fresh_install() { + setup_upgrade + plan_timescaledb_major_upgrade >"$STUB_DIR/out" 2>&1 || fail "returned $?" + assert_eq "$TIMESCALEDB_FRESH_INSTALL" true +} + +test_a_panel_with_a_volume_is_not_a_fresh_install() { + setup_upgrade + stub pvc persistentvolumeclaim/timescaledb-pvc + stub pg-version 18 + TIMESCALEDB_FRESH_INSTALL=true + plan_timescaledb_major_upgrade >"$STUB_DIR/out" 2>&1 || fail "returned $?: $(cat "$STUB_DIR/out")" + assert_eq "$TIMESCALEDB_FRESH_INSTALL" "" + assert_called "--request-timeout=120s exec timescaledb-0 -c timescaledb -- cat /var/lib/postgresql/data/PG_VERSION" +} + test_unreachable_cluster_refuses() { setup_upgrade stub stderr "The connection to the server prod:6443 was refused - did you specify the right host or port?" @@ -369,6 +385,7 @@ test_finish_runs_from_the_marker_in_a_later_run() { finish_timescaledb_major_upgrade >"$STUB_DIR/out" 2>&1 || fail "returned $?: $(cat "$STUB_DIR/out")" output="$(cat "$STUB_DIR/out")" assert_called "--all --analyze-in-stages" + assert_called "--request-timeout=3600s exec timescaledb-0 -c timescaledb -- sh -c" assert_called "rollout restart deployment/hasura deployment/api" assert_called "delete configmap timescaledb-major-upgraded timescaledb-major-upgrade" assert_contains "$output" "set the image in base/timescaledb/stateful-set.yaml to $OLD_PINNED" diff --git a/tests/timescaledb/stub/kubectl b/tests/timescaledb/stub/kubectl index fe7371f..7f9b55f 100755 --- a/tests/timescaledb/stub/kubectl +++ b/tests/timescaledb/stub/kubectl @@ -56,6 +56,14 @@ case " $* " in esac printf '%s%s\n' "$*" "${kind:+ <$kind>}" >>"$STUB_DIR/calls" +case " $* " in + *" exec timescaledb-0 "*) + if [ -s "$STUB_DIR/exec-hangs" ]; then + exec sleep "$(cat "$STUB_DIR/exec-hangs")" + fi + ;; +esac + for last; do :; done case " $* " in diff --git a/update.sh b/update.sh index bffe5de..2ad96e9 100755 --- a/update.sh +++ b/update.sh @@ -137,8 +137,10 @@ kubectl --kubeconfig=$KUBECONFIG delete deployment timescaledb -n 5stack 2>/dev kubectl --kubeconfig=$KUBECONFIG delete deployment typesense -n 5stack 2>/dev/null kubectl --kubeconfig=$KUBECONFIG delete deployment redis -n 5stack 2>/dev/null -step "Updating the TimescaleDB extension" -update_timescaledb_extension +if [ "$TIMESCALEDB_FRESH_INSTALL" != true ]; then + step "Updating the TimescaleDB extension" + update_timescaledb_extension +fi finish_timescaledb_major_upgrade GIT_SHA="${FIVESTACK_RELEASE:-$(git rev-parse HEAD)}" diff --git a/utils/colors.sh b/utils/colors.sh index 3239ccf..9ffa3e5 100644 --- a/utils/colors.sh +++ b/utils/colors.sh @@ -47,6 +47,19 @@ read_masked() { printf -v "$__outvar" '%s' "$__value" } +# Runs a command for at most SECONDS. --foreground keeps Ctrl+C reaching it; a +# host whose timeout lacks that (macOS has none, busybox's has no flag) runs it +# unbounded. +time_limited() { + local seconds="$1" + shift + if timeout --foreground 1 true >/dev/null 2>&1; then + timeout --foreground "$seconds" "$@" + else + "$@" + fi +} + # Bail out with a message. Sourced scripts (update.sh runs inside install.sh) # rely on this actually stopping the install rather than letting a half-applied # cluster carry on to the next step. diff --git a/utils/signed_clip_links.sh b/utils/signed_clip_links.sh index 8d72661..ca82fe4 100644 --- a/utils/signed_clip_links.sh +++ b/utils/signed_clip_links.sh @@ -143,6 +143,6 @@ require_signed_clip_links() { } signed_clip_links_psql() { - kubectl --kubeconfig="$PANEL_KUBECONFIG" -n 5stack exec timescaledb-0 -c timescaledb -- \ + time_limited 120 kubectl --kubeconfig="$PANEL_KUBECONFIG" -n 5stack --request-timeout=120s exec timescaledb-0 -c timescaledb -- \ sh -c 'psql -X -v ON_ERROR_STOP=1 -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"' sh "$1" 2>/dev/null } diff --git a/utils/update_timescaledb_extension.sh b/utils/update_timescaledb_extension.sh index 9d7b3ef..d40ec5d 100644 --- a/utils/update_timescaledb_extension.sh +++ b/utils/update_timescaledb_extension.sh @@ -8,14 +8,18 @@ update_timescaledb_extension() { local before available after + echo " Waiting for TimescaleDB to start (the first start pulls its image)" if ! output_redirect kubectl --kubeconfig="$KUBECONFIG" -n 5stack \ rollout status statefulset/timescaledb --timeout=300s; then warn "timescaledb never became ready; its extension was not updated" return 1 fi - before="$(timescaledb_psql "SELECT extversion FROM pg_extension WHERE extname = 'timescaledb'")" - available="$(timescaledb_psql "SELECT default_version FROM pg_available_extensions WHERE name = 'timescaledb'")" + if ! before="$(timescaledb_psql "SELECT extversion FROM pg_extension WHERE extname = 'timescaledb'")" \ + || ! available="$(timescaledb_psql "SELECT default_version FROM pg_available_extensions WHERE name = 'timescaledb'")"; then + warn "timescaledb did not answer; its extension was not updated" + return 1 + fi if [ -z "$before" ] || [ -z "$available" ]; then return 0 fi @@ -35,7 +39,14 @@ update_timescaledb_extension() { ok "timescaledb extension updated from $before to $after" } +timescaledb_exec() { + local seconds="$1" + shift + time_limited "$seconds" kubectl --kubeconfig="$KUBECONFIG" -n 5stack --request-timeout="${seconds}s" \ + exec timescaledb-0 -c timescaledb -- "$@" +} + timescaledb_psql() { - kubectl --kubeconfig="$KUBECONFIG" -n 5stack exec timescaledb-0 -c timescaledb -- \ + timescaledb_exec "${TIMESCALEDB_EXEC_TIMEOUT:-120}" \ sh -c 'psql -X -v ON_ERROR_STOP=1 -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"' sh "$1" } diff --git a/utils/upgrade_timescaledb_major.sh b/utils/upgrade_timescaledb_major.sh index 9578f0b..236545d 100644 --- a/utils/upgrade_timescaledb_major.sh +++ b/utils/upgrade_timescaledb_major.sh @@ -13,6 +13,7 @@ TIMESCALEDB_UPGRADE_JOB=timescaledb-major-upgrade TIMESCALEDB_UPGRADE_MARKER=timescaledb-major-upgraded TIMESCALEDB_UPGRADE_FROM="" TIMESCALEDB_UPGRADE_ATTACH="" +TIMESCALEDB_FRESH_INSTALL="" timescaledb_kubectl() { kubectl --kubeconfig="$KUBECONFIG" -n 5stack "$@" @@ -309,6 +310,7 @@ plan_timescaledb_major_upgrade() { local pvc job active data_major live_image free_kb need_kb minutes TIMESCALEDB_UPGRADE_FROM="" TIMESCALEDB_UPGRADE_ATTACH="" + TIMESCALEDB_FRESH_INSTALL="" timescaledb_manifest_image || return 1 @@ -317,6 +319,7 @@ plan_timescaledb_major_upgrade() { return 1 fi if [ -z "$pvc" ]; then + export TIMESCALEDB_FRESH_INSTALL=true return 0 fi if [ "$pvc" != persistentvolumeclaim/timescaledb-pvc ]; then @@ -358,7 +361,7 @@ plan_timescaledb_major_upgrade() { return 0 fi - data_major="$(timescaledb_kubectl exec timescaledb-0 -c timescaledb -- cat /var/lib/postgresql/data/PG_VERSION 2>/dev/null)" + data_major="$(timescaledb_exec "${TIMESCALEDB_EXEC_TIMEOUT:-120}" cat /var/lib/postgresql/data/PG_VERSION 2>/dev/null)" if [ -z "$data_major" ]; then timescaledb_probe_volume || return 1 data_major="$TIMESCALEDB_PROBE_MAJOR" @@ -667,7 +670,7 @@ finish_timescaledb_major_upgrade() { return 0 fi - major="$(timescaledb_kubectl exec timescaledb-0 -c timescaledb -- cat /var/lib/postgresql/data/PG_VERSION 2>/dev/null)" + major="$(timescaledb_exec "${TIMESCALEDB_EXEC_TIMEOUT:-120}" cat /var/lib/postgresql/data/PG_VERSION 2>/dev/null)" if [ "$major" = "$from" ]; then if [ "$TIMESCALEDB_MAJOR" = "$from" ]; then timescaledb_kubectl delete configmap "$TIMESCALEDB_UPGRADE_MARKER" --ignore-not-found >/dev/null 2>&1 @@ -684,7 +687,7 @@ finish_timescaledb_major_upgrade() { # pg_upgrade from 18 on carries most statistics over, so only what is # missing needs analysing. # shellcheck disable=SC2016 - if output_redirect timescaledb_kubectl exec timescaledb-0 -c timescaledb -- sh -c ' + if output_redirect timescaledb_exec 3600 sh -c ' set -- --all --analyze-in-stages if vacuumdb --help | grep -q -- --missing-stats-only; then set -- "$@" --missing-stats-only From 8cc9fce91fa54ed1933cb1e96c76cb792994ad7e Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Wed, 4 Nov 2026 14:25:46 -0500 Subject: [PATCH 3/4] fix: managed Cloudflare works behind the operator's own reverse proxy, and a key the 5Stack service passes isn't reported missing With REVERSE_PROXY=true the panel issues no certificates, so the engine skips the Let's Encrypt check through Cloudflare, its challenge-path and redirect rules and Always Use HTTPS, and keeps only the origin check. setup-env.sh keeps REVERSE_PROXY=true alongside CLOUDFLARE_MANAGED, and only warns about a missing token when neither CLOUDFLARE_API_TOKEN nor the secrets file has one. --- tests/scripts/cloudflare-stub/curl | 2 + tests/scripts/cloudflare_setup_test.sh | 72 ++++++++++++++++++++++++++ tests/scripts/setup_env_test.sh | 35 ++++++++++++- utils/cloudflare_setup.sh | 41 +++++++++++---- utils/setup-env.sh | 8 +-- 5 files changed, 143 insertions(+), 15 deletions(-) diff --git a/tests/scripts/cloudflare-stub/curl b/tests/scripts/cloudflare-stub/curl index 1d10a74..9c7a3b3 100755 --- a/tests/scripts/cloudflare-stub/curl +++ b/tests/scripts/cloudflare-stub/curl @@ -137,6 +137,8 @@ def plain_http(host, path): answer(code, edge) else: edge = {} + if path.startswith("/.well-known/acme-challenge/") and host in state.get("acme_elsewhere", []): + answer(404, dict(edge, **{"server": edge.get("server", "openresty")})) if path == "/.well-known/acme-challenge/5stack-check" and not state.get("no_acme_route"): answer(404, dict(edge, **{"x-5stack-check": "acme-path"})) answer(308, dict(edge, **https)) diff --git a/tests/scripts/cloudflare_setup_test.sh b/tests/scripts/cloudflare_setup_test.sh index d27a321..3538cbf 100644 --- a/tests/scripts/cloudflare_setup_test.sh +++ b/tests/scripts/cloudflare_setup_test.sh @@ -316,6 +316,78 @@ test_a_check_that_never_goes_through_cloudflare_is_named_and_the_host_kept() { assert_eq "$(grep -c 'HTTP example.gg/.well-known/acme-challenge/5stack-check' "$WORK_DIR/cf-calls")" 2 } +behind_a_reverse_proxy() { + ( + export REVERSE_PROXY=true + cf "$@" + ) +} + +test_behind_a_reverse_proxy_hosts_go_through_cloudflare_even_when_the_check_404s() { + panel_at example.gg + zone example.gg + state '.settings.z1.always_use_https = "on" | .acme_elsewhere = ["example.gg", "api.example.gg", "ws.example.gg", "tv.example.gg", "search.example.gg", "console.example.gg"]' + cf records >/dev/null 2>&1 || fail "records" + behind_a_reverse_proxy proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + for host in example.gg api.example.gg ws.example.gg tv.example.gg search.example.gg console.example.gg; do + assert_eq "$(record "$host")" "203.0.113.40 true" + assert_contains "$(cat "$WORK_DIR/cf-calls")" "ORIGIN $host" + done + assert_not_contains "$(cat "$WORK_DIR/out")" "Let's Encrypt" + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "/.well-known/acme-challenge/5stack-check" + assert_eq "$(jq -r '[.rulesets.z1.http_config_settings.rules[].ref] | join(",")' "$WORK_DIR/cf-state.json")" "5stack_panel" + assert_eq "$(jq -r '.rulesets.z1.http_config_settings.rules[0].action_parameters.ssl' "$WORK_DIR/cf-state.json")" strict + assert_eq "$(jq -r '.rulesets.z1.http_request_dynamic_redirect.rules // [] | length' "$WORK_DIR/cf-state.json")" 0 + assert_eq "$(jq -r '.settings.z1.always_use_https' "$WORK_DIR/cf-state.json")" on + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "PATCH /zones/z1/settings/always_use_https" +} + +test_a_panel_issuing_its_own_certificates_still_keeps_a_host_whose_check_404s_straight() { + panel_at example.gg + zone example.gg + state '.acme_elsewhere = ["api.example.gg"]' + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_contains "$(cat "$WORK_DIR/out")" "Let's Encrypt can't reach api.example.gg through Cloudflare: http://api.example.gg/.well-known/acme-challenge/5stack-check answers HTTP 404 instead of the panel" + assert_eq "$(record api.example.gg)" "203.0.113.40 false" + assert_eq "$(record ws.example.gg)" "203.0.113.40 true" +} + +test_behind_a_reverse_proxy_a_host_cloudflare_cannot_reach_goes_back_to_dns_only() { + panel_at example.gg + zone example.gg + state '.origin = {"ws.example.gg": 522}' + cf records >/dev/null 2>&1 || fail "records" + behind_a_reverse_proxy proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_contains "$(cat "$WORK_DIR/out")" "Cloudflare can't reach the panel on ws.example.gg (HTTP 522)" + assert_eq "$(record ws.example.gg)" "203.0.113.40 false" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" +} + +test_behind_a_reverse_proxy_redirect_rules_and_their_blockers_are_left_alone() { + panel_at example.gg + zone example.gg + state '.rulesets.z1.http_request_dynamic_redirect = {id: "rs7", phase: "http_request_dynamic_redirect", rules: [{id: "theirs", ref: "their_api", description: "API to HTTPS", action: "redirect", expression: "(http.host in {\"api.example.gg\"} and not ssl)", action_parameters: {from_value: {status_code: 308}}}]} | .no_redirect_rules = true' + assert_not_contains "$(behind_a_reverse_proxy check)" "redirect_rules" + cf records >/dev/null 2>&1 || fail "records" + behind_a_reverse_proxy proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + assert_not_contains "$(cat "$WORK_DIR/out")" "API to HTTPS" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" + assert_eq "$(behind_a_reverse_proxy status --json | jq -c '.zones[0].blockers')" "[]" +} + +test_reverse_proxy_mode_is_read_from_the_panels_env_config() { + panel_at example.gg + zone example.gg + state '.acme_elsewhere = ["api.example.gg"]' + printf 'REVERSE_PROXY=true\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\n' >"$WORK_DIR/panel/.5stack-env.config" + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" +} + test_a_key_without_redirect_rules_is_named_before_anything_changes() { panel_at example.gg zone example.gg diff --git a/tests/scripts/setup_env_test.sh b/tests/scripts/setup_env_test.sh index b28eb8e..3d1e6a5 100644 --- a/tests/scripts/setup_env_test.sh +++ b/tests/scripts/setup_env_test.sh @@ -285,16 +285,49 @@ test_update_sh_on_a_vault_install_still_checks_the_vault_login() { assert_contains "$(cat "$WORK_DIR/setup-output")" "Not logged into vault" } -test_managed_cloudflare_keeps_the_panel_off_reverse_proxy_mode() { +test_managed_cloudflare_keeps_a_panel_behind_its_own_reverse_proxy_there() { make_panel set_env overlays/config/api-config.env WEB_DOMAIN example.com printf 'REVERSE_PROXY=true\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=false\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config" run_setup_env assert_contains "$(cat "$WORK_DIR/setup-output")" "no API token yet" assert_eq "$(env_lines overlays/local-secrets/cloudflare-secrets.env CLOUDFLARE_API_TOKEN)" "CLOUDFLARE_API_TOKEN=" + assert_eq "$(bash -c 'source "$1/utils/utils.sh" >/dev/null 2>&1; echo "$REVERSE_PROXY $CLOUDFLARE"' _ "$PANEL" /dev/null 2>&1; echo "$REVERSE_PROXY $CLOUDFLARE"' _ "$PANEL" "$PANEL/.5stack-env.config" assert_eq "$(bash -c 'source "$1/utils/utils.sh" >/dev/null 2>&1; echo "$REVERSE_PROXY $CLOUDFLARE"' _ "$PANEL" "$WORK_DIR/setup-output" 2>&1 || fail "$(tail -n 20 "$WORK_DIR/setup-output")" + assert_eq "$(env_lines .5stack-env.config CLOUDFLARE_MANAGED)" "CLOUDFLARE_MANAGED=true" + assert_eq "$(env_lines .5stack-env.config REVERSE_PROXY)" "REVERSE_PROXY=true" +} + +test_a_cloudflare_token_the_5stack_service_passes_is_not_asked_for() { + make_panel + set_sibling_domains + printf 'REVERSE_PROXY=false\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=false\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config" + CLOUDFLARE_API_TOKEN=cf-from-the-service FIVESTACK_RELEASE=2026.11.4.1 run_setup_env + assert_not_contains "$(cat "$WORK_DIR/setup-output")" "no API token yet" + assert_eq "$(env_lines overlays/local-secrets/cloudflare-secrets.env CLOUDFLARE_API_TOKEN)" "CLOUDFLARE_API_TOKEN=" +} + +test_a_vault_install_is_never_told_it_has_no_cloudflare_token() { + make_panel + set_sibling_domains + printf 'REVERSE_PROXY=false\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=true\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config" + FIVESTACK_RELEASE=2026.11.4.1 run_setup_env + assert_not_contains "$(cat "$WORK_DIR/setup-output")" "no API token yet" +} + test_the_cloudflare_flag_saves_managed_mode() { make_panel set_env overlays/config/api-config.env WEB_DOMAIN example.com diff --git a/utils/cloudflare_setup.sh b/utils/cloudflare_setup.sh index 690cf90..8b82eb1 100755 --- a/utils/cloudflare_setup.sh +++ b/utils/cloudflare_setup.sh @@ -20,7 +20,10 @@ # proxied when HTTPS_PORT isn't 443. Certificates stay with cert-manager's # HTTP-01 check, so plain HTTP on /.well-known/acme-challenge/ must reach the # panel: Always Use HTTPS goes off and a redirect rule sends everything else to -# HTTPS. With CLOUDFLARE_ZONE set, only that zone is used. +# HTTPS. Behind the operator's own reverse proxy (REVERSE_PROXY=true) the panel +# issues no certificates, so none of that is set up or checked, and a host goes +# back to DNS-only only when Cloudflare can't reach the origin. With +# CLOUDFLARE_ZONE set, only that zone is used. CF_SETUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" : "${PANEL_DIR:=$(cd "$CF_SETUP_DIR/.." && pwd)}" @@ -65,6 +68,11 @@ cf_load_hosts() { [ -n "$TYPESENSE_HOST" ] || TYPESENSE_HOST="$(cf_env_value "$config/typesense-config.env" TYPESENSE_HOST)" [ -n "$TURN_DOMAIN" ] || TURN_DOMAIN="$(cf_env_value "$PANEL_DIR/overlays/coturn/coturn.env" TURN_DOMAIN)" [ -n "$HTTPS_PORT" ] || HTTPS_PORT="$(cf_env_value "$config/traefik-config.env" HTTPS_PORT)" + [ -n "$REVERSE_PROXY" ] || REVERSE_PROXY="$(cf_env_value "$PANEL_DIR/.5stack-env.config" REVERSE_PROXY)" +} + +cf_reverse_proxy() { + [ "$REVERSE_PROXY" = true ] } cf_load_token() { @@ -193,8 +201,12 @@ cf_private_ip() { } cf_check() { - local zone_id zone_name ids response probe phase + local zone_id zone_name ids response probe phase phases zone_id="" + phases="http_config_settings http_request_cache_settings http_request_dynamic_redirect" + if cf_reverse_proxy; then + phases="http_config_settings http_request_cache_settings" + fi ids="$(cf_plan | awk '$3 != "-" { print $3 " " $4 }' | sort -u)" if [ -z "$ids" ]; then echo "zone: none of the panel's hostnames is in a zone this token can read" @@ -217,7 +229,7 @@ cf_check() { fi cfa GET "/zones/$zone_id/ssl/universal/settings" | jq -e '.result.enabled != false' >/dev/null 2>&1 || echo "universal_ssl: $zone_name has Universal SSL off, so Cloudflare has no certificate for the proxied names." - for phase in http_config_settings http_request_cache_settings http_request_dynamic_redirect; do + for phase in $phases; do response="$(cfa GET "/zones/$zone_id/rulesets/phases/$phase/entrypoint")" if ! cfa_ok <<<"$response" && ! jq -e 'any(.errors[]?; .code == 10003)' >/dev/null 2>&1 <<<"$response"; then if [ "$phase" = http_request_dynamic_redirect ]; then @@ -528,11 +540,13 @@ cf_rules() { err "Could not save the 5Stack configuration rule" return 1 } - cf_rule "$zone" http_config_settings "$CF_ACME_REF" "5Stack panel: Let's Encrypt's check" set_config \ - '{"security_level": "essentially_off", "bic": false}' "$(cf_acme_expression "${hosts[@]}")" || { - err "Could not save the 5Stack rule that keeps challenges off Let's Encrypt's check" - return 1 - } + if ! cf_reverse_proxy; then + cf_rule "$zone" http_config_settings "$CF_ACME_REF" "5Stack panel: Let's Encrypt's check" set_config \ + '{"security_level": "essentially_off", "bic": false}' "$(cf_acme_expression "${hosts[@]}")" || { + err "Could not save the 5Stack rule that keeps challenges off Let's Encrypt's check" + return 1 + } + fi if [ "${#nocache[@]}" -gt 0 ]; then cf_rule "$zone" http_request_cache_settings "$CF_CACHE_REF" "5Stack panel: live data" set_cache_settings \ '{"cache": false}' "$(cf_host_expression "${nocache[@]}")" || { @@ -540,12 +554,17 @@ cf_rules() { return 1 } fi - cf_https "$zone" "${hosts[@]}" || return 1 + if ! cf_reverse_proxy; then + cf_https "$zone" "${hosts[@]}" || return 1 + fi done } cf_zone_blockers() { local hosts + if cf_reverse_proxy; then + return 0 + fi mapfile -t hosts < <(cf_plan | cf_ruled | awk -v z="$1" '$3 == z { print $2 }') cf_challenge_blockers "$1" "${hosts[@]}" } @@ -578,7 +597,7 @@ cf_proxy() { continue ;; esac - if blocked="$(cf_acme_blocked "$host")"; then + if ! cf_reverse_proxy && blocked="$(cf_acme_blocked "$host")"; then warn "Let's Encrypt can't reach $host through Cloudflare: $blocked. It goes straight to this server again, so its certificate keeps renewing." cf_sync_host "$zone_id" "$host" false false >/dev/null status=3 @@ -743,7 +762,7 @@ cf_stream_relay() { cf_sync_host "$zone_id" "$host" false false >/dev/null return 3 fi - if blocked="$(cf_acme_blocked "$host")"; then + if ! cf_reverse_proxy && blocked="$(cf_acme_blocked "$host")"; then warn "Let's Encrypt can't reach $host through Cloudflare: $blocked. Streams go straight to this server again." cf_sync_host "$zone_id" "$host" false false >/dev/null return 3 diff --git a/utils/setup-env.sh b/utils/setup-env.sh index e17828f..57f3f3b 100755 --- a/utils/setup-env.sh +++ b/utils/setup-env.sh @@ -142,7 +142,7 @@ cloudflare_token_url() { ask_cloudflare_token() { local file="overlays/local-secrets/cloudflare-secrets.env" token - if grep -qs '^CLOUDFLARE_API_TOKEN=..*' "$file"; then + if [ -n "$CLOUDFLARE_API_TOKEN" ] || grep -qs '^CLOUDFLARE_API_TOKEN=..*' "$file"; then return fi if [ "$ASSUME_YES" = true ] || [ ! -t 0 ]; then @@ -309,7 +309,9 @@ if [ "$CLOUDFLARE_MANAGED" = true ]; then echo "Error: managing Cloudflare needs jq. Install it with: apt-get install -y jq" exit 1 fi - REVERSE_PROXY=false + if [ "$REVERSE_PROXY" != true ]; then + REVERSE_PROXY=false + fi CLOUDFLARE=true fi @@ -327,7 +329,7 @@ fi if [ "$CLOUDFLARE_MANAGED" = true ] && ! grep -qs '^CLOUDFLARE_MANAGED=true' .5stack-env.config; then update_env_var .5stack-env.config CLOUDFLARE_MANAGED true update_env_var .5stack-env.config CLOUDFLARE true - update_env_var .5stack-env.config REVERSE_PROXY false + update_env_var .5stack-env.config REVERSE_PROXY "$REVERSE_PROXY" fi if [ -d "base/secrets" ]; then From 4c74d2a390c04f56cb3eeda05bee14dc053a4800 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Wed, 4 Nov 2026 14:25:46 -0500 Subject: [PATCH 4/4] fix: the script join uses a panel's own pod range and tailnet tag, and never adds the shared 10.42 lines for it PANEL_POD_RANGE, PANEL_SERVICE_RANGE and PANEL_TAILSCALE_TAG in .5stack-env.config set the routes, grants, tag owner and key tags game-node-server-setup.sh writes, and the k3s it installs keeps those ranges. Without them, a cluster whose pods are outside 10.42.0.0/16 is refused. --- .github/workflows/panel-scripts.yml | 3 +- game-node-server-setup.sh | 21 +++++-- tests/scripts/tailscale-stub/curl | 57 +++++++++++++++++++ tests/scripts/tailscale-stub/kubectl | 10 ++++ tests/scripts/tailscale_api_test.sh | 83 ++++++++++++++++++++++++++++ utils/tailscale-api.sh | 82 ++++++++++++++++++--------- 6 files changed, 226 insertions(+), 30 deletions(-) create mode 100755 tests/scripts/tailscale-stub/curl create mode 100755 tests/scripts/tailscale-stub/kubectl create mode 100644 tests/scripts/tailscale_api_test.sh diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml index 6a9cf53..a9d2d4a 100644 --- a/.github/workflows/panel-scripts.yml +++ b/.github/workflows/panel-scripts.yml @@ -68,7 +68,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: shellcheck -x cloudflare.sh stream-relay.sh utils/cloudflare_api.sh utils/cloudflare_setup.sh service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh .github/scripts/image_updates.sh tests/scripts/*.sh tests/scripts/stub/kubectl tests/scripts/stub/crane + - run: shellcheck -x cloudflare.sh stream-relay.sh utils/cloudflare_api.sh utils/cloudflare_setup.sh service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh .github/scripts/image_updates.sh tests/scripts/*.sh tests/scripts/stub/kubectl tests/scripts/stub/crane tests/scripts/tailscale-stub/curl tests/scripts/tailscale-stub/kubectl - run: bash tests/scripts/setup_env_test.sh - run: bash tests/scripts/apply_overlay_test.sh - run: bash tests/scripts/debug_test.sh @@ -86,3 +86,4 @@ jobs: - run: bash tests/scripts/choose_node_test.sh - run: bash tests/scripts/nvidia_fallback_test.sh - run: bash tests/scripts/image_updates_test.sh + - run: bash tests/scripts/tailscale_api_test.sh diff --git a/game-node-server-setup.sh b/game-node-server-setup.sh index 6225c01..56ae619 100755 --- a/game-node-server-setup.sh +++ b/game-node-server-setup.sh @@ -9,6 +9,12 @@ if ! command -v jq &> /dev/null; then exit 1 fi +TAILNET_PODS="$(tailnet_pod_range)" || exit 1 +TAILNET_TAGS="fivestack" +if [ "$(tailnet_tag)" != "$TAILSCALE_SHARED_TAG" ]; then + TAILNET_TAGS="fivestack and ${PANEL_TAILSCALE_TAG#tag:}" +fi + step "Installing tailscale" curl -sfL https://tailscale.com/install.sh | sh @@ -17,14 +23,14 @@ echo " This script automates Tailscale configuration using OAuth API." echo echo " ${C_STEP}1.${C_RESET} Create Access Control Tag at:" echo " ${C_OK}https://login.tailscale.com/admin/acls/visual/tags/add${C_RESET}" -echo " Required tag: ${C_OK}fivestack${C_RESET}" +echo " Required tag: ${C_OK}${TAILNET_TAGS}${C_RESET}" echo echo " ${C_STEP}2.${C_RESET} Create an OAuth Client at:" echo " ${C_OK}https://login.tailscale.com/admin/settings/trust-credentials/add${C_RESET}" echo " OAuth scopes:" echo " - General: ${C_OK}Policy File (write)${C_RESET}" echo " - Keys : ${C_OK}Auth Keys (write)${C_RESET}" -echo " Required tag: ${C_OK}fivestack${C_RESET}" +echo " Required tag: ${C_OK}${TAILNET_TAGS}${C_RESET}" echo echo " ${C_STEP}3.${C_RESET} After creating the OAuth client, you'll receive:" echo " - Client ID" @@ -62,7 +68,7 @@ step "Configuring ACL rules for fivestack tag" update_acl_for_fivestack "$ACCESS_TOKEN" if [ $? -eq 0 ]; then - ok "ACL configured (10.42.0.0/16 subnet with auto-approvers)" + ok "ACL configured ($TAILNET_PODS subnet with auto-approvers)" else warn "ACL configuration failed. You may need to configure ACL manually." fi @@ -179,7 +185,14 @@ SCRIPT ok "helper scripts written" step "Installing k3s" -curl -sfL https://get.k3s.io | sh -s - --disable=traefik --vpn-auth="name=tailscale,joinKey=${TAILSCALE_AUTH_KEY}" +K3S_ARGS=(--disable=traefik "--vpn-auth=name=tailscale,joinKey=${TAILSCALE_AUTH_KEY}") +if [ -n "$PANEL_POD_RANGE" ]; then + K3S_ARGS+=("--cluster-cidr=$PANEL_POD_RANGE") +fi +if [ -n "$PANEL_SERVICE_RANGE" ]; then + K3S_ARGS+=("--service-cidr=$PANEL_SERVICE_RANGE") +fi +curl -sfL https://get.k3s.io | sh -s - "${K3S_ARGS[@]}" step "Writing k3s config" mkdir -p /etc/rancher/k3s diff --git a/tests/scripts/tailscale-stub/curl b/tests/scripts/tailscale-stub/curl new file mode 100755 index 0000000..cdd21fe --- /dev/null +++ b/tests/scripts/tailscale-stub/curl @@ -0,0 +1,57 @@ +#!/bin/bash + +method=GET +data="" +url="" +headers=false +while [ "$#" -gt 0 ]; do + case "$1" in + -X) + method="$2" + shift 2 + ;; + -d) + data="$2" + shift 2 + ;; + -D) + headers=true + shift 2 + ;; + -H) + shift 2 + ;; + -*) + shift + ;; + *) + url="$1" + shift + ;; + esac +done + +printf '%s %s\n' "$method" "${url#https://api.tailscale.com}" >>"$WORK_DIR/curl-calls" + +case "$method ${url#https://api.tailscale.com/api/v2}" in + "POST /oauth/token") + echo '{"access_token":"ts-access"}' + ;; + "GET /tailnet/-/acl") + if [ "$headers" = true ]; then + printf 'HTTP/2 200\r\netag: "acl-1"\r\n\r\n' + fi + cat "$WORK_DIR/acl.json" + ;; + "POST /tailnet/-/acl") + printf '%s' "$data" >"$WORK_DIR/acl-posted.json" + echo '{}' + ;; + "POST /tailnet/-/keys") + printf '%s' "$data" >"$WORK_DIR/key-request.json" + echo '{"key":"tskey-auth-test"}' + ;; + *) + echo '{"message":"not stubbed"}' + ;; +esac diff --git a/tests/scripts/tailscale-stub/kubectl b/tests/scripts/tailscale-stub/kubectl new file mode 100755 index 0000000..7d6e066 --- /dev/null +++ b/tests/scripts/tailscale-stub/kubectl @@ -0,0 +1,10 @@ +#!/bin/bash + +printf '%s\n' "$*" >>"$WORK_DIR/kubectl-calls" +case " $* " in + *" get nodes "*) + if [ -f "$WORK_DIR/pod-cidrs" ]; then + cat "$WORK_DIR/pod-cidrs" + fi + ;; +esac diff --git a/tests/scripts/tailscale_api_test.sh b/tests/scripts/tailscale_api_test.sh new file mode 100644 index 0000000..b9a7ebc --- /dev/null +++ b/tests/scripts/tailscale_api_test.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TS_STUB="$(cd "$(dirname "${BASH_SOURCE[0]}")/tailscale-stub" && pwd)" + +policy() { + if [ -n "${1:-}" ]; then + printf '%s\n' "$1" >"$WORK_DIR/acl.json" + else + printf '%s\n' '{"acls": [{"action": "accept", "src": ["*"], "dst": ["*:*"]}]}' >"$WORK_DIR/acl.json" + fi +} + +tailnet() { + ( + export PATH="$TS_STUB:$PATH" + unset PANEL_POD_RANGE PANEL_SERVICE_RANGE PANEL_TAILSCALE_TAG + if [ -n "${POD_RANGE:-}" ]; then + export PANEL_POD_RANGE="$POD_RANGE" + fi + if [ -n "${TAG:-}" ]; then + export PANEL_TAILSCALE_TAG="$TAG" + fi + # shellcheck source=utils/tailscale-api.sh + source "$TEST_ROOT/utils/tailscale-api.sh" + "$@" + ) +} + +posted() { + jq -c "$1" "$WORK_DIR/acl-posted.json" +} + +test_a_panel_on_the_shared_range_keeps_the_shared_lines() { + policy + echo "10.42.0.0/24" >"$WORK_DIR/pod-cidrs" + tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(posted '.autoApprovers.routes')" '{"10.42.0.0/16":["tag:fivestack"]}' + assert_eq "$(posted '.grants')" '[{"src":["tag:fivestack","10.42.0.0/16"],"dst":["tag:fivestack","10.42.0.0/16"],"ip":["*"]}]' + assert_eq "$(posted '.tagOwners')" null +} + +test_a_panel_with_its_own_range_gets_its_own_lines_and_never_the_shared_ones() { + policy + echo "10.52.0.0/24" >"$WORK_DIR/pod-cidrs" + POD_RANGE=10.52.0.0/16 TAG=tag:fivestack-abc123 tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(posted '.autoApprovers.routes')" '{"10.52.0.0/16":["tag:fivestack-abc123"]}' + assert_eq "$(posted '.grants')" '[{"src":["tag:fivestack-abc123","10.52.0.0/16"],"dst":["tag:fivestack-abc123","10.52.0.0/16"],"ip":["*"]}]' + assert_eq "$(posted '.tagOwners')" '{"tag:fivestack-abc123":["autogroup:admin"]}' + assert_not_contains "$(cat "$WORK_DIR/acl-posted.json")" "10.42." + assert_eq "$(posted '.acls | length')" 1 +} + +test_another_panels_shared_lines_stay_as_they_are() { + policy '{"autoApprovers": {"routes": {"10.42.0.0/16": ["tag:fivestack"]}}, "grants": [{"src": ["tag:fivestack", "10.42.0.0/16"], "dst": ["tag:fivestack", "10.42.0.0/16"], "ip": ["*"]}]}' + POD_RANGE=10.52.0.0/16 TAG=tag:fivestack-abc123 tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(posted '.autoApprovers.routes')" '{"10.42.0.0/16":["tag:fivestack"],"10.52.0.0/16":["tag:fivestack-abc123"]}' + assert_eq "$(posted '.grants | length')" 2 + assert_eq "$(posted '.grants[0].src')" '["tag:fivestack-abc123","10.52.0.0/16"]' +} + +test_a_cluster_on_its_own_range_without_it_in_the_config_is_refused() { + policy + echo "10.52.0.0/24" >"$WORK_DIR/pod-cidrs" + if tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1; then + fail "the policy was edited for a panel whose pods are on 10.52.0.0/24" + fi + [ ! -e "$WORK_DIR/acl-posted.json" ] || fail "posted: $(cat "$WORK_DIR/acl-posted.json")" + assert_contains "$(cat "$WORK_DIR/out")" "this panel's pods are on 10.52.0.0/24, outside 10.42.0.0/16. Set PANEL_POD_RANGE and PANEL_TAILSCALE_TAG" +} + +test_game_node_keys_carry_the_panels_own_tag() { + tailnet create_auth_key ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(jq -c '.capabilities.devices.create.tags' "$WORK_DIR/key-request.json")" '["tag:fivestack"]' + assert_eq "$(jq -c '.capabilities.devices.create | [.reusable, .ephemeral, .preauthorized]' "$WORK_DIR/key-request.json")" '[false,false,true]' + assert_eq "$(cat "$WORK_DIR/out")" "tskey-auth-test" + TAG=tag:fivestack-abc123 tailnet create_auth_key ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(jq -c '.capabilities.devices.create.tags' "$WORK_DIR/key-request.json")" '["tag:fivestack","tag:fivestack-abc123"]' +} + +run_tests diff --git a/utils/tailscale-api.sh b/utils/tailscale-api.sh index 5215f0c..16c6b27 100644 --- a/utils/tailscale-api.sh +++ b/utils/tailscale-api.sh @@ -4,6 +4,39 @@ # Documentation: https://tailscale.com/api TAILSCALE_API_BASE="https://api.tailscale.com/api/v2" +TAILSCALE_SHARED_TAG="tag:fivestack" +TAILSCALE_SHARED_PODS="10.42.0.0/16" + +# The shared 10.42 lines would route a panel with its own pod range into +# another panel's pods, so such a cluster is refused rather than guessed at. +tailnet_pod_range() { + local cidr + if [ -n "${PANEL_POD_RANGE:-}" ]; then + echo "$PANEL_POD_RANGE" + return 0 + fi + for cidr in $(kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' 2>/dev/null); do + if [[ "$cidr" != 10.42.* ]]; then + echo "Error: this panel's pods are on $cidr, outside $TAILSCALE_SHARED_PODS. Set PANEL_POD_RANGE and PANEL_TAILSCALE_TAG in .5stack-env.config to the panel's own range and tag" >&2 + return 1 + fi + done + echo "$TAILSCALE_SHARED_PODS" +} + +tailnet_tag() { + echo "${PANEL_TAILSCALE_TAG:-$TAILSCALE_SHARED_TAG}" +} + +tailnet_key_tags() { + local tag + tag="$(tailnet_tag)" + if [ "$tag" = "$TAILSCALE_SHARED_TAG" ]; then + jq -nc --arg s "$TAILSCALE_SHARED_TAG" '[$s]' + else + jq -nc --arg s "$TAILSCALE_SHARED_TAG" --arg t "$tag" '[$s, $t]' + fi +} get_oauth_token() { local client_id=$1 @@ -36,30 +69,22 @@ get_oauth_token() { } create_auth_key() { - local access_token=$1 + local access_token=$1 tags if [ -z "$access_token" ]; then echo "Error: Access token is required" >&2 return 1 fi + tags="$(tailnet_key_tags)" || return 1 local response=$(curl -s -X POST "${TAILSCALE_API_BASE}/tailnet/-/keys" \ -H "Authorization: Bearer ${access_token}" \ -H "Content-Type: application/json" \ - -d '{ - "capabilities": { - "devices": { - "create": { - "reusable": false, - "ephemeral": false, - "preauthorized": true, - "tags": ["tag:fivestack"] - } - } - }, - "expirySeconds": 3600, - "description": "5stack game node auth key" - }') + -d "$(jq -nc --argjson tags "$tags" '{ + capabilities: {devices: {create: {reusable: false, ephemeral: false, preauthorized: true, tags: $tags}}}, + expirySeconds: 3600, + description: "5stack game node auth key" + }')") if echo "$response" | grep -q '"message"' && ! echo "$response" | grep -q '"key"'; then echo "Error: Failed to create auth key" >&2 @@ -78,13 +103,16 @@ create_auth_key() { } update_acl_for_fivestack() { - local access_token=$1 + local access_token=$1 pods tag if [ -z "$access_token" ]; then echo "Error: Access token is required" >&2 return 1 fi + pods="$(tailnet_pod_range)" || return 1 + tag="$(tailnet_tag)" + local acl_response=$(curl -s -D - -X GET "${TAILSCALE_API_BASE}/tailnet/-/acl" \ -H "Authorization: Bearer ${access_token}" \ -H "Accept: application/json") @@ -109,22 +137,26 @@ update_acl_for_fivestack() { return 1 fi - updated_acl=$(echo "$body" | jq ' - .autoApprovers = (.autoApprovers // {}) + updated_acl=$(echo "$body" | jq --arg pods "$pods" --arg tag "$tag" --arg shared "$TAILSCALE_SHARED_TAG" ' + (if $tag != $shared then + .tagOwners = (.tagOwners // {}) + | .tagOwners[$tag] = (.tagOwners[$tag] // ["autogroup:admin"]) + else . end) + | .autoApprovers = (.autoApprovers // {}) | .autoApprovers.routes = (.autoApprovers.routes // {}) - | .autoApprovers.routes["10.42.0.0/16"] = ( - (.autoApprovers.routes["10.42.0.0/16"] // []) | - if index("tag:fivestack") == null then . + ["tag:fivestack"] else . end + | .autoApprovers.routes[$pods] = ( + (.autoApprovers.routes[$pods] // []) | + if index($tag) == null then . + [$tag] else . end ) | .grants = (.grants // []) | .grants = ( - [ - { src: ["tag:fivestack", "10.42.0.0/16"], dst: ["tag:fivestack", "10.42.0.0/16"], ip: ["*"] } + [ + { src: [$tag, $pods], dst: [$tag, $pods], ip: ["*"] } ] + ( .grants | map(select( - .src != ["tag:fivestack", "10.42.0.0/16"] - or .dst != ["tag:fivestack", "10.42.0.0/16"] + .src != [$tag, $pods] + or .dst != [$tag, $pods] or .ip != ["*"] )) )