fix: fresh-install fixes: Gateway API CRD wait, quiet TimescaleDB step, Cloudflare token and reverse proxy, per-panel tailnet ranges

.github/workflows/panel-scripts.yml

@@ -68,7 +68,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - run: shellcheck -x cloudflare.sh stream-relay.sh utils/cloudflare_api.sh utils/cloudflare_setup.sh service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh .github/scripts/image_updates.sh tests/scripts/*.sh tests/scripts/stub/kubectl tests/scripts/stub/crane
+      - run: shellcheck -x cloudflare.sh stream-relay.sh utils/cloudflare_api.sh utils/cloudflare_setup.sh service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh .github/scripts/image_updates.sh tests/scripts/*.sh tests/scripts/stub/kubectl tests/scripts/stub/crane tests/scripts/tailscale-stub/curl tests/scripts/tailscale-stub/kubectl
       - run: bash tests/scripts/setup_env_test.sh
       - run: bash tests/scripts/apply_overlay_test.sh
       - run: bash tests/scripts/debug_test.sh
@@ -86,3 +86,4 @@ jobs:
       - run: bash tests/scripts/choose_node_test.sh
       - run: bash tests/scripts/nvidia_fallback_test.sh
       - run: bash tests/scripts/image_updates_test.sh
+      - run: bash tests/scripts/tailscale_api_test.sh

cloudflare-workers/backblaze-proxy/signed-clip-links.test.ts

@@ -111,7 +111,7 @@ describe("backblaze-proxy.sh signed clip links check", () => {
       assert.match(
         call,
         new RegExp(
-          `^--kubeconfig=${KUBECONFIG} -n 5stack exec timescaledb-0 -c timescaledb -- sh -c `,
+          `^--kubeconfig=${KUBECONFIG} -n 5stack --request-timeout=120s exec timescaledb-0 -c timescaledb -- sh -c `,
         ),
       );
     }

game-node-server-setup.sh

@@ -9,6 +9,12 @@ if ! command -v jq &> /dev/null; then
     exit 1
 fi
 
+TAILNET_PODS="$(tailnet_pod_range)" || exit 1
+TAILNET_TAGS="fivestack"
+if [ "$(tailnet_tag)" != "$TAILSCALE_SHARED_TAG" ]; then
+    TAILNET_TAGS="fivestack and ${PANEL_TAILSCALE_TAG#tag:}"
+fi
+
 step "Installing tailscale"
 curl -sfL https://tailscale.com/install.sh | sh
 
@@ -17,14 +23,14 @@ echo "    This script automates Tailscale configuration using OAuth API."
 echo
 echo "    ${C_STEP}1.${C_RESET} Create Access Control Tag at:"
 echo "       ${C_OK}https://login.tailscale.com/admin/acls/visual/tags/add${C_RESET}"
-echo "       Required tag: ${C_OK}fivestack${C_RESET}"
+echo "       Required tag: ${C_OK}${TAILNET_TAGS}${C_RESET}"
 echo
 echo "    ${C_STEP}2.${C_RESET} Create an OAuth Client at:"
 echo "       ${C_OK}https://login.tailscale.com/admin/settings/trust-credentials/add${C_RESET}"
 echo "       OAuth scopes:"
 echo "         - General: ${C_OK}Policy File (write)${C_RESET}"
 echo "         - Keys : ${C_OK}Auth Keys (write)${C_RESET}"
-echo "       Required tag: ${C_OK}fivestack${C_RESET}"
+echo "       Required tag: ${C_OK}${TAILNET_TAGS}${C_RESET}"
 echo
 echo "    ${C_STEP}3.${C_RESET} After creating the OAuth client, you'll receive:"
 echo "         - Client ID"
@@ -62,7 +68,7 @@ step "Configuring ACL rules for fivestack tag"
 update_acl_for_fivestack "$ACCESS_TOKEN"
 
 if [ $? -eq 0 ]; then
-    ok "ACL configured (10.42.0.0/16 subnet with auto-approvers)"
+    ok "ACL configured ($TAILNET_PODS subnet with auto-approvers)"
 else
     warn "ACL configuration failed. You may need to configure ACL manually."
 fi
@@ -179,7 +185,14 @@ SCRIPT
 ok "helper scripts written"
 
 step "Installing k3s"
-curl -sfL https://get.k3s.io | sh -s - --disable=traefik --vpn-auth="name=tailscale,joinKey=${TAILSCALE_AUTH_KEY}"
+K3S_ARGS=(--disable=traefik "--vpn-auth=name=tailscale,joinKey=${TAILSCALE_AUTH_KEY}")
+if [ -n "$PANEL_POD_RANGE" ]; then
+    K3S_ARGS+=("--cluster-cidr=$PANEL_POD_RANGE")
+fi
+if [ -n "$PANEL_SERVICE_RANGE" ]; then
+    K3S_ARGS+=("--service-cidr=$PANEL_SERVICE_RANGE")
+fi
+curl -sfL https://get.k3s.io | sh -s - "${K3S_ARGS[@]}"
 
 step "Writing k3s config"
 mkdir -p /etc/rancher/k3s

tests/scripts/cloudflare-stub/curl

@@ -137,6 +137,8 @@ def plain_http(host, path):
             answer(code, edge)
     else:
         edge = {}
+    if path.startswith("/.well-known/acme-challenge/") and host in state.get("acme_elsewhere", []):
+        answer(404, dict(edge, **{"server": edge.get("server", "openresty")}))
     if path == "/.well-known/acme-challenge/5stack-check" and not state.get("no_acme_route"):
         answer(404, dict(edge, **{"x-5stack-check": "acme-path"}))
     answer(308, dict(edge, **https))

tests/scripts/cloudflare_setup_test.sh

@@ -316,6 +316,78 @@ test_a_check_that_never_goes_through_cloudflare_is_named_and_the_host_kept() {
     assert_eq "$(grep -c 'HTTP example.gg/.well-known/acme-challenge/5stack-check' "$WORK_DIR/cf-calls")" 2
 }
 
+behind_a_reverse_proxy() {
+    (
+        export REVERSE_PROXY=true
+        cf "$@"
+    )
+}
+
+test_behind_a_reverse_proxy_hosts_go_through_cloudflare_even_when_the_check_404s() {
+    panel_at example.gg
+    zone example.gg
+    state '.settings.z1.always_use_https = "on" | .acme_elsewhere = ["example.gg", "api.example.gg", "ws.example.gg", "tv.example.gg", "search.example.gg", "console.example.gg"]'
+    cf records >/dev/null 2>&1 || fail "records"
+    behind_a_reverse_proxy proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")"
+    for host in example.gg api.example.gg ws.example.gg tv.example.gg search.example.gg console.example.gg; do
+        assert_eq "$(record "$host")" "203.0.113.40 true"
+        assert_contains "$(cat "$WORK_DIR/cf-calls")" "ORIGIN $host"
+    done
+    assert_not_contains "$(cat "$WORK_DIR/out")" "Let's Encrypt"
+    assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "/.well-known/acme-challenge/5stack-check"
+    assert_eq "$(jq -r '[.rulesets.z1.http_config_settings.rules[].ref] | join(",")' "$WORK_DIR/cf-state.json")" "5stack_panel"
+    assert_eq "$(jq -r '.rulesets.z1.http_config_settings.rules[0].action_parameters.ssl' "$WORK_DIR/cf-state.json")" strict
+    assert_eq "$(jq -r '.rulesets.z1.http_request_dynamic_redirect.rules // [] | length' "$WORK_DIR/cf-state.json")" 0
+    assert_eq "$(jq -r '.settings.z1.always_use_https' "$WORK_DIR/cf-state.json")" on
+    assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "PATCH /zones/z1/settings/always_use_https"
+}
+
+test_a_panel_issuing_its_own_certificates_still_keeps_a_host_whose_check_404s_straight() {
+    panel_at example.gg
+    zone example.gg
+    state '.acme_elsewhere = ["api.example.gg"]'
+    cf records >/dev/null 2>&1 || fail "records"
+    cf proxy >"$WORK_DIR/out" 2>&1
+    assert_eq "$?" 3
+    assert_contains "$(cat "$WORK_DIR/out")" "Let's Encrypt can't reach api.example.gg through Cloudflare: http://api.example.gg/.well-known/acme-challenge/5stack-check answers HTTP 404 instead of the panel"
+    assert_eq "$(record api.example.gg)" "203.0.113.40 false"
+    assert_eq "$(record ws.example.gg)" "203.0.113.40 true"
+}
+
+test_behind_a_reverse_proxy_a_host_cloudflare_cannot_reach_goes_back_to_dns_only() {
+    panel_at example.gg
+    zone example.gg
+    state '.origin = {"ws.example.gg": 522}'
+    cf records >/dev/null 2>&1 || fail "records"
+    behind_a_reverse_proxy proxy >"$WORK_DIR/out" 2>&1
+    assert_eq "$?" 3
+    assert_contains "$(cat "$WORK_DIR/out")" "Cloudflare can't reach the panel on ws.example.gg (HTTP 522)"
+    assert_eq "$(record ws.example.gg)" "203.0.113.40 false"
+    assert_eq "$(record api.example.gg)" "203.0.113.40 true"
+}
+
+test_behind_a_reverse_proxy_redirect_rules_and_their_blockers_are_left_alone() {
+    panel_at example.gg
+    zone example.gg
+    state '.rulesets.z1.http_request_dynamic_redirect = {id: "rs7", phase: "http_request_dynamic_redirect", rules: [{id: "theirs", ref: "their_api", description: "API to HTTPS", action: "redirect", expression: "(http.host in {\"api.example.gg\"} and not ssl)", action_parameters: {from_value: {status_code: 308}}}]} | .no_redirect_rules = true'
+    assert_not_contains "$(behind_a_reverse_proxy check)" "redirect_rules"
+    cf records >/dev/null 2>&1 || fail "records"
+    behind_a_reverse_proxy proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")"
+    assert_not_contains "$(cat "$WORK_DIR/out")" "API to HTTPS"
+    assert_eq "$(record api.example.gg)" "203.0.113.40 true"
+    assert_eq "$(behind_a_reverse_proxy status --json | jq -c '.zones[0].blockers')" "[]"
+}
+
+test_reverse_proxy_mode_is_read_from_the_panels_env_config() {
+    panel_at example.gg
+    zone example.gg
+    state '.acme_elsewhere = ["api.example.gg"]'
+    printf 'REVERSE_PROXY=true\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\n' >"$WORK_DIR/panel/.5stack-env.config"
+    cf records >/dev/null 2>&1 || fail "records"
+    cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")"
+    assert_eq "$(record api.example.gg)" "203.0.113.40 true"
+}
+
 test_a_key_without_redirect_rules_is_named_before_anything_changes() {
     panel_at example.gg
     zone example.gg

tests/scripts/setup_env_test.sh

@@ -285,16 +285,49 @@ test_update_sh_on_a_vault_install_still_checks_the_vault_login() {
     assert_contains "$(cat "$WORK_DIR/setup-output")" "Not logged into vault"
 }
 
-test_managed_cloudflare_keeps_the_panel_off_reverse_proxy_mode() {
+test_managed_cloudflare_keeps_a_panel_behind_its_own_reverse_proxy_there() {
     make_panel
     set_env overlays/config/api-config.env WEB_DOMAIN example.com
     printf 'REVERSE_PROXY=true\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=false\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config"
     run_setup_env
     assert_contains "$(cat "$WORK_DIR/setup-output")" "no API token yet"
     assert_eq "$(env_lines overlays/local-secrets/cloudflare-secrets.env CLOUDFLARE_API_TOKEN)" "CLOUDFLARE_API_TOKEN="
+    assert_eq "$(bash -c 'source "$1/utils/utils.sh" >/dev/null 2>&1; echo "$REVERSE_PROXY $CLOUDFLARE"' _ "$PANEL" </dev/null)" "true true"
+    assert_eq "$(bash -c 'source "$1/utils/utils.sh" --reverse-proxy=n >/dev/null 2>&1; echo "$REVERSE_PROXY $CLOUDFLARE"' _ "$PANEL" </dev/null)" "false true"
+}
+
+test_managed_cloudflare_without_a_reverse_proxy_issues_its_own_certificates() {
+    make_panel
+    set_env overlays/config/api-config.env WEB_DOMAIN example.com
+    printf 'CLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=false\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config"
     assert_eq "$(bash -c 'source "$1/utils/utils.sh" >/dev/null 2>&1; echo "$REVERSE_PROXY $CLOUDFLARE"' _ "$PANEL" </dev/null)" "false true"
 }
 
+test_the_cloudflare_flag_with_a_reverse_proxy_saves_both() {
+    make_panel
+    set_env overlays/config/api-config.env WEB_DOMAIN example.com
+    bash -c 'source "$1/utils/utils.sh" --cloudflare-managed --reverse-proxy=y' _ "$PANEL" </dev/null >"$WORK_DIR/setup-output" 2>&1 || fail "$(tail -n 20 "$WORK_DIR/setup-output")"
+    assert_eq "$(env_lines .5stack-env.config CLOUDFLARE_MANAGED)" "CLOUDFLARE_MANAGED=true"
+    assert_eq "$(env_lines .5stack-env.config REVERSE_PROXY)" "REVERSE_PROXY=true"
+}
+
+test_a_cloudflare_token_the_5stack_service_passes_is_not_asked_for() {
+    make_panel
+    set_sibling_domains
+    printf 'REVERSE_PROXY=false\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=false\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config"
+    CLOUDFLARE_API_TOKEN=cf-from-the-service FIVESTACK_RELEASE=2026.11.4.1 run_setup_env
+    assert_not_contains "$(cat "$WORK_DIR/setup-output")" "no API token yet"
+    assert_eq "$(env_lines overlays/local-secrets/cloudflare-secrets.env CLOUDFLARE_API_TOKEN)" "CLOUDFLARE_API_TOKEN="
+}
+
+test_a_vault_install_is_never_told_it_has_no_cloudflare_token() {
+    make_panel
+    set_sibling_domains
+    printf 'REVERSE_PROXY=false\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=true\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config"
+    FIVESTACK_RELEASE=2026.11.4.1 run_setup_env
+    assert_not_contains "$(cat "$WORK_DIR/setup-output")" "no API token yet"
+}
+
 test_the_cloudflare_flag_saves_managed_mode() {
     make_panel
     set_env overlays/config/api-config.env WEB_DOMAIN example.com

tests/scripts/stub/kubectl

@@ -25,6 +25,17 @@ done
 
 case "$args" in
     *" get crd helmcharts.helm.cattle.io "*) ;;
+    *" crd/"*)
+        if [ -s "$WORK_DIR/crds-appear-after" ]; then
+            left="$(cat "$WORK_DIR/crds-appear-after")"
+            echo $((left - 1)) >"$WORK_DIR/crds-appear-after"
+            if [ "$left" -gt 0 ]; then
+                crd="$(printf '%s' "$args" | sed -n 's#.* crd/\([^ ]*\).*#\1#p')"
+                echo "Error from server (NotFound): customresourcedefinitions.apiextensions.k8s.io \"$crd\" not found" >&2
+                exit 1
+            fi
+        fi
+        ;;
     *" apply --server-side "*) if [ -n "$file" ]; then cat "$file" >>"$WORK_DIR/server-side-applied"; fi ;;
     *" apply -f - "*) cat >"$WORK_DIR/applied" ;;
     *" get service traefik "*'{.spec.type}'*)

tests/scripts/tailscale-stub/curl

@@ -0,0 +1,57 @@
+#!/bin/bash
+
+method=GET
+data=""
+url=""
+headers=false
+while [ "$#" -gt 0 ]; do
+    case "$1" in
+        -X)
+            method="$2"
+            shift 2
+            ;;
+        -d)
+            data="$2"
+            shift 2
+            ;;
+        -D)
+            headers=true
+            shift 2
+            ;;
+        -H)
+            shift 2
+            ;;
+        -*)
+            shift
+            ;;
+        *)
+            url="$1"
+            shift
+            ;;
+    esac
+done
+
+printf '%s %s\n' "$method" "${url#https://api.tailscale.com}" >>"$WORK_DIR/curl-calls"
+
+case "$method ${url#https://api.tailscale.com/api/v2}" in
+    "POST /oauth/token")
+        echo '{"access_token":"ts-access"}'
+        ;;
+    "GET /tailnet/-/acl")
+        if [ "$headers" = true ]; then
+            printf 'HTTP/2 200\r\netag: "acl-1"\r\n\r\n'
+        fi
+        cat "$WORK_DIR/acl.json"
+        ;;
+    "POST /tailnet/-/acl")
+        printf '%s' "$data" >"$WORK_DIR/acl-posted.json"
+        echo '{}'
+        ;;
+    "POST /tailnet/-/keys")
+        printf '%s' "$data" >"$WORK_DIR/key-request.json"
+        echo '{"key":"tskey-auth-test"}'
+        ;;
+    *)
+        echo '{"message":"not stubbed"}'
+        ;;
+esac

tests/scripts/tailscale-stub/kubectl

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+printf '%s\n' "$*" >>"$WORK_DIR/kubectl-calls"
+case " $* " in
+    *" get nodes "*)
+        if [ -f "$WORK_DIR/pod-cidrs" ]; then
+            cat "$WORK_DIR/pod-cidrs"
+        fi
+        ;;
+esac

tests/scripts/tailscale_api_test.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+# shellcheck source=tests/scripts/lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+TS_STUB="$(cd "$(dirname "${BASH_SOURCE[0]}")/tailscale-stub" && pwd)"
+
+policy() {
+    if [ -n "${1:-}" ]; then
+        printf '%s\n' "$1" >"$WORK_DIR/acl.json"
+    else
+        printf '%s\n' '{"acls": [{"action": "accept", "src": ["*"], "dst": ["*:*"]}]}' >"$WORK_DIR/acl.json"
+    fi
+}
+
+tailnet() {
+    (
+        export PATH="$TS_STUB:$PATH"
+        unset PANEL_POD_RANGE PANEL_SERVICE_RANGE PANEL_TAILSCALE_TAG
+        if [ -n "${POD_RANGE:-}" ]; then
+            export PANEL_POD_RANGE="$POD_RANGE"
+        fi
+        if [ -n "${TAG:-}" ]; then
+            export PANEL_TAILSCALE_TAG="$TAG"
+        fi
+        # shellcheck source=utils/tailscale-api.sh
+        source "$TEST_ROOT/utils/tailscale-api.sh"
+        "$@"
+    )
+}
+
+posted() {
+    jq -c "$1" "$WORK_DIR/acl-posted.json"
+}
+
+test_a_panel_on_the_shared_range_keeps_the_shared_lines() {
+    policy
+    echo "10.42.0.0/24" >"$WORK_DIR/pod-cidrs"
+    tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")"
+    assert_eq "$(posted '.autoApprovers.routes')" '{"10.42.0.0/16":["tag:fivestack"]}'
+    assert_eq "$(posted '.grants')" '[{"src":["tag:fivestack","10.42.0.0/16"],"dst":["tag:fivestack","10.42.0.0/16"],"ip":["*"]}]'
+    assert_eq "$(posted '.tagOwners')" null
+}
+
+test_a_panel_with_its_own_range_gets_its_own_lines_and_never_the_shared_ones() {
+    policy
+    echo "10.52.0.0/24" >"$WORK_DIR/pod-cidrs"
+    POD_RANGE=10.52.0.0/16 TAG=tag:fivestack-abc123 tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")"
+    assert_eq "$(posted '.autoApprovers.routes')" '{"10.52.0.0/16":["tag:fivestack-abc123"]}'
+    assert_eq "$(posted '.grants')" '[{"src":["tag:fivestack-abc123","10.52.0.0/16"],"dst":["tag:fivestack-abc123","10.52.0.0/16"],"ip":["*"]}]'
+    assert_eq "$(posted '.tagOwners')" '{"tag:fivestack-abc123":["autogroup:admin"]}'
+    assert_not_contains "$(cat "$WORK_DIR/acl-posted.json")" "10.42."
+    assert_eq "$(posted '.acls | length')" 1
+}
+
+test_another_panels_shared_lines_stay_as_they_are() {
+    policy '{"autoApprovers": {"routes": {"10.42.0.0/16": ["tag:fivestack"]}}, "grants": [{"src": ["tag:fivestack", "10.42.0.0/16"], "dst": ["tag:fivestack", "10.42.0.0/16"], "ip": ["*"]}]}'
+    POD_RANGE=10.52.0.0/16 TAG=tag:fivestack-abc123 tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")"
+    assert_eq "$(posted '.autoApprovers.routes')" '{"10.42.0.0/16":["tag:fivestack"],"10.52.0.0/16":["tag:fivestack-abc123"]}'
+    assert_eq "$(posted '.grants | length')" 2
+    assert_eq "$(posted '.grants[0].src')" '["tag:fivestack-abc123","10.52.0.0/16"]'
+}
+
+test_a_cluster_on_its_own_range_without_it_in_the_config_is_refused() {
+    policy
+    echo "10.52.0.0/24" >"$WORK_DIR/pod-cidrs"
+    if tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1; then
+        fail "the policy was edited for a panel whose pods are on 10.52.0.0/24"
+    fi
+    [ ! -e "$WORK_DIR/acl-posted.json" ] || fail "posted: $(cat "$WORK_DIR/acl-posted.json")"
+    assert_contains "$(cat "$WORK_DIR/out")" "this panel's pods are on 10.52.0.0/24, outside 10.42.0.0/16. Set PANEL_POD_RANGE and PANEL_TAILSCALE_TAG"
+}
+
+test_game_node_keys_carry_the_panels_own_tag() {
+    tailnet create_auth_key ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")"
+    assert_eq "$(jq -c '.capabilities.devices.create.tags' "$WORK_DIR/key-request.json")" '["tag:fivestack"]'
+    assert_eq "$(jq -c '.capabilities.devices.create | [.reusable, .ephemeral, .preauthorized]' "$WORK_DIR/key-request.json")" '[false,false,true]'
+    assert_eq "$(cat "$WORK_DIR/out")" "tskey-auth-test"
+    TAG=tag:fivestack-abc123 tailnet create_auth_key ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")"
+    assert_eq "$(jq -c '.capabilities.devices.create.tags' "$WORK_DIR/key-request.json")" '["tag:fivestack","tag:fivestack-abc123"]'
+}
+
+run_tests