diff --git a/.github/workflows/cloudflare-workers.yml b/.github/workflows/cloudflare-workers.yml index 32ef000..51f5c75 100644 --- a/.github/workflows/cloudflare-workers.yml +++ b/.github/workflows/cloudflare-workers.yml @@ -6,11 +6,13 @@ on: paths: - "cloudflare-workers/**" - "backblaze-proxy.sh" + - "stream-relay.sh" - "utils/signed_clip_links.sh" pull_request: paths: - "cloudflare-workers/**" - "backblaze-proxy.sh" + - "stream-relay.sh" - "utils/signed_clip_links.sh" jobs: diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml index 4c1b465..edcd0b1 100644 --- a/.github/workflows/panel-scripts.yml +++ b/.github/workflows/panel-scripts.yml @@ -26,6 +26,11 @@ on: - "base/backups/**" - "base/game-server-node-connector/**" - "overlays/nvidia/**" + - cloudflare.sh + - stream-relay.sh + - "cloudflare-workers/stream-relay/**" + - "overlays/cert-manager/**" + - "overlays/http/**" - ".github/scripts/**" - .github/workflows/panel-scripts.yml pull_request: @@ -52,6 +57,11 @@ on: - "base/backups/**" - "base/game-server-node-connector/**" - "overlays/nvidia/**" + - cloudflare.sh + - stream-relay.sh + - "cloudflare-workers/stream-relay/**" + - "overlays/cert-manager/**" + - "overlays/http/**" - ".github/scripts/**" - .github/workflows/panel-scripts.yml @@ -60,7 +70,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - run: shellcheck -x service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh .github/scripts/image_updates.sh gpu-node/*.sh tests/scripts/*.sh tests/scripts/stub/kubectl tests/scripts/stub/crane + - run: shellcheck -x cloudflare.sh stream-relay.sh utils/cloudflare_api.sh utils/cloudflare_setup.sh service.sh utils/ensure_auth_cookie_domain.sh utils/ensure_node_ip.sh utils/dev_game_server.sh utils/gpu_nodes.sh utils/choose_node.sh .github/scripts/image_updates.sh gpu-node/*.sh tests/scripts/*.sh tests/scripts/stub/kubectl tests/scripts/stub/crane tests/scripts/tailscale-stub/curl tests/scripts/tailscale-stub/kubectl - run: bash tests/scripts/setup_env_test.sh - run: bash tests/scripts/apply_overlay_test.sh - run: bash tests/scripts/debug_test.sh @@ -70,6 +80,7 @@ jobs: - run: bash tests/scripts/node_ip_test.sh - run: bash tests/scripts/traefik_install_test.sh - run: bash tests/scripts/write_overlays_test.sh + - run: bash tests/scripts/cloudflare_setup_test.sh - run: bash tests/scripts/dev_game_server_test.sh - run: bash tests/scripts/gpu_nodes_test.sh - run: bash tests/scripts/service_test.sh @@ -77,5 +88,6 @@ jobs: - run: bash tests/scripts/choose_node_test.sh - run: bash tests/scripts/nvidia_fallback_test.sh - run: bash tests/scripts/image_updates_test.sh + - run: bash tests/scripts/tailscale_api_test.sh - run: bash tests/scripts/proxmox_gpu_lxc_test.sh - run: bash tests/scripts/gpu_node_check_test.sh diff --git a/cloudflare-workers/backblaze-proxy/signed-clip-links.test.ts b/cloudflare-workers/backblaze-proxy/signed-clip-links.test.ts index 7953371..3c9cddf 100644 --- a/cloudflare-workers/backblaze-proxy/signed-clip-links.test.ts +++ b/cloudflare-workers/backblaze-proxy/signed-clip-links.test.ts @@ -111,7 +111,7 @@ describe("backblaze-proxy.sh signed clip links check", () => { assert.match( call, new RegExp( - `^--kubeconfig=${KUBECONFIG} -n 5stack exec timescaledb-0 -c timescaledb -- sh -c `, + `^--kubeconfig=${KUBECONFIG} -n 5stack --request-timeout=120s exec timescaledb-0 -c timescaledb -- sh -c `, ), ); } diff --git a/cloudflare-workers/stream-relay/package.json b/cloudflare-workers/stream-relay/package.json new file mode 100644 index 0000000..094aa89 --- /dev/null +++ b/cloudflare-workers/stream-relay/package.json @@ -0,0 +1,4 @@ +{ + "type": "module", + "private": true +} diff --git a/cloudflare-workers/stream-relay/worker.js b/cloudflare-workers/stream-relay/worker.js new file mode 100644 index 0000000..dc810c0 --- /dev/null +++ b/cloudflare-workers/stream-relay/worker.js @@ -0,0 +1,158 @@ +const NAME = "5stack-stream-relay"; +const VERSION = "1"; + +const PLAYLIST = /\.m3u8$/i; +const SEGMENT = /\.(ts|m4s|mp4)$/i; +const PLAYLIST_SECONDS = 1; +const SEGMENT_SECONDS = 60; +const ACCESS_SECONDS = 30; +const CORS_HEADERS = [ + "Access-Control-Allow-Origin", + "Access-Control-Allow-Credentials", + "Access-Control-Expose-Headers", +]; + +export default { + async fetch(request, env, ctx) { + const url = new URL(request.url); + if (url.pathname === "/health") { + return Response.json( + { ok: true, worker: NAME, version: VERSION }, + { + headers: { + "Access-Control-Allow-Origin": "*", + "Cache-Control": "no-store", + }, + }, + ); + } + if (request.method !== "GET") { + return fetch(request); + } + const parts = url.pathname.split("/").filter(Boolean); + if (parts.length !== 2) { + return origin(request); + } + const [stream, file] = parts; + if (PLAYLIST.test(file)) { + return serve(request, env, ctx, stream, PLAYLIST_SECONDS); + } + if (SEGMENT.test(file)) { + return serve(request, env, ctx, stream, SEGMENT_SECONDS); + } + return origin(request); + }, +}; + +function origin(request) { + return fetch(request, { cache: "no-store" }); +} + +async function serve(request, env, ctx, stream, seconds) { + const url = new URL(request.url); + const key = new Request(`${url.origin}${url.pathname}${url.search}`); + const pass = await accessKey(request, url, stream); + const range = request.headers.get("Range"); + + if (await caches.default.match(pass)) { + const hit = await caches.default.match(key); + if (hit) { + return fromCache(hit, request, env, range, seconds); + } + } + + const response = await origin(request); + if (response.ok) { + ctx.waitUntil( + caches.default.put( + pass, + new Response("allowed", { + headers: { "Cache-Control": `max-age=${ACCESS_SECONDS}` }, + }), + ), + ); + } + if (response.status !== 200 || range) { + return response; + } + + const body = await response.arrayBuffer(); + const headers = new Headers(response.headers); + for (const name of [...CORS_HEADERS, "Set-Cookie", "Vary"]) { + headers.delete(name); + } + headers.set("Cache-Control", `public, max-age=${seconds}`); + const stored = new Response(body, { status: 200, headers }); + ctx.waitUntil(caches.default.put(key, stored.clone())); + return fromCache(stored, request, env, null, seconds); +} + +async function accessKey(request, url, stream) { + const cookie = request.headers.get("Cookie") ?? ""; + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(cookie), + ); + const hash = [...new Uint8Array(digest)] + .map((b) => b.toString(16).padStart(2, "0")) + .join(""); + return new Request( + `${url.origin}/__5stack-access/${encodeURIComponent(stream)}/${hash}`, + ); +} + +async function fromCache(cached, request, env, range, seconds) { + const headers = new Headers(cached.headers); + headers.set("Cache-Control", `private, max-age=${seconds}`); + cors(headers, request, env); + if (!range) { + return new Response(cached.body, { status: 200, headers }); + } + const body = await cached.arrayBuffer(); + const bounds = byteRange(range, body.byteLength); + if (!bounds) { + headers.set("Content-Range", `bytes */${body.byteLength}`); + headers.delete("Content-Length"); + return new Response(null, { status: 416, headers }); + } + const [start, end] = bounds; + headers.set("Content-Range", `bytes ${start}-${end}/${body.byteLength}`); + headers.set("Content-Length", String(end - start + 1)); + return new Response(body.slice(start, end + 1), { status: 206, headers }); +} + +function byteRange(header, size) { + const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim()); + if (!match || (match[1] === "" && match[2] === "")) { + return null; + } + let start; + let end; + if (match[1] === "") { + const suffix = Number(match[2]); + start = Math.max(0, size - suffix); + end = size - 1; + } else { + start = Number(match[1]); + end = match[2] === "" ? size - 1 : Math.min(Number(match[2]), size - 1); + } + if (start > end || start >= size) { + return null; + } + return [start, end]; +} + +function cors(headers, request, env) { + for (const name of CORS_HEADERS) { + headers.delete(name); + } + const from = request.headers.get("Origin"); + if (from && env.WEB_ORIGIN && from === env.WEB_ORIGIN) { + headers.set("Access-Control-Allow-Origin", from); + headers.set("Access-Control-Allow-Credentials", "true"); + headers.set("Access-Control-Expose-Headers", "Location"); + } else if (from) { + headers.set("Access-Control-Allow-Origin", "*"); + } + headers.set("Vary", "Origin"); +} diff --git a/cloudflare-workers/stream-relay/worker.test.mjs b/cloudflare-workers/stream-relay/worker.test.mjs new file mode 100644 index 0000000..c996396 --- /dev/null +++ b/cloudflare-workers/stream-relay/worker.test.mjs @@ -0,0 +1,310 @@ +import { beforeEach, describe, it } from "node:test"; +import assert from "node:assert/strict"; +import worker from "./worker.js"; + +const HLS = "https://hls.example.gg"; +const WEB = "https://example.gg"; +const ENV = { WEB_ORIGIN: WEB }; +const VIEWER = "session=viewer"; +const PLAYER = "session=player"; + +let now; +let origin; +let cache; + +class Origin { + constructor() { + this.requests = []; + this.sequence = 0; + this.segments = new Map(); + this.blocked = new Set([PLAYER]); + this.loginRequired = true; + } + + live(seconds) { + for (let i = 0; i < seconds; i++) { + this.segments.set( + `seg${this.sequence}.mp4`, + `segment-${this.sequence}-`.padEnd(64, "x"), + ); + this.sequence++; + } + for (const name of [...this.segments.keys()]) { + if (Number(name.slice(3, -4)) < this.sequence - 7) { + this.segments.delete(name); + } + } + } + + playlist() { + const names = [...this.segments.keys()]; + const first = Number(names[0].slice(3, -4)); + return [ + "#EXTM3U", + "#EXT-X-VERSION:9", + "#EXT-X-TARGETDURATION:1", + `#EXT-X-MEDIA-SEQUENCE:${first}`, + '#EXT-X-MAP:URI="init.mp4"', + ...names.flatMap((name) => ["#EXTINF:1.00000,", name]), + "", + ].join("\n"); + } + + answer(request) { + this.requests.push(request); + const url = new URL(request.url); + const cookie = request.headers.get("Cookie") ?? ""; + const headers = new Headers({ + "Access-Control-Allow-Credentials": "true", + Vary: "Origin", + "Set-Cookie": "seen=1", + }); + const from = request.headers.get("Origin"); + if (from === WEB) { + headers.set("Access-Control-Allow-Origin", WEB); + } else { + headers.set("Access-Control-Allow-Origin", "*"); + } + if (this.loginRequired && (!cookie || this.blocked.has(cookie))) { + return new Response(null, { status: 401, headers }); + } + const [, stream, file] = url.pathname.split("/"); + if (stream !== "match-1") { + return new Response("no stream", { status: 404, headers }); + } + if (file === "index.m3u8") { + headers.set("Content-Type", "application/vnd.apple.mpegurl"); + return new Response(this.playlist(), { status: 200, headers }); + } + const body = + file === "init.mp4" ? "init-segment" : this.segments.get(file); + if (body === undefined) { + return new Response("not found", { status: 404, headers }); + } + headers.set("Content-Type", "video/mp4"); + const range = request.headers.get("Range"); + if (range) { + const [start, end] = range.slice(6).split("-").map(Number); + headers.set("Content-Range", `bytes ${start}-${end}/${body.length}`); + return new Response(body.slice(start, end + 1), { status: 206, headers }); + } + return new Response(body, { status: 200, headers }); + } + + hits(file) { + return this.requests.filter((r) => new URL(r.url).pathname.endsWith(file)) + .length; + } +} + +beforeEach(() => { + now = 0; + origin = new Origin(); + origin.live(7); + cache = new Map(); + globalThis.fetch = async (input, init) => { + assert.equal(init?.cache, "no-store"); + return origin.answer(input instanceof Request ? input : new Request(input)); + }; + globalThis.caches = { + default: { + async match(request) { + const hit = cache.get(request.url); + if (!hit || hit.until <= now) { + return undefined; + } + return hit.response.clone(); + }, + async put(request, response) { + const age = /max-age=(\d+)/.exec( + response.headers.get("Cache-Control") ?? "", + ); + if (response.headers.has("Set-Cookie")) { + throw new Error("the Cache API was given a Set-Cookie response"); + } + cache.set(request.url, { + response: response.clone(), + until: now + Number(age?.[1] ?? 0) * 1000, + }); + }, + }, + }; +}); + +async function get(path, { cookie = VIEWER, from = WEB, range } = {}) { + const headers = new Headers(); + if (cookie) { + headers.set("Cookie", cookie); + } + if (from) { + headers.set("Origin", from); + } + if (range) { + headers.set("Range", range); + } + const pending = []; + const response = await worker.fetch( + new Request(`${HLS}${path}`, { headers }), + ENV, + { waitUntil: (p) => pending.push(p) }, + ); + await Promise.all(pending); + return response; +} + +describe("stream relay worker", () => { + it("tells the service it is live without asking the origin", async () => { + const response = await get("/health"); + assert.deepEqual(await response.json(), { + ok: true, + worker: "5stack-stream-relay", + version: "1", + }); + assert.equal(origin.requests.length, 0); + }); + + it("sends each segment once and serves every other viewer from the edge", async () => { + const first = await get("/match-1/seg3.mp4"); + assert.equal(first.status, 200); + assert.match(await first.text(), /^segment-3-/); + for (let i = 0; i < 4; i++) { + const again = await get("/match-1/seg3.mp4", { cookie: `session=v${i}` }); + assert.equal(again.status, 200); + } + assert.equal(origin.hits("seg3.mp4"), 1 + 4); + for (let i = 0; i < 10; i++) { + assert.match(await (await get("/match-1/seg3.mp4")).text(), /^segment-3-/); + } + assert.equal(origin.hits("seg3.mp4"), 5); + }); + + it("keeps a live playlist for at most a second, so viewers see new segments", async () => { + const first = await (await get("/match-1/index.m3u8")).text(); + assert.match(first, /#EXT-X-MEDIA-SEQUENCE:0/); + now += 500; + assert.equal(await (await get("/match-1/index.m3u8")).text(), first); + assert.equal(origin.hits("index.m3u8"), 1); + origin.live(2); + now += 600; + const next = await (await get("/match-1/index.m3u8")).text(); + assert.match(next, /#EXT-X-MEDIA-SEQUENCE:2/); + assert.match(next, /seg8\.mp4/); + assert.equal(origin.hits("index.m3u8"), 2); + }); + + it("follows a live stream the way a player does", async () => { + const fetched = new Set(); + for (let second = 0; second < 5; second++) { + for (const viewer of ["a", "b", "c"]) { + const text = await ( + await get("/match-1/index.m3u8", { cookie: `session=${viewer}` }) + ).text(); + for (const name of text.split("\n").filter((l) => l.endsWith(".mp4"))) { + const response = await get(`/match-1/${name}`, { + cookie: `session=${viewer}`, + }); + assert.equal(response.status, 200); + fetched.add(name); + } + } + origin.live(1); + now += 1000; + } + assert.ok(fetched.size >= 11); + for (const name of fetched) { + assert.equal(origin.hits(name), 1, `${name} reached the origin ${origin.hits(name)} times`); + } + assert.ok(origin.hits("index.m3u8") >= 5); + }); + + it("never gives a cached segment to a viewer the panel refuses", async () => { + await get("/match-1/seg2.mp4"); + const refused = await get("/match-1/seg2.mp4", { cookie: PLAYER }); + assert.equal(refused.status, 401); + const anonymous = await get("/match-1/seg2.mp4", { cookie: "" }); + assert.equal(anonymous.status, 401); + const playlist = await get("/match-1/index.m3u8", { cookie: PLAYER }); + assert.equal(playlist.status, 401); + }); + + it("asks the panel again once a viewer's allow is 30 seconds old", async () => { + await get("/match-1/seg2.mp4"); + now += 29_000; + await get("/match-1/seg2.mp4"); + assert.equal(origin.hits("seg2.mp4"), 1); + origin.blocked.add(VIEWER); + now += 2_000; + const response = await get("/match-1/seg2.mp4"); + assert.equal(response.status, 401); + }); + + it("passes a 404 from the origin through unchanged and caches nothing", async () => { + const missing = await get("/match-1/seg99.mp4"); + assert.equal(missing.status, 404); + assert.equal(await missing.text(), "not found"); + assert.equal(missing.headers.get("Access-Control-Allow-Origin"), WEB); + origin.segments.set("seg99.mp4", "late"); + const late = await get("/match-1/seg99.mp4"); + assert.equal(late.status, 200); + assert.equal(await late.text(), "late"); + const noStream = await get("/match-9/index.m3u8"); + assert.equal(noStream.status, 404); + assert.equal(await noStream.text(), "no stream"); + }); + + it("answers a range from the edge copy, and passes a first range through", async () => { + const first = await get("/match-1/seg4.mp4", { range: "bytes=0-9" }); + assert.equal(first.status, 206); + assert.equal(await first.text(), "segment-4-"); + assert.equal(first.headers.get("Content-Range"), "bytes 0-9/64"); + const lastRequest = origin.requests.at(-1); + assert.equal(lastRequest.headers.get("Range"), "bytes=0-9"); + assert.equal(cache.has(`${HLS}/match-1/seg4.mp4`), false); + + await get("/match-1/seg4.mp4"); + const hits = origin.hits("seg4.mp4"); + const ranged = await get("/match-1/seg4.mp4", { range: "bytes=8-11" }); + assert.equal(ranged.status, 206); + assert.equal(await ranged.text(), "4-xx"); + assert.equal(ranged.headers.get("Content-Range"), "bytes 8-11/64"); + assert.equal(ranged.headers.get("Content-Length"), "4"); + const tail = await get("/match-1/seg4.mp4", { range: "bytes=-3" }); + assert.equal(await tail.text(), "xxx"); + const beyond = await get("/match-1/seg4.mp4", { range: "bytes=100-" }); + assert.equal(beyond.status, 416); + assert.equal(beyond.headers.get("Content-Range"), "bytes */64"); + assert.equal(origin.hits("seg4.mp4"), hits); + }); + + it("answers CORS for the panel's site the way the panel does, from the edge copy too", async () => { + await get("/match-1/seg5.mp4", { from: null }); + const fromSite = await get("/match-1/seg5.mp4"); + assert.equal(fromSite.headers.get("Access-Control-Allow-Origin"), WEB); + assert.equal(fromSite.headers.get("Access-Control-Allow-Credentials"), "true"); + assert.equal(fromSite.headers.get("Vary"), "Origin"); + const elsewhere = await get("/match-1/seg5.mp4", { from: "https://other.example" }); + assert.equal(elsewhere.headers.get("Access-Control-Allow-Origin"), "*"); + assert.equal(elsewhere.headers.get("Access-Control-Allow-Credentials"), null); + assert.equal(fromSite.headers.get("Set-Cookie"), null); + assert.match(fromSite.headers.get("Cache-Control"), /^private/); + }); + + it("passes WebRTC signalling and the player page straight through", async () => { + const seen = []; + globalThis.fetch = async (input) => { + seen.push(input instanceof Request ? `${input.method} ${input.url}` : input); + return new Response("ok", { status: 201 }); + }; + const pending = []; + const ctx = { waitUntil: (p) => pending.push(p) }; + const whep = await worker.fetch( + new Request(`${HLS}/match-1/whep`, { method: "POST", body: "offer" }), + ENV, + ctx, + ); + assert.equal(whep.status, 201); + await worker.fetch(new Request(`${HLS}/match-1/`), ENV, ctx); + assert.deepEqual(seen, [`POST ${HLS}/match-1/whep`, `GET ${HLS}/match-1/`]); + assert.equal(cache.size, 0); + }); +}); diff --git a/cloudflare-workers/stream-relay/wrangler.toml b/cloudflare-workers/stream-relay/wrangler.toml new file mode 100644 index 0000000..da75eb6 --- /dev/null +++ b/cloudflare-workers/stream-relay/wrangler.toml @@ -0,0 +1,4 @@ +name = "5stack-stream-relay" +main = "worker.js" +compatibility_date = "2025-04-29" +workers_dev = false diff --git a/cloudflare.sh b/cloudflare.sh new file mode 100755 index 0000000..aed4cc6 --- /dev/null +++ b/cloudflare.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Lets 5Stack manage this panel's hostnames on Cloudflare: the DNS records, the +# proxy on every hostname but demo uploads and streams, and HTTPS Full (strict), +# with Let's Encrypt's plain HTTP check let through. Then applies the panel. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/utils/utils.sh" --cloudflare-managed "$@" + +if [ "$VAULT_MANAGER" = true ] && ! vault kv get -field=CLOUDFLARE_API_TOKEN kv/cloudflare >/dev/null 2>&1; then + step "Cloudflare API token" + echo " Make a token on Cloudflare's page; the permissions 5Stack needs are filled in:" + echo " $(cloudflare_token_url)" + read -rs -p " Paste the token: " CLOUDFLARE_API_TOKEN + echo + [ -n "$CLOUDFLARE_API_TOKEN" ] || die "No token, so nothing changed." + jq -n --arg t "$CLOUDFLARE_API_TOKEN" '{CLOUDFLARE_API_TOKEN: $t}' | vault kv put kv/cloudflare - >/dev/null || die "Could not save the token in Vault at kv/cloudflare." + export CLOUDFLARE_API_TOKEN +fi + +step "Checking Cloudflare" +problems="$(cloudflare_setup check)" || exit 1 +if [ -n "$problems" ]; then + while read -r line; do + err "${line#*: }" + done <<<"$problems" + die "Fix these in Cloudflare, then run ./cloudflare.sh again." +fi +cloudflare_setup plan | while read -r _ host _ zone proxied why; do + case "$proxied" in + yes) ok "$host · through Cloudflare ($zone)" ;; + no) ok "$host · straight to this server ($why)" ;; + skip) ok "$host · left alone ($why)" ;; + esac +done + +"$SCRIPT_DIR/update.sh" "$@" diff --git a/game-node-server-setup.sh b/game-node-server-setup.sh index 6225c01..56ae619 100755 --- a/game-node-server-setup.sh +++ b/game-node-server-setup.sh @@ -9,6 +9,12 @@ if ! command -v jq &> /dev/null; then exit 1 fi +TAILNET_PODS="$(tailnet_pod_range)" || exit 1 +TAILNET_TAGS="fivestack" +if [ "$(tailnet_tag)" != "$TAILSCALE_SHARED_TAG" ]; then + TAILNET_TAGS="fivestack and ${PANEL_TAILSCALE_TAG#tag:}" +fi + step "Installing tailscale" curl -sfL https://tailscale.com/install.sh | sh @@ -17,14 +23,14 @@ echo " This script automates Tailscale configuration using OAuth API." echo echo " ${C_STEP}1.${C_RESET} Create Access Control Tag at:" echo " ${C_OK}https://login.tailscale.com/admin/acls/visual/tags/add${C_RESET}" -echo " Required tag: ${C_OK}fivestack${C_RESET}" +echo " Required tag: ${C_OK}${TAILNET_TAGS}${C_RESET}" echo echo " ${C_STEP}2.${C_RESET} Create an OAuth Client at:" echo " ${C_OK}https://login.tailscale.com/admin/settings/trust-credentials/add${C_RESET}" echo " OAuth scopes:" echo " - General: ${C_OK}Policy File (write)${C_RESET}" echo " - Keys : ${C_OK}Auth Keys (write)${C_RESET}" -echo " Required tag: ${C_OK}fivestack${C_RESET}" +echo " Required tag: ${C_OK}${TAILNET_TAGS}${C_RESET}" echo echo " ${C_STEP}3.${C_RESET} After creating the OAuth client, you'll receive:" echo " - Client ID" @@ -62,7 +68,7 @@ step "Configuring ACL rules for fivestack tag" update_acl_for_fivestack "$ACCESS_TOKEN" if [ $? -eq 0 ]; then - ok "ACL configured (10.42.0.0/16 subnet with auto-approvers)" + ok "ACL configured ($TAILNET_PODS subnet with auto-approvers)" else warn "ACL configuration failed. You may need to configure ACL manually." fi @@ -179,7 +185,14 @@ SCRIPT ok "helper scripts written" step "Installing k3s" -curl -sfL https://get.k3s.io | sh -s - --disable=traefik --vpn-auth="name=tailscale,joinKey=${TAILSCALE_AUTH_KEY}" +K3S_ARGS=(--disable=traefik "--vpn-auth=name=tailscale,joinKey=${TAILSCALE_AUTH_KEY}") +if [ -n "$PANEL_POD_RANGE" ]; then + K3S_ARGS+=("--cluster-cidr=$PANEL_POD_RANGE") +fi +if [ -n "$PANEL_SERVICE_RANGE" ]; then + K3S_ARGS+=("--service-cidr=$PANEL_SERVICE_RANGE") +fi +curl -sfL https://get.k3s.io | sh -s - "${K3S_ARGS[@]}" step "Writing k3s config" mkdir -p /etc/rancher/k3s diff --git a/overlays/http/acme-check.yaml b/overlays/http/acme-check.yaml new file mode 100644 index 0000000..6bd470c --- /dev/null +++ b/overlays/http/acme-check.yaml @@ -0,0 +1,31 @@ +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: acme-check + namespace: 5stack +spec: + headers: + customResponseHeaders: + X-5Stack-Check: acme-path +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: acme-check + namespace: 5stack + annotations: + traefik.ingress.kubernetes.io/router.entrypoints: web + traefik.ingress.kubernetes.io/router.priority: "1000000" + traefik.ingress.kubernetes.io/router.middlewares: 5stack-acme-check@kubernetescrd +spec: + ingressClassName: traefik + rules: + - http: + paths: + - path: /.well-known/acme-challenge/5stack-check + pathType: Exact + backend: + service: + name: web + port: + number: 3000 diff --git a/overlays/local-secrets/cloudflare-secrets.env.example b/overlays/local-secrets/cloudflare-secrets.env.example new file mode 100644 index 0000000..2cf8144 --- /dev/null +++ b/overlays/local-secrets/cloudflare-secrets.env.example @@ -0,0 +1,2 @@ +# Written by 5Stack when Cloudflare is managed for you (./cloudflare.sh or the setup page). +CLOUDFLARE_API_TOKEN= diff --git a/stream-relay.sh b/stream-relay.sh new file mode 100755 index 0000000..2e4dfca --- /dev/null +++ b/stream-relay.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +PANEL_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$PANEL_DIR/utils/colors.sh" +source "$PANEL_DIR/utils/interactive_select.sh" +source "$PANEL_DIR/utils/print_domains_and_hosts.sh" +source "$PANEL_DIR/utils/update_env_var.sh" +source "$PANEL_DIR/utils/cloudflare_workers.sh" +source "$PANEL_DIR/utils/cloudflare_setup.sh" + +WORKER_NAME="5stack-stream-relay" +WORKER_CONFIG="$PANEL_DIR/cloudflare-workers/stream-relay/wrangler.toml" + +load_domains_and_hosts + +if [ -z "$GAME_STREAM_DOMAIN" ]; then + die "GAME_STREAM_DOMAIN is not set in overlays/config/api-config.env. Set up streaming first." +fi + +banner "Stream edge relay" +echo " Puts a Cloudflare Worker in front of https://$GAME_STREAM_DOMAIN, so viewers load" +echo " streams from Cloudflare and never see this server's address. Each video piece" +echo " leaves this server once per Cloudflare location instead of once per viewer." +echo +echo " Each viewer makes about two requests a second. Cloudflare's free Workers plan" +echo " covers 100,000 requests a day, and streams stop until the next day once they" +echo " run out. Busy panels need Workers Paid: \$5 a month for 10 million requests." + +if cf_load_token; then + step "Setting it up with the panel's Cloudflare key" + cloudflare_setup stream-relay + status=$? + if [ "$status" = 0 ]; then + update_env_var "$PANEL_DIR/.5stack-env.config" CLOUDFLARE_STREAM_RELAY true + ok "Updates keep the Worker deployed." + fi + exit "$status" +fi + +cf_require_node +cf_sign_in "$GAME_STREAM_DOMAIN" + +step "Finding $GAME_STREAM_DOMAIN in Cloudflare" +cf_find_zone "$GAME_STREAM_DOMAIN" + +cf_wait_for_proxied "$GAME_STREAM_DOMAIN" +cf_check_origin "$GAME_STREAM_DOMAIN" + +step "Ready to deploy" +ok "Worker: $WORKER_NAME" +ok "Route: $GAME_STREAM_DOMAIN/*" +read -r -p " Deploy it? [Y/n] " CONFIRM +if [[ "$CONFIRM" =~ ^[Nn] ]]; then + warn "Nothing deployed." + exit 0 +fi + +step "Deploying the worker" +if ! wrangler deploy --config "$WORKER_CONFIG" --var "WEB_ORIGIN:https://$WEB_DOMAIN"; then + die "The deploy failed. See the wrangler output above." +fi + +step "Routing $GAME_STREAM_DOMAIN through it" +cf_ensure_routes "$WORKER_NAME" false "$GAME_STREAM_DOMAIN/*" + +step "Waiting for it to answer on https://$GAME_STREAM_DOMAIN/health" +for _ in $(seq 1 24); do + if cf_curl "$GAME_STREAM_DOMAIN" -fsS --max-time 10 "https://$GAME_STREAM_DOMAIN/health" 2>/dev/null | grep -q "\"worker\":\"$WORKER_NAME\""; then + ok "The stream edge relay is active on $GAME_STREAM_DOMAIN" + exit 0 + fi + sleep 5 +done + +err "https://$GAME_STREAM_DOMAIN/health is not answering from the worker yet." +err "Check the route in $CF_ZONE_NAME's Workers Routes, or run this again in a minute." +cf_link "https://dash.cloudflare.com/$CLOUDFLARE_ACCOUNT_ID/$CF_ZONE_NAME/workers" +exit 1 diff --git a/tests/gateway/fresh_install_test.sh b/tests/gateway/fresh_install_test.sh index 483412a..466ffc2 100644 --- a/tests/gateway/fresh_install_test.sh +++ b/tests/gateway/fresh_install_test.sh @@ -64,6 +64,21 @@ test_http01_is_answered_beside_the_https_redirect() { expect_header Location "https://$API/me" } +test_lets_encrypts_check_path_answers_from_the_panel_beside_the_https_redirect() { + local host + for host in "$WEB" "$API" "$WS" "$DEMOS"; do + wait_for 200 http "$host" /.well-known/acme-challenge/5stack-check + expect_json .backend web + expect_header X-5Stack-Check acme-path + done + call http "$API" /.well-known/acme-challenge/test-token + expect_json .backend acme-solver + call http "$WEB" /.well-known/acme-challenge/other-token + expect_code 308 + call https "$WEB" /.well-known/acme-challenge/5stack-check + [ -z "$(header X-5Stack-Check)" ] || fail "the check answers over HTTPS too" +} + test_hosts_serve_while_the_stream_certificate_is_missing() { wait_for 200 https "$API" /me expect_json .backend api @@ -582,6 +597,7 @@ routing_layer https || exit 1 solver_ingress "$API" "$HLS" | kubectl apply -f - >/dev/null || exit 1 run_test test_http01_is_answered_beside_the_https_redirect +run_test test_lets_encrypts_check_path_answers_from_the_panel_beside_the_https_redirect tls_secret 5stack-ssl "$WEB" "$API" "$DEMOS" "$WS" "$RELAY" "$SEARCH" "$CONSOLE" || exit 1 run_test test_hosts_serve_while_the_stream_certificate_is_missing diff --git a/tests/gateway/lib.sh b/tests/gateway/lib.sh index 15dc9c2..baa594a 100644 --- a/tests/gateway/lib.sh +++ b/tests/gateway/lib.sh @@ -163,7 +163,7 @@ routing_layer() { import re, sys docs = open(sys.argv[1]).read().split("\n---\n") kind = lambda d: (re.search(r"(?m)^kind: (\S+)$", d) or [None, None])[1] -routing = [d for d in docs if kind(d) in ("Namespace", "Gateway", "HTTPRoute", "Middleware", "IngressClass") +routing = [d for d in docs if kind(d) in ("Namespace", "Gateway", "HTTPRoute", "Middleware", "IngressClass", "Ingress") or (kind(d) == "Secret" and re.search(r"(?m)^ name: 5stack-reverse-proxy-tls$", d))] open(sys.argv[2], "w").write("\n---\n".join(routing) + "\n") open(sys.argv[3], "w").write("\n---\n".join(d for d in docs if kind(d) == "Issuer") + "\n") diff --git a/tests/gateway/upgrade_test.sh b/tests/gateway/upgrade_test.sh index 8854126..9b732ad 100644 --- a/tests/gateway/upgrade_test.sh +++ b/tests/gateway/upgrade_test.sh @@ -110,13 +110,15 @@ test_before_the_update_ingress_nginx_serves_everything() { } test_ingress_nginx_is_gone() { - local phase + local phase left phase="$(kubectl get namespace ingress-nginx -o jsonpath='{.status.phase}' 2>/dev/null)" [ -z "$phase" ] || [ "$phase" = Terminating ] || fail "ingress-nginx namespace is $phase" kubectl get validatingwebhookconfiguration ingress-nginx-admission >/dev/null 2>&1 && fail "the admission webhook is still there" kubectl get clusterrole ingress-nginx >/dev/null 2>&1 && fail "the ingress-nginx ClusterRole is still there" [ "$(kubectl get ingressclass nginx -o jsonpath='{.spec.controller}')" = k8s.io/ingress-nginx ] || fail "IngressClass nginx is gone" - [ -z "$(kubectl -n 5stack get ingress -o name)" ] || fail "panel Ingresses left: $(kubectl -n 5stack get ingress -o name)" + left="$(kubectl -n 5stack get ingress -o name | grep -v '^ingress.networking.k8s.io/acme-check$')" + [ -z "$left" ] || fail "panel Ingresses left: $left" + [ "$(kubectl -n 5stack get ingress acme-check -o jsonpath='{.spec.ingressClassName}')" = traefik ] || fail "the acme-check Ingress isn't Traefik's" } test_the_crds_k3s_left_belong_to_gateway_crds() { diff --git a/tests/scripts/cloudflare-stub/curl b/tests/scripts/cloudflare-stub/curl new file mode 100755 index 0000000..9c7a3b3 --- /dev/null +++ b/tests/scripts/cloudflare-stub/curl @@ -0,0 +1,320 @@ +#!/usr/bin/env python3 +import json +import os +import re +import sys +import urllib.parse + +work = os.environ["WORK_DIR"] +state_path = os.path.join(work, "cf-state.json") +state = json.load(open(state_path)) +args = sys.argv[1:] + +method, data, url, out_null, write_out, from_stdin = "GET", None, None, False, None, False +dump_headers, resolve = False, None +forms = {} +i = 0 +while i < len(args): + a = args[i] + if a == "-X": + method = args[i + 1] + i += 1 + elif a in ("--data-binary", "--data", "-d"): + data = args[i + 1] + i += 1 + elif a == "-o": + out_null = args[i + 1] == "/dev/null" + i += 1 + elif a == "-w": + write_out = args[i + 1] + i += 1 + elif a == "-K": + from_stdin = args[i + 1] == "-" + i += 1 + elif a == "-D": + dump_headers = args[i + 1] == "-" + i += 1 + elif a == "--resolve": + resolve = args[i + 1].rsplit(":", 1)[1] + i += 1 + elif a == "-F": + name, _, value = args[i + 1].partition("=") + value = value.split(";type=")[0] + if value[:1] in ("<", "@"): + value = open(value[1:]).read() + forms[name] = value + i += 1 + elif a in ("-H", "--max-time"): + if a == "-H" and "authorization" in args[i + 1].lower(): + open(os.path.join(work, "cf-calls"), "a").write("TOKEN-IN-ARGS\n") + i += 1 + elif not a.startswith("-"): + url = a + i += 1 + +auth = sys.stdin.read() if from_stdin else "" + + +def log(line): + with open(os.path.join(work, "cf-calls"), "a") as f: + f.write(line + "\n") + + +def save(): + json.dump(state, open(state_path, "w")) + + +def reply(result=None, success=True, errors=None, code=None): + body = {"success": success, "errors": errors or [], "result": result} + if code is not None: + body["errors"] = [{"code": code, "message": "not found"}] + sys.stdout.write(json.dumps(body)) + sys.exit(0) + + +EDGE = "104.21.0.1" + + +def proxied(host): + return any(r["name"] == host and r.get("proxied") and r["type"] == "A" for r in state.get("records", [])) + + +def zone_of(host): + for z in state.get("zones", []): + if host == z["name"] or host.endswith("." + z["name"]): + return z["id"] + return None + + +def rule_matches(expr, host, path): + hosts = re.search(r"http\.host in \{([^}]*)\}", expr) + if hosts and host not in re.findall(r'"([^"]+)"', hosts.group(1)): + return False + for neg, prefix in re.findall(r'(not )?starts_with\(http\.request\.uri\.path, "([^"]+)"\)', expr): + if bool(neg) == path.startswith(prefix): + return False + return True + + +def page_rule_matches(target, host): + if target.startswith("https://"): + return False + pattern = target.split("://", 1)[-1].split("/", 1)[0] + return re.fullmatch(re.escape(pattern).replace("\\*", ".*"), host) is not None + + +def answer(code, headers): + if dump_headers: + sys.stdout.write("HTTP/1.1 %d X\r\n" % code) + for k, v in headers.items(): + sys.stdout.write("%s: %s\r\n" % (k, v)) + sys.stdout.write("\r\n") + if write_out: + sys.stdout.write(str(code)) + sys.exit(0) + + +def plain_http(host, path): + through = proxied(host) and (resolve == EDGE or (resolve is None and not state.get("stale_dns"))) + log("HTTP %s%s%s" % (host, path, " cloudflare" if through else "")) + edge = {"server": "cloudflare", "cf-ray": "8f00-AMS"} + https = {"location": "https://%s%s" % (host, path)} + if through: + zone = zone_of(host) + if state.get("settings", {}).get(zone, {}).get("always_use_https") == "on": + answer(301, dict(edge, **https)) + for pr in state.get("pagerules", {}).get(zone, []): + ids = [a["id"] for a in pr.get("actions", [])] + if pr.get("status", "active") == "active" and ("always_use_https" in ids or "forwarding_url" in ids) and page_rule_matches(pr["targets"][0]["constraint"]["value"], host): + answer(301, dict(edge, **https)) + for rule in state.get("rulesets", {}).get(zone, {}).get("http_request_dynamic_redirect", {}).get("rules", []): + if rule.get("enabled", True) and rule.get("action") == "redirect" and rule_matches(rule["expression"], host, path): + answer(rule["action_parameters"]["from_value"]["status_code"], dict(edge, **https)) + if host in state.get("challenge", []): + answer(403, dict(edge, **{"cf-mitigated": "challenge"})) + code = state.get("origin", {}).get(host, 200) + if code >= 520: + answer(code, edge) + else: + edge = {} + if path.startswith("/.well-known/acme-challenge/") and host in state.get("acme_elsewhere", []): + answer(404, dict(edge, **{"server": edge.get("server", "openresty")})) + if path == "/.well-known/acme-challenge/5stack-check" and not state.get("no_acme_route"): + answer(404, dict(edge, **{"x-5stack-check": "acme-path"})) + answer(308, dict(edge, **https)) + + +if url.startswith("https://1.1.1.1/cdn-cgi/trace"): + sys.stdout.write("fl=1\nip=%s\nts=1\n" % state.get("ip", "203.0.113.40")) + sys.exit(0) + +if url.startswith("https://1.1.1.1/dns-query"): + name = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)["name"][0] + ip = EDGE if proxied(name) and not state.get("stale_dns") else state.get("ip", "203.0.113.40") + sys.stdout.write(json.dumps({"Status": 0, "Answer": [{"name": name, "type": 1, "data": ip}]})) + sys.exit(0) + +if url.startswith("http://"): + parsed = urllib.parse.urlparse(url) + plain_http(parsed.hostname, parsed.path or "/") + +parsed = urllib.parse.urlparse(url) +if not parsed.path.startswith("/client/v4"): + log("ORIGIN %s%s" % (parsed.hostname, parsed.path if parsed.path not in ("", "/") else "")) + if parsed.path == "/health": + worker = next((r["script"] for z in state.get("routes", {}).values() for r in z + if r["pattern"] == parsed.hostname + "/*" and state.get("workers", {}).get(r["script"])), None) + proxied = any(r["name"] == parsed.hostname and r.get("proxied") for r in state.get("records", [])) + if worker and proxied and not state.get("worker_down"): + sys.stdout.write(json.dumps({"ok": True, "worker": worker}, separators=(",", ":"))) + sys.exit(0) + sys.exit(22) + code = state.get("origin", {}).get(parsed.hostname, 200) + if write_out: + sys.stdout.write(str(code)) + sys.exit(0) + +if "Bearer %s" % state.get("token", "") not in auth: + reply(None, False, [{"code": 9109, "message": "Invalid access token"}]) + +path = parsed.path[len("/client/v4"):] +query = urllib.parse.parse_qs(parsed.query) +log("%s %s%s" % (method, path, (" " + data) if data else "")) +parts = path.strip("/").split("/") +zones = state.setdefault("zones", []) +records = state.setdefault("records", []) + +if parts == ["zones"]: + name = query.get("name", [""])[0] + reply([z for z in zones if z["name"] == name]) +if parts == ["user", "tokens", "verify"]: + reply({"status": "active"}) +if parts[:1] == ["accounts"] and parts[2:4] == ["workers", "scripts"] and method == "PUT": + if state.get("no_workers_write"): + reply(None, False, [{"code": 10000, "message": "Authentication error"}]) + state.setdefault("workers", {})[parts[4]] = {"account": parts[1], "metadata": json.loads(forms["metadata"]), "module": forms.get("worker.js", "")} + save() + reply({"id": parts[4]}) +if parts[0] != "zones": + reply(None, False, [{"code": 7003, "message": "no route"}]) +zone = parts[1] +rest = parts[2:] +if rest == [] and method == "GET": + reply({"id": zone, "account": {"id": "acct1"}}) + +if rest[:1] == ["dns_records"]: + if method == "GET": + out = [r for r in records if r["zone"] == zone] + if "name" in query: + out = [r for r in out if r["name"] == query["name"][0]] + if "type" in query: + out = [r for r in out if r["type"] == query["type"][0]] + reply(out) + if method == "POST": + body = json.loads(data) + if body["type"] == "TXT" and state.get("no_dns_edit"): + reply(None, False, [{"code": 10000, "message": "Authentication error"}]) + state["next"] = state.get("next", 0) + 1 + body.update({"id": "r%d" % state["next"], "zone": zone, "modified_on": "2026-11-03T12:00:00Z"}) + records.append(body) + save() + reply(body) + rid = rest[1] + rec = next((r for r in records if r["id"] == rid), None) + if rec is None: + reply(None, False, [{"code": 81044, "message": "record not found"}]) + if method == "DELETE": + records.remove(rec) + save() + reply({"id": rid}) + if method in ("PATCH", "PUT"): + rec.update(json.loads(data)) + save() + reply(rec) + +if rest == ["ssl", "certificate_packs"]: + reply(state.get("packs", {}).get(zone, [])) +if rest[:2] == ["workers", "routes"]: + if state.get("no_routes_read"): + reply(None, False, [{"code": 10000, "message": "Authentication error"}]) + routes = state.setdefault("routes", {}).setdefault(zone, []) + if method == "POST": + body = json.loads(data) + state["next"] = state.get("next", 0) + 1 + body["id"] = "route%d" % state["next"] + routes.append(body) + save() + reply(body) + if method == "PUT": + body = json.loads(data) + for j, r in enumerate(routes): + if r["id"] == rest[2]: + body["id"] = r["id"] + routes[j] = body + save() + reply(body) + reply(routes) +if rest[:1] == ["settings"]: + settings = state.setdefault("settings", {}).setdefault(zone, {}) + if method == "PATCH": + settings[rest[1]] = json.loads(data)["value"] + save() + reply({"id": rest[1], "value": settings.get(rest[1], "off" if rest[1] == "always_use_https" else "on")}) +if rest == ["pagerules"]: + if state.get("no_pagerules_read"): + reply(None, False, [{"code": 10000, "message": "Authentication error"}]) + reply(state.get("pagerules", {}).get(zone, [])) +if rest == ["bot_management"]: + reply({"fight_mode": state.get("bot_fight", {}).get(zone, False)}) +if rest == ["ssl", "universal", "settings"]: + reply({"enabled": True}) +if rest == ["argo", "tiered_caching"]: + argo = state.setdefault("argo", {}) + if method == "PATCH": + argo[zone] = json.loads(data)["value"] + save() + reply({"value": argo.get(zone, "off")}) +if rest == ["cache", "tiered_cache_smart_topology_enable"]: + tiered = state.setdefault("tiered", {}) + if method == "PATCH": + tiered[zone] = json.loads(data)["value"] + save() + reply({"value": tiered.get(zone, "off")}) + +rulesets = state.setdefault("rulesets", {}).setdefault(zone, {}) +if rest[:2] == ["rulesets", "phases"]: + phase = rest[2] + if phase == "http_request_dynamic_redirect" and state.get("no_redirect_rules"): + reply(None, False, [{"code": 10000, "message": "Authentication error"}]) + if method == "GET": + if phase not in rulesets: + reply(None, False, code=10003) + reply(rulesets[phase]) + if method == "PUT": + if phase in rulesets: + log("PUT-OVER-EXISTING %s" % phase) + state["next"] = state.get("next", 0) + 1 + rs = {"id": "rs%d" % state["next"], "phase": phase, "rules": []} + for rule in json.loads(data)["rules"]: + state["next"] += 1 + rule["id"] = "rule%d" % state["next"] + rs["rules"].append(rule) + rulesets[phase] = rs + save() + reply(rs) +if rest[:1] == ["rulesets"]: + rs = next((r for r in rulesets.values() if r["id"] == rest[1]), None) + rule = json.loads(data) + if method == "POST": + state["next"] = state.get("next", 0) + 1 + rule["id"] = "rule%d" % state["next"] + rs["rules"].append(rule) + if method == "PATCH": + for j, r in enumerate(rs["rules"]): + if r["id"] == rest[3]: + rule["id"] = r["id"] + rs["rules"][j] = rule + save() + reply(rs) + +reply(None, False, [{"code": 7003, "message": "no route for %s" % path}]) diff --git a/tests/scripts/cloudflare_setup_test.sh b/tests/scripts/cloudflare_setup_test.sh new file mode 100644 index 0000000..61499ff --- /dev/null +++ b/tests/scripts/cloudflare_setup_test.sh @@ -0,0 +1,782 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +CF_STUB="$(cd "$(dirname "${BASH_SOURCE[0]}")/cloudflare-stub" && pwd)" +TOKEN="cf-test-token-0123456789" +STUB_PATH="$CF_STUB:$PATH" + +panel_at() { + local domain="$1" + mkdir -p "$WORK_DIR/panel/overlays/config" "$WORK_DIR/panel/overlays/coturn" "$WORK_DIR/panel/overlays/local-secrets" + cat >"$WORK_DIR/panel/overlays/config/api-config.env" <"$WORK_DIR/panel/overlays/config/s3-config.env" + printf 'TYPESENSE_HOST=search.%s\n' "$domain" >"$WORK_DIR/panel/overlays/config/typesense-config.env" + printf 'HTTP_PORT=80\nHTTPS_PORT=443\n' >"$WORK_DIR/panel/overlays/config/traefik-config.env" + printf 'CLOUDFLARE_API_TOKEN=%s\n' "$TOKEN" >"$WORK_DIR/panel/overlays/local-secrets/cloudflare-secrets.env" +} + +zone() { + local name="$1" plan="${2:-Free Website}" + jq -nc --arg t "$TOKEN" --arg n "$name" --arg p "$plan" \ + '{token: $t, ip: "203.0.113.40", zones: [{id: "z1", name: $n, plan: {name: $p}}], records: []}' >"$WORK_DIR/cf-state.json" +} + +state() { + jq -c "$1" "$WORK_DIR/cf-state.json" >"$WORK_DIR/cf-state.next" && mv "$WORK_DIR/cf-state.next" "$WORK_DIR/cf-state.json" +} + +cf() { + ( + export PANEL_DIR="$WORK_DIR/panel" PATH="$STUB_PATH" + unset CLOUDFLARE_API_TOKEN CLOUDFLARE_ORIGIN_IPV4 CLOUDFLARE_ORIGIN_IPV6 HTTPS_PORT TURN_DOMAIN CLOUDFLARE_WORKER_DOMAIN VAULT_MANAGER + CF_RELAY_WAIT=2 CF_RELAY_PAUSE=0 CF_PROBE_WAIT=2 CF_PROBE_PAUSE=0 bash "$TEST_ROOT/utils/cloudflare_setup.sh" "$@" "$WORK_DIR/panel/overlays/coturn/coturn.env" + for host in example.gg api.example.gg ws.example.gg tv.example.gg search.example.gg console.example.gg; do + assert_eq "$(plan_of "$host")" "yes -" + done + assert_eq "$(plan_of demos.example.gg)" "no uploads" + assert_eq "$(plan_of hls.example.gg)" "no video" + assert_eq "$(plan_of turn.example.gg)" "no udp" + cf records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" + assert_eq "$(record demos.example.gg)" "203.0.113.40 false" + assert_eq "$(record hls.example.gg)" "203.0.113.40 false" + assert_eq "$(record turn.example.gg)" "203.0.113.40 false" + if grep -E '"name":"(demos|hls|turn)\.example\.gg"[^}]*"proxied":true' "$WORK_DIR/cf-calls"; then + fail "demos, hls or turn was sent with proxied:true" + fi + if grep -E 'ORIGIN (demos|hls|turn)\.example\.gg' "$WORK_DIR/cf-calls"; then + fail "the proxy step checked a host that stays straight to this server" + fi +} + +test_one_level_below_the_zone_is_covered_and_deeper_names_are_not() { + panel_at panel.example.gg + zone example.gg + assert_eq "$(plan_of panel.example.gg)" "yes -" + assert_eq "$(plan_of api.panel.example.gg)" "no depth" + assert_eq "$(plan_of ws.panel.example.gg)" "no depth" +} + +test_flat_names_beside_the_panel_are_covered_by_the_free_certificate() { + panel_at panel.example.gg + local key sub + for key in API_DOMAIN:api WS_DOMAIN:ws RELAY_DOMAIN:tv DEMOS_DOMAIN:demos GAME_STREAM_DOMAIN:hls; do + sub="${key#*:}" + sed -i.bak "s/^${key%%:*}=.*/${key%%:*}=panel-$sub.example.gg/" "$WORK_DIR/panel/overlays/config/api-config.env" + done + printf 'S3_CONSOLE_HOST=panel-console.example.gg\n' >"$WORK_DIR/panel/overlays/config/s3-config.env" + printf 'TYPESENSE_HOST=panel-search.example.gg\n' >"$WORK_DIR/panel/overlays/config/typesense-config.env" + zone example.gg + for host in panel.example.gg panel-api.example.gg panel-ws.example.gg panel-tv.example.gg panel-search.example.gg panel-console.example.gg; do + assert_eq "$(plan_of "$host")" "yes -" + done + assert_eq "$(plan_of panel-demos.example.gg)" "no uploads" + assert_eq "$(plan_of panel-hls.example.gg)" "no video" + assert_eq "$(plan_of api.panel.example.gg)" "" + cf records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + assert_eq "$(record panel-api.example.gg)" "203.0.113.40 true" + assert_eq "$(record panel-ws.example.gg)" "203.0.113.40 true" + assert_eq "$(record panel-demos.example.gg)" "203.0.113.40 false" +} + +test_two_levels_deep_nothing_is_proxied() { + panel_at cs.panel.example.gg + zone example.gg + assert_eq "$(plan_of cs.panel.example.gg)" "no depth" + assert_eq "$(plan_of api.cs.panel.example.gg)" "no depth" +} + +test_a_paid_plan_alone_does_not_cover_deeper_names() { + panel_at panel.example.gg + zone example.gg "Pro Website" + assert_eq "$(plan_of api.panel.example.gg)" "no depth" +} + +test_an_advanced_certificate_covers_deeper_names() { + panel_at panel.example.gg + zone example.gg + state '.packs.z1 = [{status: "active", type: "advanced", hosts: ["panel.example.gg", "*.panel.example.gg"]}]' + assert_eq "$(plan_of api.panel.example.gg)" "yes -" + state '.packs.z1[0].status = "pending_validation"' + assert_eq "$(plan_of api.panel.example.gg)" "no depth" +} + +test_a_panel_off_port_443_proxies_nothing() { + panel_at example.gg + zone example.gg + printf 'HTTP_PORT=8080\nHTTPS_PORT=8443\n' >"$WORK_DIR/panel/overlays/config/traefik-config.env" + assert_eq "$(plan_of api.example.gg)" "no port" + assert_eq "$(plan_of example.gg)" "no port" +} + +test_worker_hostnames_are_never_touched() { + panel_at example.gg + zone example.gg + printf 'CLOUDFLARE_WORKER_DOMAIN=cf.example.gg\n' >>"$WORK_DIR/panel/overlays/config/api-config.env" + state '.routes.z1 = [{id: "w1", pattern: "tv.example.gg/*", script: "5stack-playcast-relay"}]' + assert_eq "$(plan_of tv.example.gg)" "skip worker" + assert_eq "$(plan_of cf.example.gg)" "" + cf records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + if grep -E '(POST|PATCH|DELETE) .*(tv|cf)\.example\.gg' "$WORK_DIR/cf-calls"; then + fail "a Worker's hostname was changed" + fi +} + +test_records_start_through_cloudflare_and_keep_their_cloud() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 || fail "records" + assert_eq "$(record ws.example.gg)" "203.0.113.40 true" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + assert_eq "$(record ws.example.gg)" "203.0.113.40 true" + state '.ip = "203.0.113.41"' + cf records >/dev/null 2>&1 || fail "records again" + assert_eq "$(record ws.example.gg)" "203.0.113.41 true" + assert_eq "$(jq '[.records[] | select(.name == "ws.example.gg")] | length' "$WORK_DIR/cf-state.json")" 1 +} + +test_records_that_are_not_ours_are_left_until_replaced() { + panel_at example.gg + zone example.gg + state '.records = [{id: "old1", zone: "z1", type: "A", name: "api.example.gg", content: "198.51.100.7", proxied: true}, {id: "mx", zone: "z1", type: "MX", name: "example.gg", content: "mail.example.gg"}, {id: "txt", zone: "z1", type: "TXT", name: "example.gg", content: "v=spf1"}]' + cf records >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 2 + assert_contains "$(cat "$WORK_DIR/out")" "conflict: api.example.gg A 198.51.100.7 proxied" + assert_eq "$(record api.example.gg)" "198.51.100.7 true" + cf records --replace api.example.gg >"$WORK_DIR/out" 2>&1 || fail "replace: $(cat "$WORK_DIR/out")" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" + assert_eq "$(jq -r '[.records[] | select(.type == "MX" or .type == "TXT") | .id] | join(",")' "$WORK_DIR/cf-state.json")" "mx,txt" +} + +test_rules_are_added_by_ref_and_an_entrypoint_is_never_replaced() { + panel_at example.gg + zone example.gg + state '.rulesets.z1.http_config_settings = {id: "rs9", phase: "http_config_settings", rules: [{id: "theirs", ref: "their_rule", expression: "true"}]}' + cf records >/dev/null 2>&1 + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy again: $(cat "$WORK_DIR/out")" + assert_eq "$(jq -r '[.rulesets.z1.http_config_settings.rules[].ref] | join(",")' "$WORK_DIR/cf-state.json")" "their_rule,5stack_panel,5stack_panel_acme" + assert_eq "$(jq -r '.rulesets.z1.http_config_settings.rules[1].action_parameters | "\(.ssl) \(.rocket_loader) \(.email_obfuscation) \(.disable_zaraz) \(.bic)"' "$WORK_DIR/cf-state.json")" "strict false false true false" + assert_eq "$(jq -r '.rulesets.z1.http_request_cache_settings.rules | length' "$WORK_DIR/cf-state.json")" 1 + assert_contains "$(jq -r '.rulesets.z1.http_request_cache_settings.rules[0].expression' "$WORK_DIR/cf-state.json")" '"api.example.gg"' + assert_not_contains "$(jq -r '.rulesets.z1.http_request_cache_settings.rules[0].expression' "$WORK_DIR/cf-state.json")" '"example.gg"' + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "PUT-OVER-EXISTING" + assert_eq "$(grep -c 'POST /zones/z1/rulesets/rs9/rules ' "$WORK_DIR/cf-calls")" 2 + assert_eq "$(grep -c 'PATCH /zones/z1/rulesets/rs9/rules/' "$WORK_DIR/cf-calls")" 4 +} + +through_cloudflare() { + PATH="$STUB_PATH" curl -sS -o /dev/null -D - "http://$1$2" | tr -d '\r' +} + +redirect_rule() { + jq -c '.rulesets.z1.http_request_dynamic_redirect.rules[] | select(.ref == "5stack_panel_https")' "$WORK_DIR/cf-state.json" +} + +test_always_use_https_goes_off_and_a_rule_redirects_everything_but_the_challenge_path() { + panel_at example.gg + zone example.gg + state '.settings.z1.always_use_https = "on"' + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + assert_eq "$(jq -r '.settings.z1.always_use_https' "$WORK_DIR/cf-state.json")" off + assert_eq "$(redirect_rule | jq -r '.expression')" '(not ssl and not starts_with(http.request.uri.path, "/.well-known/acme-challenge/"))' + assert_eq "$(redirect_rule | jq -r '"\(.action) \(.enabled) \(.action_parameters.from_value.status_code) \(.action_parameters.from_value.preserve_query_string)"')" "redirect true 301 true" + assert_eq "$(redirect_rule | jq -r '.action_parameters.from_value.target_url.expression')" 'concat("https://", http.host, http.request.uri.path)' + rule_at="$(grep -n 'POST /zones/z1/rulesets/.*5stack_panel_https\|PUT /zones/z1/rulesets/phases/http_request_dynamic_redirect' "$WORK_DIR/cf-calls" | head -n 1 | cut -d: -f1)" + off_at="$(grep -n 'PATCH /zones/z1/settings/always_use_https' "$WORK_DIR/cf-calls" | head -n 1 | cut -d: -f1)" + if [ -z "$rule_at" ] || [ -z "$off_at" ] || [ "$rule_at" -ge "$off_at" ]; then + fail "Always Use HTTPS went off before the rule was in" + fi + assert_contains "$(through_cloudflare api.example.gg /.well-known/acme-challenge/5stack-check)" "x-5stack-check: acme-path" + assert_contains "$(through_cloudflare api.example.gg /.well-known/acme-challenge/5stack-check)" "cf-ray:" + assert_contains "$(through_cloudflare api.example.gg /login)" "HTTP/1.1 301" + assert_contains "$(through_cloudflare example.gg /login)" "location: https://example.gg/login" + assert_not_contains "$(cat "$WORK_DIR/out")" "(y/n)" + assert_not_contains "$(cat "$WORK_DIR/out")" "other sites" + for host in example.gg api.example.gg ws.example.gg tv.example.gg search.example.gg console.example.gg; do + assert_eq "$(record "$host")" "203.0.113.40 true" + assert_contains "$(cat "$WORK_DIR/cf-calls")" "HTTP $host/.well-known/acme-challenge/5stack-check cloudflare" + done + if grep -E 'HTTP (demos|hls)\.example\.gg' "$WORK_DIR/cf-calls"; then + fail "probed a host that stays straight to this server" + fi + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy again: $(cat "$WORK_DIR/out")" + assert_eq "$(jq '[.rulesets.z1.http_request_dynamic_redirect.rules[] | select(.ref == "5stack_panel_https")] | length' "$WORK_DIR/cf-state.json")" 1 + assert_eq "$(redirect_rule | jq -r '.expression')" '(not ssl and not starts_with(http.request.uri.path, "/.well-known/acme-challenge/"))' + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "PUT-OVER-EXISTING" +} + +test_with_always_use_https_off_the_rule_covers_only_the_panel() { + panel_at example.gg + zone example.gg + state '.rulesets.z1.http_request_dynamic_redirect = {id: "rs7", phase: "http_request_dynamic_redirect", rules: [{id: "theirs", ref: "their_shop", action: "redirect", expression: "(http.host in {\"shop.example.gg\"} and not ssl)", action_parameters: {from_value: {status_code: 301}}}]}' + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + expression="$(redirect_rule | jq -r '.expression')" + assert_contains "$expression" '(http.host in {"example.gg" "api.example.gg"' + assert_contains "$expression" 'and (not ssl and not starts_with(http.request.uri.path, "/.well-known/acme-challenge/"))' + assert_not_contains "$expression" "demos.example.gg" + assert_not_contains "$expression" "hls.example.gg" + assert_eq "$(jq -r '[.rulesets.z1.http_request_dynamic_redirect.rules[].ref] | join(",")' "$WORK_DIR/cf-state.json")" "their_shop,5stack_panel_https" + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "PATCH /zones/z1/settings/always_use_https" + assert_not_contains "$(cat "$WORK_DIR/out")" "their_shop" +} + +test_the_challenge_path_is_kept_off_security_challenges() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + rule="$(jq -c '.rulesets.z1.http_config_settings.rules[] | select(.ref == "5stack_panel_acme")' "$WORK_DIR/cf-state.json")" + assert_eq "$(jq -r '"\(.action) \(.action_parameters.security_level) \(.action_parameters.bic)"' <<<"$rule")" "set_config essentially_off false" + assert_contains "$(jq -r '.expression' <<<"$rule")" '"api.example.gg"' + assert_contains "$(jq -r '.expression' <<<"$rule")" 'and starts_with(http.request.uri.path, "/.well-known/acme-challenge/")' +} + +test_a_page_rule_that_redirects_the_check_keeps_the_panel_straight_to_this_server() { + panel_at example.gg + zone example.gg + state '.pagerules.z1 = [{id: "pr1", status: "active", targets: [{target: "url", constraint: {operator: "matches", value: "http://*example.gg/*"}}], actions: [{id: "always_use_https"}]}]' + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_contains "$(cat "$WORK_DIR/out")" "The Page Rule for http://*example.gg/* (always_use_https) sends Let's Encrypt's check to HTTPS" + assert_contains "$(cat "$WORK_DIR/out")" "Let's Encrypt can't reach api.example.gg through Cloudflare: Cloudflare redirects http://api.example.gg/.well-known/acme-challenge/5stack-check to https://api.example.gg/.well-known/acme-challenge/5stack-check" + for host in example.gg api.example.gg ws.example.gg tv.example.gg search.example.gg console.example.gg; do + assert_eq "$(record "$host")" "203.0.113.40 false" + done +} + +test_someone_elses_redirect_rule_is_named_and_only_its_host_stays_straight() { + panel_at example.gg + zone example.gg + state '.rulesets.z1.http_request_dynamic_redirect = {id: "rs7", phase: "http_request_dynamic_redirect", rules: [{id: "theirs", ref: "their_api", description: "API to HTTPS", action: "redirect", expression: "(http.host in {\"api.example.gg\"} and not ssl)", action_parameters: {from_value: {status_code: 308}}}, {id: "fine", ref: "their_fine", description: "Fine", action: "redirect", expression: "(not ssl and not starts_with(http.request.uri.path, \"/.well-known/acme-challenge/\"))", action_parameters: {from_value: {status_code: 301}}}]}' + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_contains "$(cat "$WORK_DIR/out")" 'The Redirect Rule "API to HTTPS" can send Let'"'"'s Encrypt'"'"'s check to HTTPS. Leave /.well-known/acme-challenge/ out of it.' + assert_not_contains "$(cat "$WORK_DIR/out")" '"Fine"' + assert_eq "$(record api.example.gg)" "203.0.113.40 false" + assert_eq "$(record ws.example.gg)" "203.0.113.40 true" + status="$(cf status --json)" + assert_contains "$(jq -r '.zones[0].blockers[0]' <<<"$status")" 'The Redirect Rule "API to HTTPS"' + assert_eq "$(jq -r '.zones[0] | "\(.always_https) \(.https_rule)"' <<<"$status")" "off true" +} + +test_a_security_challenge_or_a_panel_without_the_check_keeps_the_host_straight() { + panel_at example.gg + zone example.gg + state '.challenge = ["ws.example.gg"]' + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_contains "$(cat "$WORK_DIR/out")" "Let's Encrypt can't reach ws.example.gg through Cloudflare: Cloudflare answers http://ws.example.gg/.well-known/acme-challenge/5stack-check with a security challenge" + assert_eq "$(record ws.example.gg)" "203.0.113.40 false" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" + state '.challenge = [] | .no_acme_route = true' + cf proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_eq "$(record api.example.gg)" "203.0.113.40 false" +} + +test_a_check_that_never_goes_through_cloudflare_is_named_and_the_host_kept() { + panel_at example.gg + zone example.gg + state '.stale_dns = true' + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + assert_contains "$(cat "$WORK_DIR/out")" "http://example.gg/.well-known/acme-challenge/5stack-check didn't answer through Cloudflare yet" + assert_eq "$(record example.gg)" "203.0.113.40 true" + assert_eq "$(grep -c 'HTTP example.gg/.well-known/acme-challenge/5stack-check' "$WORK_DIR/cf-calls")" 2 +} + +behind_a_reverse_proxy() { + ( + export REVERSE_PROXY=true + cf "$@" + ) +} + +test_behind_a_reverse_proxy_hosts_go_through_cloudflare_even_when_the_check_404s() { + panel_at example.gg + zone example.gg + state '.settings.z1.always_use_https = "on" | .acme_elsewhere = ["example.gg", "api.example.gg", "ws.example.gg", "tv.example.gg", "search.example.gg", "console.example.gg"]' + behind_a_reverse_proxy records >/dev/null 2>&1 || fail "records" + behind_a_reverse_proxy proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + for host in example.gg api.example.gg ws.example.gg tv.example.gg search.example.gg console.example.gg; do + assert_eq "$(record "$host")" "203.0.113.40 true" + assert_contains "$(cat "$WORK_DIR/cf-calls")" "ORIGIN $host" + done + assert_not_contains "$(cat "$WORK_DIR/out")" "Let's Encrypt" + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "/.well-known/acme-challenge/5stack-check" + assert_eq "$(jq -r '[.rulesets.z1.http_config_settings.rules[].ref] | join(",")' "$WORK_DIR/cf-state.json")" "5stack_panel" + assert_eq "$(jq -r '.rulesets.z1.http_config_settings.rules[0].action_parameters.ssl' "$WORK_DIR/cf-state.json")" strict + assert_eq "$(jq -r '.rulesets.z1.http_request_dynamic_redirect.rules // [] | length' "$WORK_DIR/cf-state.json")" 0 + assert_eq "$(jq -r '.settings.z1.always_use_https' "$WORK_DIR/cf-state.json")" on + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "PATCH /zones/z1/settings/always_use_https" +} + +test_a_panel_issuing_its_own_certificates_still_keeps_a_host_whose_check_404s_straight() { + panel_at example.gg + zone example.gg + state '.acme_elsewhere = ["api.example.gg"]' + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_contains "$(cat "$WORK_DIR/out")" "Let's Encrypt can't reach api.example.gg through Cloudflare: http://api.example.gg/.well-known/acme-challenge/5stack-check answers HTTP 404 instead of the panel" + assert_eq "$(record api.example.gg)" "203.0.113.40 false" + assert_eq "$(record ws.example.gg)" "203.0.113.40 true" +} + +test_behind_a_reverse_proxy_a_host_cloudflare_cannot_reach_goes_back_to_dns_only() { + panel_at example.gg + zone example.gg + state '.origin = {"ws.example.gg": 522}' + behind_a_reverse_proxy records >/dev/null 2>&1 || fail "records" + behind_a_reverse_proxy proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_contains "$(cat "$WORK_DIR/out")" "Cloudflare can't reach the panel on ws.example.gg (HTTP 522)" + assert_eq "$(record ws.example.gg)" "203.0.113.40 false" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" +} + +test_behind_a_reverse_proxy_redirect_rules_and_their_blockers_are_left_alone() { + panel_at example.gg + zone example.gg + state '.rulesets.z1.http_request_dynamic_redirect = {id: "rs7", phase: "http_request_dynamic_redirect", rules: [{id: "theirs", ref: "their_api", description: "API to HTTPS", action: "redirect", expression: "(http.host in {\"api.example.gg\"} and not ssl)", action_parameters: {from_value: {status_code: 308}}}]} | .no_redirect_rules = true' + assert_not_contains "$(behind_a_reverse_proxy check)" "redirect_rules" + behind_a_reverse_proxy records >/dev/null 2>&1 || fail "records" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" + behind_a_reverse_proxy proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + assert_not_contains "$(cat "$WORK_DIR/out")" "API to HTTPS" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" + assert_eq "$(behind_a_reverse_proxy status --json | jq -c '.zones[0].blockers')" "[]" +} + +proxied_hosts="example.gg api.example.gg ws.example.gg tv.example.gg search.example.gg console.example.gg" + +created_with() { + grep -E "^POST /zones/z1/dns_records \{\"type\":\"A\",\"name\":\"$1\"" "$WORK_DIR/cf-calls" | grep -o '"proxied":[a-z]*' +} + +test_a_new_host_the_plan_proxies_is_created_through_cloudflare() { + panel_at example.gg + zone example.gg + cf records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + for host in $proxied_hosts; do + assert_eq "$(record "$host")" "203.0.113.40 true" + assert_eq "$(created_with "$host")" '"proxied":true' + done + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "PATCH /zones/z1/dns_records" + assert_contains "$(cat "$WORK_DIR/out")" "api.example.gg → 203.0.113.40 (through Cloudflare)" +} + +test_behind_a_reverse_proxy_a_new_host_the_plan_proxies_is_created_through_cloudflare() { + panel_at example.gg + zone example.gg + state '.settings.z1.always_use_https = "on"' + behind_a_reverse_proxy records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + for host in $proxied_hosts; do + assert_eq "$(record "$host")" "203.0.113.40 true" + assert_eq "$(created_with "$host")" '"proxied":true' + done + assert_eq "$(jq -r '[.rulesets.z1.http_config_settings.rules[].ref] | join(",")' "$WORK_DIR/cf-state.json")" "5stack_panel" + assert_eq "$(jq -r '.rulesets.z1.http_request_dynamic_redirect.rules // [] | length' "$WORK_DIR/cf-state.json")" 0 + assert_eq "$(jq -r '.settings.z1.always_use_https' "$WORK_DIR/cf-state.json")" on + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "/.well-known/acme-challenge/5stack-check" +} + +test_demo_uploads_and_streams_are_created_dns_only_beside_proxied_hosts() { + panel_at example.gg + zone example.gg + printf 'TURN_DOMAIN=turn.example.gg\n' >"$WORK_DIR/panel/overlays/coturn/coturn.env" + cf records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + assert_eq "$(created_with api.example.gg)" '"proxied":true' + for host in demos.example.gg hls.example.gg turn.example.gg; do + assert_eq "$(record "$host")" "203.0.113.40 false" + assert_eq "$(created_with "$host")" '"proxied":false' + done + assert_contains "$(cat "$WORK_DIR/out")" "demos.example.gg → 203.0.113.40 (straight to this server)" +} + +test_the_rules_are_written_before_the_first_proxied_record() { + panel_at example.gg + zone example.gg + state '.settings.z1.always_use_https = "on"' + cf records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + first_proxied="$(grep -n '^POST /zones/z1/dns_records .*"proxied":true' "$WORK_DIR/cf-calls" | head -n 1 | cut -d: -f1)" + [ -n "$first_proxied" ] || fail "no record was created proxied" + for call in 'PUT /zones/z1/rulesets/phases/http_config_settings/entrypoint .*"ref":"5stack_panel"' \ + '/zones/z1/rulesets/.*"ref":"5stack_panel_acme"' \ + '/zones/z1/rulesets/.*"ref":"5stack_panel_no_cache"' \ + '/zones/z1/rulesets/.*"ref":"5stack_panel_https"' \ + 'PATCH /zones/z1/settings/always_use_https'; do + at="$(grep -n -e "$call" "$WORK_DIR/cf-calls" | head -n 1 | cut -d: -f1)" + if [ -z "$at" ] || [ "$at" -ge "$first_proxied" ]; then + fail "$call wasn't in place before the first proxied record: $(cat "$WORK_DIR/cf-calls")" + fi + done + assert_eq "$(jq -r '.settings.z1.always_use_https' "$WORK_DIR/cf-state.json")" off + assert_contains "$(through_cloudflare api.example.gg /.well-known/acme-challenge/5stack-check)" "x-5stack-check: acme-path" + assert_contains "$(through_cloudflare api.example.gg /.well-known/acme-challenge/5stack-check)" "cf-ray:" +} + +test_an_existing_dns_only_record_of_ours_for_a_proxied_host_goes_through_cloudflare() { + panel_at example.gg + zone example.gg + state '.records = [{id: "w1", zone: "z1", type: "A", name: "ws.example.gg", content: "203.0.113.40", proxied: false, ttl: 1, comment: "5stack-panel (managed by update.sh)"}, {id: "d1", zone: "z1", type: "A", name: "demos.example.gg", content: "203.0.113.40", proxied: false, ttl: 1, comment: "5stack-panel (managed by update.sh)"}]' + cf records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + assert_eq "$(record ws.example.gg)" "203.0.113.40 true" + assert_eq "$(jq -r '[.records[] | select(.name == "ws.example.gg") | .id] | join(",")' "$WORK_DIR/cf-state.json")" "w1" + assert_contains "$(cat "$WORK_DIR/cf-calls")" 'PATCH /zones/z1/dns_records/w1' + assert_eq "$(record demos.example.gg)" "203.0.113.40 false" + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" 'dns_records/d1' +} + +test_a_host_whose_acme_check_fails_is_proxied_until_the_late_check_puts_it_back() { + panel_at example.gg + zone example.gg + state '.acme_elsewhere = ["api.example.gg"]' + cf records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" + cf proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_contains "$(cat "$WORK_DIR/out")" "Let's Encrypt can't reach api.example.gg through Cloudflare" + assert_contains "$(cat "$WORK_DIR/out")" "It goes straight to this server again" + assert_eq "$(record api.example.gg)" "203.0.113.40 false" + assert_eq "$(record ws.example.gg)" "203.0.113.40 true" + state '(.records[] | select(.name == "ws.example.gg" and .type == "A")) |= (.proxied = false)' + cf proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$(record ws.example.gg)" "203.0.113.40 true" +} + +test_a_zone_that_is_not_ready_gets_its_records_but_no_rules_or_orange_cloud() { + panel_at example.gg + zone example.gg + state '.bot_fight.z1 = true' + cf records >"$WORK_DIR/out" 2>&1 || fail "records: $(cat "$WORK_DIR/out")" + assert_contains "$(cat "$WORK_DIR/out")" "Bot Fight Mode on" + assert_contains "$(cat "$WORK_DIR/out")" "don't go through Cloudflare yet" + for host in $proxied_hosts; do + assert_eq "$(record "$host")" "203.0.113.40 false" + done + if grep -E '^(PUT|POST|PATCH) /zones/z1/(rulesets|settings)' "$WORK_DIR/cf-calls"; then + fail "rules changed in a zone that isn't ready" + fi +} + +test_reverse_proxy_mode_is_read_from_the_panels_env_config() { + panel_at example.gg + zone example.gg + state '.acme_elsewhere = ["api.example.gg"]' + printf 'REVERSE_PROXY=true\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\n' >"$WORK_DIR/panel/.5stack-env.config" + cf records >/dev/null 2>&1 || fail "records" + cf proxy >"$WORK_DIR/out" 2>&1 || fail "proxy: $(cat "$WORK_DIR/out")" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" +} + +test_a_key_without_redirect_rules_is_named_before_anything_changes() { + panel_at example.gg + zone example.gg + state '.rulesets.z1.http_request_dynamic_redirect = {id: "rs7", phase: "http_request_dynamic_redirect", rules: []} | .no_redirect_rules = true' + assert_contains "$(cf check)" "redirect_rules: The Cloudflare key can't manage Redirect Rules for example.gg" +} + +test_the_key_comes_from_the_file_or_vault_and_never_a_kubernetes_secret() { + panel_at example.gg + zone example.gg + printf 'CLOUDFLARE_API_TOKEN=VAULT\n' >"$WORK_DIR/panel/overlays/local-secrets/cloudflare-secrets.env" + mkdir -p "$WORK_DIR/bin" + FAKE_SECRET="$(printf %s "$TOKEN" | base64)" + FAKE_TOKEN="$TOKEN" + export FAKE_SECRET FAKE_TOKEN + cat >"$WORK_DIR/bin/kubectl" <<'STUB' +#!/bin/bash +echo "$*" >>"$WORK_DIR/kubectl-calls" +printf %s "$FAKE_SECRET" +STUB + chmod +x "$WORK_DIR/bin/kubectl" + STUB_PATH="$WORK_DIR/bin:$STUB_PATH" + cf plan >"$WORK_DIR/out" 2>&1 && fail "read the key from somewhere other than the file or Vault" + assert_contains "$(cat "$WORK_DIR/out")" "No Cloudflare API token" + [ ! -e "$WORK_DIR/kubectl-calls" ] || fail "asked Kubernetes for the key: $(cat "$WORK_DIR/kubectl-calls")" + cat >"$WORK_DIR/bin/vault" <<'STUB' +#!/bin/bash +[ "$*" = "kv get -field=CLOUDFLARE_API_TOKEN kv/cloudflare" ] && printf %s "$FAKE_TOKEN" +STUB + chmod +x "$WORK_DIR/bin/vault" + cf plan >"$WORK_DIR/out" 2>&1 || fail "plan with the key in Vault: $(cat "$WORK_DIR/out")" + assert_contains "$(cat "$WORK_DIR/out")" "api.example.gg" + [ ! -e "$WORK_DIR/kubectl-calls" ] || fail "asked Kubernetes for the key" +} + +test_only_the_selected_zone_is_used() { + panel_at example.gg + zone example.gg + assert_eq "$(CLOUDFLARE_ZONE=other.gg plan_of api.example.gg)" "skip not_in_account" + assert_eq "$(CLOUDFLARE_ZONE=example.gg plan_of api.example.gg)" "yes -" +} + +test_bot_fight_mode_stops_the_proxy_before_anything_changes() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 + state '.bot_fight.z1 = true' + : >"$WORK_DIR/cf-calls" + cf proxy >"$WORK_DIR/out" 2>&1 && fail "proxied with Bot Fight Mode on" + assert_contains "$(cat "$WORK_DIR/out")" "Bot Fight Mode on" + if grep -E '^(PATCH|PUT) ' "$WORK_DIR/cf-calls" || grep -E '^POST /zones/z1/rulesets' "$WORK_DIR/cf-calls"; then + fail "something changed: $(cat "$WORK_DIR/cf-calls")" + fi +} + +test_a_host_cloudflare_cannot_reach_goes_back_to_dns_only() { + panel_at example.gg + zone example.gg + state '.origin = {"ws.example.gg": 522}' + cf records >/dev/null 2>&1 + cf proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_eq "$(record ws.example.gg)" "203.0.113.40 false" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" +} + +test_the_token_never_reaches_curl_arguments() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "TOKEN-IN-ARGS" + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "$TOKEN" + state '.token = "another"' + cf records >"$WORK_DIR/out" 2>&1 && fail "a wrong token worked" + true +} + +test_a_private_address_stops_the_records() { + panel_at example.gg + zone example.gg + state '.ip = "100.72.1.4"' + cf records >"$WORK_DIR/out" 2>&1 && fail "records pointed at a CGNAT address" + assert_contains "$(cat "$WORK_DIR/out")" "private address" + assert_eq "$(jq '.records | length' "$WORK_DIR/cf-state.json")" 0 +} + +test_a_token_without_dns_edit_is_named_before_anything_changes() { + panel_at example.gg + zone example.gg + state '.no_dns_edit = true' + assert_contains "$(cf check)" "dns_edit: example.gg" +} + +test_status_names_a_moved_record_and_fix_puts_it_back() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 + cf proxy >/dev/null 2>&1 + state '(.records[] | select(.name == "ws.example.gg" and .type == "A")) |= (.content = "192.0.2.10")' + state '(.records[] | select(.name == "hls.example.gg" and .type == "A")) |= (.proxied = true)' + status="$(cf status --json)" + assert_eq "$(jq -r '.hosts[] | select(.host == "ws.example.gg") | .state' <<<"$status")" elsewhere + assert_eq "$(jq -r '.hosts[] | select(.host == "hls.example.gg") | "\(.state) \(.want_proxied) \(.why)"' <<<"$status")" "cloud false video" + assert_eq "$(jq -r '.hosts[] | select(.host == "demos.example.gg") | "\(.state) \(.want_proxied)"' <<<"$status")" "ok false" + assert_eq "$(jq -r '.zones[0] | "\(.name) \(.ssl) \(.websockets) \(.bot_fight) \(.tiered_cache)"' <<<"$status")" "example.gg strict on false off" + cf fix ws.example.gg >/dev/null 2>&1 || fail "fix" + assert_eq "$(record ws.example.gg)" "203.0.113.40 true" + cf fix hls.example.gg >/dev/null 2>&1 || fail "fix hls" + assert_eq "$(record hls.example.gg)" "203.0.113.40 false" + cf tiered-cache || fail "tiered cache" + assert_eq "$(jq -r '"\(.argo.z1) \(.tiered.z1)"' "$WORK_DIR/cf-state.json")" "on on" +} + +test_the_stream_relay_puts_a_worker_in_front_of_streams_and_records_stay_its_own() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 || fail "records" + cf proxy >/dev/null 2>&1 || fail "proxy" + assert_eq "$(record hls.example.gg)" "203.0.113.40 false" + cf stream-relay >"$WORK_DIR/out" 2>&1 || fail "stream-relay: $(cat "$WORK_DIR/out")" + assert_eq "$(jq -r '.workers["5stack-stream-relay"] | "\(.account) \(.metadata.main_module) \(.metadata.bindings[0].name)=\(.metadata.bindings[0].text)"' "$WORK_DIR/cf-state.json")" "acct1 worker.js WEB_ORIGIN=https://example.gg" + assert_contains "$(jq -r '.workers["5stack-stream-relay"].module' "$WORK_DIR/cf-state.json")" "5stack-stream-relay" + assert_eq "$(jq -r '.routes.z1[] | "\(.pattern) \(.script) \(.request_limit_fail_open)"' "$WORK_DIR/cf-state.json")" "hls.example.gg/* 5stack-stream-relay false" + assert_eq "$(record hls.example.gg)" "203.0.113.40 true" + assert_eq "$(plan_of hls.example.gg)" "skip worker" + assert_contains "$(jq -r '.rulesets.z1.http_config_settings.rules[0].expression' "$WORK_DIR/cf-state.json")" '"hls.example.gg"' + assert_contains "$(jq -r '.rulesets.z1.http_request_cache_settings.rules[0].expression' "$WORK_DIR/cf-state.json")" '"hls.example.gg"' + : >"$WORK_DIR/cf-calls" + cf records >/dev/null 2>&1 || fail "records again" + cf proxy >/dev/null 2>&1 || fail "proxy again" + assert_eq "$(record hls.example.gg)" "203.0.113.40 true" + if grep -E '(POST|PATCH|DELETE) /zones/z1/dns_records.*hls\.example\.gg' "$WORK_DIR/cf-calls"; then + fail "an update changed the stream relay's record" + fi + cf stream-relay >/dev/null 2>&1 || fail "stream-relay again" + assert_eq "$(jq '.routes.z1 | length' "$WORK_DIR/cf-state.json")" 1 +} + +test_an_unreadable_route_list_leaves_every_host_alone() { + panel_at example.gg + zone example.gg + state '.no_routes_read = true' + assert_eq "$(plan_of hls.example.gg)" "skip routes" + assert_eq "$(plan_of api.example.gg)" "skip routes" + cf records >/dev/null 2>&1 + assert_eq "$(jq '.records | length' "$WORK_DIR/cf-state.json")" 0 +} + +test_a_stream_relay_that_never_answers_puts_streams_back_straight_to_the_server() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 || fail "records" + state '.worker_down = true' + cf stream-relay >"$WORK_DIR/out" 2>&1 + assert_eq "$?" 3 + assert_contains "$(cat "$WORK_DIR/out")" "isn't answering on hls.example.gg" + assert_eq "$(record hls.example.gg)" "203.0.113.40 false" +} + +test_the_stream_relay_stops_on_a_token_that_cannot_deploy_workers() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 || fail "records" + state '.no_workers_write = true' + cf stream-relay >"$WORK_DIR/out" 2>&1 && fail "deployed without Workers Scripts write" + assert_contains "$(cat "$WORK_DIR/out")" "refused the stream relay Worker" + assert_eq "$(record hls.example.gg)" "203.0.113.40 false" + assert_eq "$(jq '.routes.z1 // [] | length' "$WORK_DIR/cf-state.json")" 0 +} + +test_the_stream_relay_needs_a_name_the_free_certificate_covers() { + panel_at panel.example.gg + zone example.gg + cf stream-relay >"$WORK_DIR/out" 2>&1 && fail "relayed a name too deep for the certificate" + assert_contains "$(cat "$WORK_DIR/out")" "deeper than Cloudflare's free certificate" + assert_eq "$(jq '.workers // {} | length' "$WORK_DIR/cf-state.json")" 0 +} + +test_stream_relay_sh_uses_the_managed_key_and_keeps_the_worker_through_updates() { + panel_at example.gg + zone example.gg + cp -R "$TEST_ROOT/utils" "$TEST_ROOT/stream-relay.sh" "$WORK_DIR/panel/" + mkdir -p "$WORK_DIR/panel/cloudflare-workers" + cp -R "$TEST_ROOT/cloudflare-workers/stream-relay" "$WORK_DIR/panel/cloudflare-workers/" + printf 'CLOUDFLARE_MANAGED=true\n' >"$WORK_DIR/panel/.5stack-env.config" + cf records >/dev/null 2>&1 || fail "records" + env -u CLOUDFLARE_API_TOKEN -u CLOUDFLARE_ORIGIN_IPV4 -u HTTPS_PORT PATH="$STUB_PATH" CF_RELAY_WAIT=2 CF_RELAY_PAUSE=0 \ + bash "$WORK_DIR/panel/stream-relay.sh" "$WORK_DIR/out" 2>&1 || fail "stream-relay.sh: $(cat "$WORK_DIR/out")" + assert_contains "$(cat "$WORK_DIR/out")" "Workers Paid" + assert_eq "$(record hls.example.gg)" "203.0.113.40 true" + assert_contains "$(cat "$WORK_DIR/panel/.5stack-env.config")" "CLOUDFLARE_STREAM_RELAY=true" + assert_not_contains "$(cat "$WORK_DIR/out")" "npx" +} + +test_records_set_at_the_start_leave_the_updates_records_step_out() { + panel_at example.gg + zone example.gg + CLOUDFLARE_RECORDS_SET=true update_step cloudflare_update_records >"$WORK_DIR/out" 2>&1 + assert_eq "$(cat "$WORK_DIR/out")" "" + if [ -s "$WORK_DIR/cf-calls" ]; then + fail "Cloudflare was called: $(cat "$WORK_DIR/cf-calls")" + fi + assert_eq "$(jq '.records | length' "$WORK_DIR/cf-state.json")" 0 + update_step cloudflare_update_records >"$WORK_DIR/out" 2>&1 + assert_contains "$(cat "$WORK_DIR/out")" "==> Updating the panel's Cloudflare records" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" + assert_eq "$(record demos.example.gg)" "203.0.113.40 false" +} + +test_records_set_at_the_start_leave_the_proxy_step_out_and_only_put_back_a_host_cloudflare_cannot_reach() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 || fail "records" + : >"$WORK_DIR/cf-calls" + CLOUDFLARE_RECORDS_SET=true update_step cloudflare_update_proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$(cat "$WORK_DIR/out")" "" + if writes; then + fail "the check changed something" + fi + assert_contains "$(cat "$WORK_DIR/cf-calls")" "ORIGIN api.example.gg" + assert_not_contains "$(cat "$WORK_DIR/cf-calls")" "ORIGIN demos.example.gg" + state '.origin = {"ws.example.gg": 522}' + : >"$WORK_DIR/cf-calls" + CLOUDFLARE_RECORDS_SET=true update_step cloudflare_update_proxy >"$WORK_DIR/out" 2>&1 + assert_eq "$(sed 's/^ *//' "$WORK_DIR/out")" "Cloudflare can't reach the panel on ws.example.gg (HTTP 522), so it goes straight to this server again." + assert_eq "$(record ws.example.gg)" "203.0.113.40 false" + assert_eq "$(record api.example.gg)" "203.0.113.40 true" + assert_eq "$(writes | wc -l | tr -d ' ')" 1 + assert_contains "$(writes)" '"name":"ws.example.gg"' +} + +test_records_set_at_the_start_still_put_the_stream_relay_worker_in_front_of_streams() { + panel_at example.gg + zone example.gg + cf records >/dev/null 2>&1 || fail "records" + CLOUDFLARE_RECORDS_SET=true CLOUDFLARE_STREAM_RELAY=true update_step cloudflare_update_proxy >"$WORK_DIR/out" 2>&1 + assert_contains "$(cat "$WORK_DIR/out")" "==> Putting the stream relay Worker in front of streams" + assert_not_contains "$(cat "$WORK_DIR/out")" "==> Putting Cloudflare in front of the panel" + assert_eq "$(record hls.example.gg)" "203.0.113.40 true" + assert_eq "$(jq -r '.routes.z1[] | "\(.pattern) \(.script)"' "$WORK_DIR/cf-state.json")" "hls.example.gg/* 5stack-stream-relay" +} + +test_without_records_set_at_the_start_an_update_puts_cloudflare_in_front_as_before() { + panel_at example.gg + zone example.gg + state '.origin = {"ws.example.gg": 522}' + update_step cloudflare_update_records >/dev/null 2>&1 + : >"$WORK_DIR/cf-calls" + update_step cloudflare_update_proxy >"$WORK_DIR/out" 2>&1 + assert_contains "$(cat "$WORK_DIR/out")" "==> Putting Cloudflare in front of the panel" + assert_contains "$(cat "$WORK_DIR/out")" "api.example.gg through Cloudflare" + assert_contains "$(cat "$WORK_DIR/out")" "Cloudflare isn't in front of every panel hostname yet" + assert_not_contains "$(cat "$WORK_DIR/out")" "stream relay" + assert_contains "$(writes)" "/rulesets/" + assert_eq "$(record ws.example.gg)" "203.0.113.40 false" +} + +test_update_sh_runs_both_cloudflare_steps_through_the_records_set_check() { + local script + script="$(cat "$TEST_ROOT/update.sh")" + assert_contains "$script" "cloudflare_update_records" + assert_contains "$script" "cloudflare_update_proxy" + assert_not_contains "$script" "cloudflare_sync_records" + assert_not_contains "$script" "cloudflare_sync_proxy" + assert_not_contains "$script" "cloudflare_sync_stream_relay" +} + +run_tests diff --git a/tests/scripts/setup_env_test.sh b/tests/scripts/setup_env_test.sh index 36b2229..3d1e6a5 100644 --- a/tests/scripts/setup_env_test.sh +++ b/tests/scripts/setup_env_test.sh @@ -102,11 +102,69 @@ test_hosts_beside_the_web_domain_share_its_parent_as_the_cookie_domain() { test_hosts_under_the_web_domain_leave_the_cookie_domain_alone() { make_panel set_env overlays/config/api-config.env WEB_DOMAIN panel.example.com + export FLAT_HOSTS=false run_setup_env assert_eq "$(env_lines overlays/config/api-config.env API_DOMAIN)" "API_DOMAIN=api.panel.example.com" assert_eq "$(env_lines overlays/config/api-config.env AUTH_COOKIE_DOMAIN)" "" } +test_a_subdomain_puts_every_address_one_level_deep_by_default() { + make_panel + set_env overlays/config/api-config.env WEB_DOMAIN panel.example.com + run_setup_env + assert_eq "$(env_lines overlays/config/api-config.env API_DOMAIN)" "API_DOMAIN=panel-api.example.com" + assert_eq "$(env_lines overlays/config/typesense-config.env TYPESENSE_HOST)" "TYPESENSE_HOST=panel-search.example.com" + assert_eq "$(env_lines overlays/config/api-config.env AUTH_COOKIE_DOMAIN)" "AUTH_COOKIE_DOMAIN=.example.com" + assert_contains "$(cat "$WORK_DIR/setup-output")" "Sign-in then covers all of example.com, so other sites on example.com get the panel's sign-in cookie." +} + +test_flat_hosts_put_every_address_one_level_deep_beside_the_panel() { + make_panel + set_env overlays/config/api-config.env WEB_DOMAIN panel.example.com + export FLAT_HOSTS=true + run_setup_env + assert_eq "$(env_lines overlays/config/api-config.env API_DOMAIN)" "API_DOMAIN=panel-api.example.com" + assert_eq "$(env_lines overlays/config/api-config.env WS_DOMAIN)" "WS_DOMAIN=panel-ws.example.com" + assert_eq "$(env_lines overlays/config/api-config.env RELAY_DOMAIN)" "RELAY_DOMAIN=panel-tv.example.com" + assert_eq "$(env_lines overlays/config/api-config.env DEMOS_DOMAIN)" "DEMOS_DOMAIN=panel-demos.example.com" + assert_eq "$(env_lines overlays/config/api-config.env GAME_STREAM_DOMAIN)" "GAME_STREAM_DOMAIN=panel-hls.example.com" + assert_eq "$(env_lines overlays/config/s3-config.env S3_CONSOLE_HOST)" "S3_CONSOLE_HOST=panel-console.example.com" + assert_eq "$(env_lines overlays/config/typesense-config.env TYPESENSE_HOST)" "TYPESENSE_HOST=panel-search.example.com" + assert_eq "$(env_lines overlays/config/api-config.env AUTH_COOKIE_DOMAIN)" "AUTH_COOKIE_DOMAIN=.example.com" +} + +test_the_free_cloudflare_plan_puts_every_address_one_level_deep_and_says_so() { + make_panel + sed -i.bak 's/^CLOUDFLARE=false$/CLOUDFLARE=true/' "$PANEL/.5stack-env.config" + set_env overlays/config/api-config.env WEB_DOMAIN panel.example.com + export CLOUDFLARE_PLAN=free + run_setup_env + assert_eq "$(env_lines overlays/config/api-config.env API_DOMAIN)" "API_DOMAIN=panel-api.example.com" + assert_eq "$(env_lines overlays/config/typesense-config.env TYPESENSE_HOST)" "TYPESENSE_HOST=panel-search.example.com" + assert_contains "$(cat "$WORK_DIR/setup-output")" "Cloudflare's free plan only covers one level below example.com, so the addresses become panel-api.example.com, panel-ws.example.com, panel-tv.example.com, panel-demos.example.com, panel-hls.example.com, panel-console.example.com, panel-search.example.com." + assert_contains "$(cat "$WORK_DIR/setup-output")" "Other sites on example.com will receive the panel's sign-in cookie." +} + +test_a_paid_cloudflare_plan_keeps_the_default_without_the_free_plan_alert() { + make_panel + sed -i.bak 's/^CLOUDFLARE=false$/CLOUDFLARE=true/' "$PANEL/.5stack-env.config" + set_env overlays/config/api-config.env WEB_DOMAIN panel.example.com + export CLOUDFLARE_PLAN=paid + run_setup_env + assert_eq "$(env_lines overlays/config/api-config.env API_DOMAIN)" "API_DOMAIN=panel-api.example.com" + assert_not_contains "$(cat "$WORK_DIR/setup-output")" "Cloudflare's free plan" + assert_contains "$(cat "$WORK_DIR/setup-output")" "Sign-in then covers all of example.com" +} + +test_a_root_domain_keeps_its_addresses_under_it() { + make_panel + set_env overlays/config/api-config.env WEB_DOMAIN example.com + export FLAT_HOSTS=true + run_setup_env + assert_eq "$(env_lines overlays/config/api-config.env API_DOMAIN)" "API_DOMAIN=api.example.com" + assert_eq "$(env_lines overlays/config/typesense-config.env TYPESENSE_HOST)" "TYPESENSE_HOST=search.example.com" +} + test_an_operator_cookie_domain_that_covers_the_hosts_is_kept() { make_panel set_env overlays/config/api-config.env WEB_DOMAIN panel.example.com @@ -227,4 +285,65 @@ test_update_sh_on_a_vault_install_still_checks_the_vault_login() { assert_contains "$(cat "$WORK_DIR/setup-output")" "Not logged into vault" } +test_managed_cloudflare_keeps_a_panel_behind_its_own_reverse_proxy_there() { + make_panel + set_env overlays/config/api-config.env WEB_DOMAIN example.com + printf 'REVERSE_PROXY=true\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=false\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config" + run_setup_env + assert_contains "$(cat "$WORK_DIR/setup-output")" "no API token yet" + assert_eq "$(env_lines overlays/local-secrets/cloudflare-secrets.env CLOUDFLARE_API_TOKEN)" "CLOUDFLARE_API_TOKEN=" + assert_eq "$(bash -c 'source "$1/utils/utils.sh" >/dev/null 2>&1; echo "$REVERSE_PROXY $CLOUDFLARE"' _ "$PANEL" /dev/null 2>&1; echo "$REVERSE_PROXY $CLOUDFLARE"' _ "$PANEL" "$PANEL/.5stack-env.config" + assert_eq "$(bash -c 'source "$1/utils/utils.sh" >/dev/null 2>&1; echo "$REVERSE_PROXY $CLOUDFLARE"' _ "$PANEL" "$WORK_DIR/setup-output" 2>&1 || fail "$(tail -n 20 "$WORK_DIR/setup-output")" + assert_eq "$(env_lines .5stack-env.config CLOUDFLARE_MANAGED)" "CLOUDFLARE_MANAGED=true" + assert_eq "$(env_lines .5stack-env.config REVERSE_PROXY)" "REVERSE_PROXY=true" +} + +test_a_cloudflare_token_the_5stack_service_passes_is_not_asked_for() { + make_panel + set_sibling_domains + printf 'REVERSE_PROXY=false\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=false\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config" + CLOUDFLARE_API_TOKEN=cf-from-the-service FIVESTACK_RELEASE=2026.11.4.1 run_setup_env + assert_not_contains "$(cat "$WORK_DIR/setup-output")" "no API token yet" + assert_eq "$(env_lines overlays/local-secrets/cloudflare-secrets.env CLOUDFLARE_API_TOKEN)" "CLOUDFLARE_API_TOKEN=" +} + +test_a_vault_install_is_never_told_it_has_no_cloudflare_token() { + make_panel + set_sibling_domains + printf 'REVERSE_PROXY=false\nCLOUDFLARE=true\nCLOUDFLARE_MANAGED=true\nVAULT_MANAGER=true\nKUBECONFIG=/dev/null\n' >"$PANEL/.5stack-env.config" + FIVESTACK_RELEASE=2026.11.4.1 run_setup_env + assert_not_contains "$(cat "$WORK_DIR/setup-output")" "no API token yet" +} + +test_the_cloudflare_flag_saves_managed_mode() { + make_panel + set_env overlays/config/api-config.env WEB_DOMAIN example.com + bash -c 'source "$1/utils/utils.sh" --cloudflare-managed' _ "$PANEL" "$WORK_DIR/setup-output" 2>&1 || fail "$(tail -n 20 "$WORK_DIR/setup-output")" + assert_eq "$(env_lines .5stack-env.config CLOUDFLARE_MANAGED)" "CLOUDFLARE_MANAGED=true" + assert_eq "$(env_lines .5stack-env.config REVERSE_PROXY)" "REVERSE_PROXY=false" +} + +test_the_token_page_asks_for_exactly_what_setup_uses() { + make_panel + set_env overlays/config/api-config.env WEB_DOMAIN example.com + url="$(bash -c 'source "$1/utils/utils.sh" >/dev/null 2>&1; cloudflare_token_url' _ "$PANEL" >"$WORK_DIR/curl-calls" + +case "$method ${url#https://api.tailscale.com/api/v2}" in + "POST /oauth/token") + echo '{"access_token":"ts-access"}' + ;; + "GET /tailnet/-/acl") + if [ "$headers" = true ]; then + printf 'HTTP/2 200\r\netag: "acl-1"\r\n\r\n' + fi + cat "$WORK_DIR/acl.json" + ;; + "POST /tailnet/-/acl") + printf '%s' "$data" >"$WORK_DIR/acl-posted.json" + echo '{}' + ;; + "POST /tailnet/-/keys") + printf '%s' "$data" >"$WORK_DIR/key-request.json" + echo '{"key":"tskey-auth-test"}' + ;; + *) + echo '{"message":"not stubbed"}' + ;; +esac diff --git a/tests/scripts/tailscale-stub/kubectl b/tests/scripts/tailscale-stub/kubectl new file mode 100755 index 0000000..7d6e066 --- /dev/null +++ b/tests/scripts/tailscale-stub/kubectl @@ -0,0 +1,10 @@ +#!/bin/bash + +printf '%s\n' "$*" >>"$WORK_DIR/kubectl-calls" +case " $* " in + *" get nodes "*) + if [ -f "$WORK_DIR/pod-cidrs" ]; then + cat "$WORK_DIR/pod-cidrs" + fi + ;; +esac diff --git a/tests/scripts/tailscale_api_test.sh b/tests/scripts/tailscale_api_test.sh new file mode 100644 index 0000000..b9a7ebc --- /dev/null +++ b/tests/scripts/tailscale_api_test.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +TS_STUB="$(cd "$(dirname "${BASH_SOURCE[0]}")/tailscale-stub" && pwd)" + +policy() { + if [ -n "${1:-}" ]; then + printf '%s\n' "$1" >"$WORK_DIR/acl.json" + else + printf '%s\n' '{"acls": [{"action": "accept", "src": ["*"], "dst": ["*:*"]}]}' >"$WORK_DIR/acl.json" + fi +} + +tailnet() { + ( + export PATH="$TS_STUB:$PATH" + unset PANEL_POD_RANGE PANEL_SERVICE_RANGE PANEL_TAILSCALE_TAG + if [ -n "${POD_RANGE:-}" ]; then + export PANEL_POD_RANGE="$POD_RANGE" + fi + if [ -n "${TAG:-}" ]; then + export PANEL_TAILSCALE_TAG="$TAG" + fi + # shellcheck source=utils/tailscale-api.sh + source "$TEST_ROOT/utils/tailscale-api.sh" + "$@" + ) +} + +posted() { + jq -c "$1" "$WORK_DIR/acl-posted.json" +} + +test_a_panel_on_the_shared_range_keeps_the_shared_lines() { + policy + echo "10.42.0.0/24" >"$WORK_DIR/pod-cidrs" + tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(posted '.autoApprovers.routes')" '{"10.42.0.0/16":["tag:fivestack"]}' + assert_eq "$(posted '.grants')" '[{"src":["tag:fivestack","10.42.0.0/16"],"dst":["tag:fivestack","10.42.0.0/16"],"ip":["*"]}]' + assert_eq "$(posted '.tagOwners')" null +} + +test_a_panel_with_its_own_range_gets_its_own_lines_and_never_the_shared_ones() { + policy + echo "10.52.0.0/24" >"$WORK_DIR/pod-cidrs" + POD_RANGE=10.52.0.0/16 TAG=tag:fivestack-abc123 tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(posted '.autoApprovers.routes')" '{"10.52.0.0/16":["tag:fivestack-abc123"]}' + assert_eq "$(posted '.grants')" '[{"src":["tag:fivestack-abc123","10.52.0.0/16"],"dst":["tag:fivestack-abc123","10.52.0.0/16"],"ip":["*"]}]' + assert_eq "$(posted '.tagOwners')" '{"tag:fivestack-abc123":["autogroup:admin"]}' + assert_not_contains "$(cat "$WORK_DIR/acl-posted.json")" "10.42." + assert_eq "$(posted '.acls | length')" 1 +} + +test_another_panels_shared_lines_stay_as_they_are() { + policy '{"autoApprovers": {"routes": {"10.42.0.0/16": ["tag:fivestack"]}}, "grants": [{"src": ["tag:fivestack", "10.42.0.0/16"], "dst": ["tag:fivestack", "10.42.0.0/16"], "ip": ["*"]}]}' + POD_RANGE=10.52.0.0/16 TAG=tag:fivestack-abc123 tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(posted '.autoApprovers.routes')" '{"10.42.0.0/16":["tag:fivestack"],"10.52.0.0/16":["tag:fivestack-abc123"]}' + assert_eq "$(posted '.grants | length')" 2 + assert_eq "$(posted '.grants[0].src')" '["tag:fivestack-abc123","10.52.0.0/16"]' +} + +test_a_cluster_on_its_own_range_without_it_in_the_config_is_refused() { + policy + echo "10.52.0.0/24" >"$WORK_DIR/pod-cidrs" + if tailnet update_acl_for_fivestack ts-access >"$WORK_DIR/out" 2>&1; then + fail "the policy was edited for a panel whose pods are on 10.52.0.0/24" + fi + [ ! -e "$WORK_DIR/acl-posted.json" ] || fail "posted: $(cat "$WORK_DIR/acl-posted.json")" + assert_contains "$(cat "$WORK_DIR/out")" "this panel's pods are on 10.52.0.0/24, outside 10.42.0.0/16. Set PANEL_POD_RANGE and PANEL_TAILSCALE_TAG" +} + +test_game_node_keys_carry_the_panels_own_tag() { + tailnet create_auth_key ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(jq -c '.capabilities.devices.create.tags' "$WORK_DIR/key-request.json")" '["tag:fivestack"]' + assert_eq "$(jq -c '.capabilities.devices.create | [.reusable, .ephemeral, .preauthorized]' "$WORK_DIR/key-request.json")" '[false,false,true]' + assert_eq "$(cat "$WORK_DIR/out")" "tskey-auth-test" + TAG=tag:fivestack-abc123 tailnet create_auth_key ts-access >"$WORK_DIR/out" 2>&1 || fail "$(cat "$WORK_DIR/out")" + assert_eq "$(jq -c '.capabilities.devices.create.tags' "$WORK_DIR/key-request.json")" '["tag:fivestack","tag:fivestack-abc123"]' +} + +run_tests diff --git a/tests/scripts/write_overlays_test.sh b/tests/scripts/write_overlays_test.sh index dd3589f..b3a755e 100644 --- a/tests/scripts/write_overlays_test.sh +++ b/tests/scripts/write_overlays_test.sh @@ -210,4 +210,31 @@ test_the_api_only_takes_the_shared_secret_from_ai_secrets() { fi } +test_managed_cloudflare_keeps_http01_and_no_cloudflare_key_in_the_cluster() { + with_s3_endpoint "" + printf 'CLOUDFLARE_API_TOKEN=cf-key-0123456789abcdef\n' >"$(overlay local-secrets)/cloudflare-secrets.env" + ( + export CLOUDFLARE_MANAGED=true CLOUDFLARE_ZONES=example.gg,example.net + write + ) || fail "write_overlays failed: $(cat "$WORK_DIR/output")" + for o in local-secrets-https vault-https; do + manifest="$(kubectl kustomize "$(overlay "$o")")" || fail "kubectl kustomize $o failed" + issuer="$(awk '/^kind: Issuer/,/^---/' <<<"$manifest")" + assert_contains "$issuer" "http01:" + assert_contains "$issuer" "traefik.ingress.kubernetes.io/router.priority: \"1000000\"" + assert_not_contains "$manifest" "dns01" + assert_not_contains "$manifest" "cloudflare-secrets" + assert_not_contains "$manifest" "CLOUDFLARE_API_TOKEN" + assert_not_contains "$manifest" "cf-key-0123456789abcdef" + rendered="$(kinds_and_names "$manifest")" + assert_contains "$rendered" "Ingress/acme-check" + assert_contains "$rendered" "Middleware/acme-check" + done + check="$(awk '/^kind: Ingress/,/^---/' <<<"$(kubectl kustomize "$(overlay local-secrets-https)")")" + assert_contains "$check" "path: /.well-known/acme-challenge/5stack-check" + assert_contains "$check" "traefik.ingress.kubernetes.io/router.entrypoints: web" + assert_contains "$check" "traefik.ingress.kubernetes.io/router.middlewares: 5stack-acme-check@kubernetescrd" + assert_not_contains "$(cat "$(overlay local-secrets-http)/kustomization.yaml")" "acme-check" +} + run_tests diff --git a/tests/timescaledb/extension_test.sh b/tests/timescaledb/extension_test.sh index 7d16bf4..ca50508 100755 --- a/tests/timescaledb/extension_test.sh +++ b/tests/timescaledb/extension_test.sh @@ -27,11 +27,44 @@ test_extension_not_installed_returns_quietly() { load_scripts update_timescaledb_extension.sh stub default-version 2.30.2 output="$(update_timescaledb_extension 2>&1)" || fail "returned $?: $output" - assert_eq "$output" "" + assert_eq "$output" " Waiting for TimescaleDB to start (the first start pulls its image)" assert_not_called "ALTER EXTENSION" assert_not_called "rollout restart" } +test_the_wait_for_timescaledb_says_what_it_waits_for() { + load_scripts update_timescaledb_extension.sh + stub extversion 2.30.2 + stub default-version 2.30.2 + output="$(update_timescaledb_extension 2>&1)" || fail "returned $?: $output" + assert_eq "$(head -n 1 <<<"$output")" " Waiting for TimescaleDB to start (the first start pulls its image)" + assert_called "rollout status statefulset/timescaledb --timeout=300s" +} + +test_every_query_has_a_time_limit() { + load_scripts update_timescaledb_extension.sh + stub extversion 2.26.1 + stub default-version 2.30.2 + update_timescaledb_extension >"$STUB_DIR/out" 2>&1 || fail "returned $?: $(cat "$STUB_DIR/out")" + assert_eq "$(grep -c ' exec timescaledb-0 ' "$STUB_DIR/calls")" 4 + assert_eq "$(grep ' exec timescaledb-0 ' "$STUB_DIR/calls" | grep -vc -- '--request-timeout=120s exec timescaledb-0 ')" 0 +} + +test_a_query_that_hangs_is_cut_off_and_warned_about() { + load_scripts update_timescaledb_extension.sh + stub exec-hangs 30 + started="$(date +%s)" + status=0 + TIMESCALEDB_EXEC_TIMEOUT=1 update_timescaledb_extension >"$STUB_DIR/out" 2>&1 || status=$? + took=$(($(date +%s) - started)) + assert_eq "$status" 1 + if [ "$took" -ge 10 ]; then + fail "a hung query held the update for ${took}s" + fi + assert_contains "$(cat "$STUB_DIR/out")" "timescaledb did not answer; its extension was not updated" + assert_not_called "ALTER EXTENSION" +} + test_rollout_never_ready_warns_without_exiting() { load_scripts update_timescaledb_extension.sh stub_rc rollout 1 diff --git a/tests/timescaledb/major_upgrade_test.sh b/tests/timescaledb/major_upgrade_test.sh index 9aa6366..fdcca03 100755 --- a/tests/timescaledb/major_upgrade_test.sh +++ b/tests/timescaledb/major_upgrade_test.sh @@ -9,7 +9,7 @@ KUBECONFIG_FLAG=--kubeconfig=/home/ops/.kube/prod.yaml METRICS_WARNING='E1014 18:50:17.402803 77293 memcache.go:287] "Unhandled Error" err="couldn'"'"'t get resource list for metrics.k8s.io/v1beta1: the server is currently unable to handle the request"' setup_upgrade() { - load_scripts upgrade_timescaledb_major.sh + load_scripts update_timescaledb_extension.sh upgrade_timescaledb_major.sh export TIMESCALEDB_UPGRADE_TTY=/nonexistent/tty PANEL_DIR="$STUB_DIR/panel" manifest "$TARGET" @@ -88,6 +88,22 @@ test_fresh_install_without_a_volume_is_a_no_op() { assert_not_called "apply " } +test_a_panel_without_a_volume_is_a_fresh_install() { + setup_upgrade + plan_timescaledb_major_upgrade >"$STUB_DIR/out" 2>&1 || fail "returned $?" + assert_eq "$TIMESCALEDB_FRESH_INSTALL" true +} + +test_a_panel_with_a_volume_is_not_a_fresh_install() { + setup_upgrade + stub pvc persistentvolumeclaim/timescaledb-pvc + stub pg-version 18 + TIMESCALEDB_FRESH_INSTALL=true + plan_timescaledb_major_upgrade >"$STUB_DIR/out" 2>&1 || fail "returned $?: $(cat "$STUB_DIR/out")" + assert_eq "$TIMESCALEDB_FRESH_INSTALL" "" + assert_called "--request-timeout=120s exec timescaledb-0 -c timescaledb -- cat /var/lib/postgresql/data/PG_VERSION" +} + test_unreachable_cluster_refuses() { setup_upgrade stub stderr "The connection to the server prod:6443 was refused - did you specify the right host or port?" @@ -369,6 +385,7 @@ test_finish_runs_from_the_marker_in_a_later_run() { finish_timescaledb_major_upgrade >"$STUB_DIR/out" 2>&1 || fail "returned $?: $(cat "$STUB_DIR/out")" output="$(cat "$STUB_DIR/out")" assert_called "--all --analyze-in-stages" + assert_called "--request-timeout=3600s exec timescaledb-0 -c timescaledb -- sh -c" assert_called "rollout restart deployment/hasura deployment/api" assert_called "delete configmap timescaledb-major-upgraded timescaledb-major-upgrade" assert_contains "$output" "set the image in base/timescaledb/stateful-set.yaml to $OLD_PINNED" diff --git a/tests/timescaledb/stub/kubectl b/tests/timescaledb/stub/kubectl index fe7371f..7f9b55f 100755 --- a/tests/timescaledb/stub/kubectl +++ b/tests/timescaledb/stub/kubectl @@ -56,6 +56,14 @@ case " $* " in esac printf '%s%s\n' "$*" "${kind:+ <$kind>}" >>"$STUB_DIR/calls" +case " $* " in + *" exec timescaledb-0 "*) + if [ -s "$STUB_DIR/exec-hangs" ]; then + exec sleep "$(cat "$STUB_DIR/exec-hangs")" + fi + ;; +esac + for last; do :; done case " $* " in diff --git a/update.sh b/update.sh index f963857..64f0173 100755 --- a/update.sh +++ b/update.sh @@ -59,6 +59,10 @@ step "Installing Traefik" install_traefik || die "failed to install Traefik" ok "Traefik is ready" +if [ "$CLOUDFLARE_MANAGED" = true ]; then + cloudflare_update_records +fi + step "Building overlay manifests" # Whether an operator has asked for a TURN relay at all. Read straight from # coturn's own env file rather than the environment: this is the same file the @@ -132,8 +136,10 @@ kubectl --kubeconfig=$KUBECONFIG delete deployment timescaledb -n 5stack 2>/dev kubectl --kubeconfig=$KUBECONFIG delete deployment typesense -n 5stack 2>/dev/null kubectl --kubeconfig=$KUBECONFIG delete deployment redis -n 5stack 2>/dev/null -step "Updating the TimescaleDB extension" -update_timescaledb_extension +if [ "$TIMESCALEDB_FRESH_INSTALL" != true ]; then + step "Updating the TimescaleDB extension" + update_timescaledb_extension +fi finish_timescaledb_major_upgrade GIT_SHA="${FIVESTACK_RELEASE:-$(git rev-parse HEAD)}" @@ -153,6 +159,10 @@ if [ "$REVERSE_PROXY" != true ]; then watch_ssl_status || SSL_OK=false fi +if [ "$CLOUDFLARE_MANAGED" = true ] && [ "$SSL_OK" = true ]; then + cloudflare_update_proxy +fi + if [ "$SSL_OK" != true ]; then banner "5Stack : Updated (SSL incomplete)" # install.sh and game-node-server-setup.sh source this file, and an `exit` diff --git a/utils/cloudflare_api.sh b/utils/cloudflare_api.sh new file mode 100644 index 0000000..312d992 --- /dev/null +++ b/utils/cloudflare_api.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Calls the Cloudflare API with CLOUDFLARE_API_TOKEN. The token reaches curl on +# stdin as a config line, never as an argument, so ps and the shell's history +# never see it. + +: "${CLOUDFLARE_API:=https://api.cloudflare.com/client/v4}" + +cfa() { + local method="$1" path="$2" body="${3:-}" + local args=(-sS --max-time 30 -X "$method" -H "Content-Type: application/json") + if [ -n "$body" ]; then + args+=(--data-binary "$body") + fi + printf 'header = "Authorization: Bearer %s"\n' "$CLOUDFLARE_API_TOKEN" | + curl -K - "${args[@]}" "$CLOUDFLARE_API$path" +} + +cfa_ok() { + jq -e '.success == true' >/dev/null 2>&1 +} + +cfa_errors() { + jq -r '[.errors[]?.message] | join("; ")' 2>/dev/null +} diff --git a/utils/cloudflare_setup.sh b/utils/cloudflare_setup.sh new file mode 100755 index 0000000..4020d98 --- /dev/null +++ b/utils/cloudflare_setup.sh @@ -0,0 +1,908 @@ +#!/bin/bash + +# Manages the panel's hostnames on Cloudflare with CLOUDFLARE_API_TOKEN. +# +# cloudflare_setup.sh plan [--json] what each hostname gets; changes nothing +# cloudflare_setup.sh records [--replace a,b] rules, then DNS records, proxied from the start +# cloudflare_setup.sh proxy checks each proxied host, and puts one back to +# DNS-only when Cloudflare or Let's Encrypt can't reach it +# cloudflare_setup.sh origins changes nothing, unless Cloudflare can't reach a proxied +# host: that one goes back to DNS-only +# cloudflare_setup.sh status --json records and settings against what setup left +# cloudflare_setup.sh fix puts one record back +# cloudflare_setup.sh tiered-cache turns on Smart Tiered Cache +# cloudflare_setup.sh stream-relay the stream relay Worker on GAME_STREAM_DOMAIN +# +# Every hostname goes through Cloudflare except demo uploads (DEMOS_DOMAIN, +# over Cloudflare's 100 MB request limit), streams (GAME_STREAM_DOMAIN: the CDN +# terms keep video off the plain proxy) and TURN (UDP). The Worker hostname +# and hosts a Workers route answers are never touched. A proxied name must be +# the zone, one label below it, or covered by one of the zone's certificate +# packs: Cloudflare's free certificate covers nothing deeper. Nothing is +# proxied when HTTPS_PORT isn't 443. Certificates stay with cert-manager's +# HTTP-01 check, so plain HTTP on /.well-known/acme-challenge/ must reach the +# panel: Always Use HTTPS goes off and a redirect rule sends everything else to +# HTTPS. Behind the operator's own reverse proxy (REVERSE_PROXY=true) the panel +# issues no certificates, so none of that is set up or checked, and a host goes +# back to DNS-only only when Cloudflare can't reach the origin. With +# CLOUDFLARE_ZONE set, only that zone is used. + +CF_SETUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +: "${PANEL_DIR:=$(cd "$CF_SETUP_DIR/.." && pwd)}" +# shellcheck source=utils/colors.sh +source "$CF_SETUP_DIR/colors.sh" +# shellcheck source=utils/cloudflare_api.sh +source "$CF_SETUP_DIR/cloudflare_api.sh" + +CF_COMMENT="5stack-panel (managed by update.sh)" +CF_CONFIG_REF="5stack_panel" +CF_CACHE_REF="5stack_panel_no_cache" +CF_REDIRECT_REF="5stack_panel_https" +CF_ACME_REF="5stack_panel_acme" +CF_ACME_PATH="/.well-known/acme-challenge/" +CF_ACME_CHECK="/.well-known/acme-challenge/5stack-check" +CF_PROXIED_KEYS="WEB_DOMAIN API_DOMAIN WS_DOMAIN TYPESENSE_HOST RELAY_DOMAIN S3_CONSOLE_HOST" +CF_DIRECT_KEYS="DEMOS_DOMAIN GAME_STREAM_DOMAIN TURN_DOMAIN" +CF_NO_CACHE_KEYS="API_DOMAIN WS_DOMAIN TYPESENSE_HOST RELAY_DOMAIN GAME_STREAM_DOMAIN" +CF_RELAY_WORKER="5stack-stream-relay" +: "${CF_RELAY_WAIT:=12}" +: "${CF_RELAY_PAUSE:=5}" +: "${CF_TRACE_URL:=https://1.1.1.1/cdn-cgi/trace}" +: "${CF_DOH_URL:=https://1.1.1.1/dns-query}" +: "${CF_PROBE_WAIT:=12}" +: "${CF_PROBE_PAUSE:=5}" +: "${CLOUDFLARE_ORIGIN_IPV6:=}" + +cf_env_value() { + local file="$1" key="$2" + grep -hs "^$key=" "$file" | tail -n 1 | cut -d= -f2- | tr -d '\r"' +} + +cf_load_hosts() { + local config="$PANEL_DIR/overlays/config" key value + for key in WEB_DOMAIN API_DOMAIN WS_DOMAIN RELAY_DOMAIN DEMOS_DOMAIN GAME_STREAM_DOMAIN CLOUDFLARE_WORKER_DOMAIN; do + if [ -z "${!key}" ]; then + value="$(cf_env_value "$config/api-config.env" "$key")" + printf -v "$key" '%s' "$value" + fi + done + [ -n "$S3_CONSOLE_HOST" ] || S3_CONSOLE_HOST="$(cf_env_value "$config/s3-config.env" S3_CONSOLE_HOST)" + [ -n "$TYPESENSE_HOST" ] || TYPESENSE_HOST="$(cf_env_value "$config/typesense-config.env" TYPESENSE_HOST)" + [ -n "$TURN_DOMAIN" ] || TURN_DOMAIN="$(cf_env_value "$PANEL_DIR/overlays/coturn/coturn.env" TURN_DOMAIN)" + [ -n "$HTTPS_PORT" ] || HTTPS_PORT="$(cf_env_value "$config/traefik-config.env" HTTPS_PORT)" + [ -n "$REVERSE_PROXY" ] || REVERSE_PROXY="$(cf_env_value "$PANEL_DIR/.5stack-env.config" REVERSE_PROXY)" +} + +cf_reverse_proxy() { + [ "$REVERSE_PROXY" = true ] +} + +cf_load_token() { + if [ -n "$CLOUDFLARE_API_TOKEN" ]; then + return 0 + fi + CLOUDFLARE_API_TOKEN="$(cf_env_value "$PANEL_DIR/overlays/local-secrets/cloudflare-secrets.env" CLOUDFLARE_API_TOKEN)" + if [ "$CLOUDFLARE_API_TOKEN" = VAULT ] || { [ -z "$CLOUDFLARE_API_TOKEN" ] && [ "$VAULT_MANAGER" = true ]; }; then + CLOUDFLARE_API_TOKEN="" + if command -v vault >/dev/null 2>&1; then + CLOUDFLARE_API_TOKEN="$(vault kv get -field=CLOUDFLARE_API_TOKEN kv/cloudflare 2>/dev/null)" + fi + fi + [ -n "$CLOUDFLARE_API_TOKEN" ] +} + +cf_lower() { + tr '[:upper:]' '[:lower:]' <<<"$1" +} + +cf_zone() { + local host candidate response + host="$(cf_lower "$1")" + candidate="$host" + while [[ "$candidate" == *.* ]]; do + if [ -z "$CLOUDFLARE_ZONE" ] || [ "$candidate" = "$(cf_lower "$CLOUDFLARE_ZONE")" ]; then + response="$(cfa GET "/zones?name=$candidate&status=active")" || return 1 + if jq -e '.result | length > 0' >/dev/null 2>&1 <<<"$response"; then + jq -r '.result[0] | "\(.id) \(.name) \(.plan.name // "unknown")"' <<<"$response" + return 0 + fi + fi + candidate="${candidate#*.}" + done + return 1 +} + +cf_covered() { + local host="$1" zone="$2" zone_id="$3" rel packs + if [ "$host" = "$zone" ]; then + return 0 + fi + rel="${host%".$zone"}" + if [[ "$rel" != *.* ]]; then + return 0 + fi + packs="$(cfa GET "/zones/$zone_id/ssl/certificate_packs?status=all")" || return 1 + jq -e --arg h "$host" ' + [.result[]? | select(.status == "active") | .hosts[]?] as $names + | ($h | split(".")[1:] | join(".")) as $parent + | any($names[]; . == $h or . == ("*." + $parent)) + ' >/dev/null 2>&1 <<<"$packs" +} + +cf_worker_routed() { + local host="$1" zone_id="$2" routes + routes="$(cfa GET "/zones/$zone_id/workers/routes")" + if ! cfa_ok <<<"$routes"; then + echo routes + return 0 + fi + jq -e --arg h "$host" 'any(.result[]?; (.pattern | split("/")[0]) == $h)' >/dev/null 2>&1 <<<"$routes" && echo worker +} + +cf_keyed_hosts() { + local key + for key in $CF_PROXIED_KEYS $CF_DIRECT_KEYS; do + if [ -n "${!key}" ]; then + printf '%s %s\n' "$key" "$(cf_lower "${!key}")" + fi + done | awk '!seen[$2]++' +} + +cf_plan() { + local key host zone zone_id zone_name proxied why routed + while read -r key host; do + if [ -n "$CLOUDFLARE_WORKER_DOMAIN" ] && [ "$host" = "$(cf_lower "$CLOUDFLARE_WORKER_DOMAIN")" ]; then + continue + fi + if ! zone="$(cf_zone "$host")"; then + printf '%s %s - - skip %s\n' "$key" "$host" "not_in_account" + continue + fi + read -r zone_id zone_name _ <<<"$zone" + proxied=yes + why=- + if [[ " $CF_DIRECT_KEYS " == *" $key "* ]]; then + proxied=no + case "$key" in + TURN_DOMAIN) why=udp ;; + GAME_STREAM_DOMAIN) why=video ;; + *) why=uploads ;; + esac + elif [ -n "$HTTPS_PORT" ] && [ "$HTTPS_PORT" != 443 ]; then + proxied=no + why=port + elif ! cf_covered "$host" "$zone_name" "$zone_id"; then + proxied=no + why=depth + fi + routed="$(cf_worker_routed "$host" "$zone_id")" + if [ -n "$routed" ]; then + proxied=skip + why="$routed" + fi + printf '%s %s %s %s %s %s\n' "$key" "$host" "$zone_id" "$zone_name" "$proxied" "$why" + done < <(cf_keyed_hosts) +} + +cf_public_ip() { + if [ -n "$CLOUDFLARE_ORIGIN_IPV4" ]; then + return 0 + fi + CLOUDFLARE_ORIGIN_IPV4="$(curl -sS --max-time 10 "$CF_TRACE_URL" | sed -n 's/^ip=//p' | tr -d '\r')" + [ -n "$CLOUDFLARE_ORIGIN_IPV4" ] +} + +cf_private_ip() { + local ip="$1" a b + IFS=. read -r a b _ _ <<<"$ip" + [ "$a" = 10 ] || [ "$a" = 127 ] || [ "$a" = 0 ] || + { [ "$a" = 172 ] && [ "$b" -ge 16 ] && [ "$b" -le 31 ]; } || + { [ "$a" = 192 ] && [ "$b" = 168 ]; } || + { [ "$a" = 169 ] && [ "$b" = 254 ]; } || + { [ "$a" = 100 ] && [ "$b" -ge 64 ] && [ "$b" -le 127 ]; } +} + +cf_check() { + local zone_id zone_name ids response probe phase phases + zone_id="" + phases="http_config_settings http_request_cache_settings http_request_dynamic_redirect" + if cf_reverse_proxy; then + phases="http_config_settings http_request_cache_settings" + fi + ids="$(cf_plan | awk '$3 != "-" { print $3 " " $4 }' | sort -u)" + if [ -z "$ids" ]; then + echo "zone: none of the panel's hostnames is in a zone this token can read" + return + fi + while read -r zone_id zone_name; do + response="$(cfa POST "/zones/$zone_id/dns_records" "$(jq -nc --arg n "_5stack-check.$zone_name" '{type: "TXT", name: $n, content: "\"5stack\"", ttl: 60}')")" + if probe="$(jq -er '.result.id' 2>/dev/null <<<"$response")"; then + cfa DELETE "/zones/$zone_id/dns_records/$probe" >/dev/null + else + echo "dns_edit: $zone_name: $(cfa_errors <<<"$response")" + fi + cfa GET "/zones/$zone_id/settings/websockets" | jq -e '.result.value == "on"' >/dev/null 2>&1 || + echo "websockets: $zone_name has WebSockets off, so live updates can't connect. Turn it on under Network." + response="$(cfa GET "/zones/$zone_id/bot_management")" + if ! cfa_ok <<<"$response"; then + echo "bot_read: $zone_name: $(cfa_errors <<<"$response")" + elif jq -e '.result.fight_mode == true' >/dev/null 2>&1 <<<"$response"; then + echo "bot_fight: $zone_name has Bot Fight Mode on, which blocks game servers talking to the panel. Turn it off under Security, Bots." + fi + cfa GET "/zones/$zone_id/ssl/universal/settings" | jq -e '.result.enabled != false' >/dev/null 2>&1 || + echo "universal_ssl: $zone_name has Universal SSL off, so Cloudflare has no certificate for the proxied names." + for phase in $phases; do + response="$(cfa GET "/zones/$zone_id/rulesets/phases/$phase/entrypoint")" + if ! cfa_ok <<<"$response" && ! jq -e 'any(.errors[]?; .code == 10003)' >/dev/null 2>&1 <<<"$response"; then + if [ "$phase" = http_request_dynamic_redirect ]; then + echo "redirect_rules: The Cloudflare key can't manage Redirect Rules for $zone_name, which keep Let's Encrypt's check reaching this server. Connect Cloudflare again, or make a new key with ./cloudflare.sh." + else + echo "rules: $zone_name: $phase: $(cfa_errors <<<"$response")" + fi + fi + done + done <<<"$ids" +} + +cf_records_json() { + local zone_id="$1" host="$2" + cfa GET "/zones/$zone_id/dns_records?name=$host&per_page=100" | + jq -c '[.result[]? | select(.type == "A" or .type == "AAAA" or .type == "CNAME")]' +} + +cf_record_body() { + local type="$1" host="$2" content="$3" proxied="$4" + jq -nc --arg t "$type" --arg n "$host" --arg c "$content" --argjson p "$proxied" --arg m "$CF_COMMENT" \ + '{type: $t, name: $n, content: $c, proxied: $p, ttl: 1, comment: $m}' +} + +cf_sync_host() { + local zone_id="$1" host="$2" want="$3" replace="$4" records foreign type content id proxied current + records="$(cf_records_json "$zone_id" "$host")" + foreign="$(jq -c --arg m "$CF_COMMENT" '[.[] | select(.comment != $m)]' <<<"$records")" + if [ "$(jq length <<<"$foreign")" -gt 0 ]; then + if [ "$replace" != true ]; then + jq -r --arg h "$host" '.[] | "conflict: \($h) \(.type) \(.content)\(if .proxied then " proxied" else "" end)"' <<<"$foreign" + return 0 + fi + for id in $(jq -r '.[].id' <<<"$foreign"); do + cfa DELETE "/zones/$zone_id/dns_records/$id" | cfa_ok || { + err "Could not remove a record for $host" + return 1 + } + done + fi + for type in A AAAA; do + content="$CLOUDFLARE_ORIGIN_IPV4" + if [ "$type" = AAAA ]; then + content="$CLOUDFLARE_ORIGIN_IPV6" + fi + current="$(jq -c --arg t "$type" --arg m "$CF_COMMENT" '[.[] | select(.type == $t and .comment == $m)][0] // empty' <<<"$records")" + if [ -z "$content" ]; then + if [ -n "$current" ]; then + cfa DELETE "/zones/$zone_id/dns_records/$(jq -r .id <<<"$current")" >/dev/null + fi + continue + fi + proxied="$want" + if [ "$want" = keep ]; then + proxied="$(jq -r 'if . == null then false else (.proxied // false) end' <<<"${current:-null}")" + fi + if [ -z "$current" ]; then + cfa POST "/zones/$zone_id/dns_records" "$(cf_record_body "$type" "$host" "$content" "$proxied")" | cfa_ok || { + err "Could not add the $type record for $host" + return 1 + } + elif [ "$(jq -r '"\(.content) \(.proxied)"' <<<"$current")" != "$content $proxied" ]; then + cfa PATCH "/zones/$zone_id/dns_records/$(jq -r .id <<<"$current")" "$(cf_record_body "$type" "$host" "$content" "$proxied")" | cfa_ok || { + err "Could not update the $type record for $host" + return 1 + } + fi + done +} + +cf_caa_warning() { + local zone_id="$1" zone_name="$2" + cfa GET "/zones/$zone_id/dns_records?type=CAA&name=$zone_name" | + jq -e '(.result | length > 0) and (all(.result[]; (.content | ascii_downcase | contains("letsencrypt.org")) | not))' >/dev/null 2>&1 && + warn "$zone_name has CAA records without letsencrypt.org, so Let's Encrypt can't issue the panel's certificates." +} + +cf_in_list() { + [[ ",$2," == *",$1,"* ]] +} + +cf_records() { + local replace_list="$1" key host zone_id zone_name proxied why want orange=keep note plan conflicts="" out + cf_public_ip || { + err "Could not find this server's public IP. Set CLOUDFLARE_ORIGIN_IPV4." + return 1 + } + if cf_private_ip "$CLOUDFLARE_ORIGIN_IPV4"; then + err "$CLOUDFLARE_ORIGIN_IPV4 is a private address, so Cloudflare can't reach this server there." + return 1 + fi + plan="$(cf_plan)" + if awk '$5 == "yes" { found = 1 } END { exit !found }' <<<"$plan"; then + if cf_ready && cf_rules; then + orange=true + else + warn "The panel's Cloudflare rules aren't in place, so its hostnames don't go through Cloudflare yet." + fi + fi + while read -r key host zone_id zone_name proxied why; do + if [ -z "$host" ] || [ "$proxied" = skip ]; then + continue + fi + want="$orange" + if [ "$proxied" = no ]; then + want=false + fi + out="$(cf_sync_host "$zone_id" "$host" "$want" "$(cf_in_list "$host" "$replace_list" && echo true || echo false)")" || return 1 + if [ -n "$out" ]; then + conflicts+="$out"$'\n' + else + case "$want" in + true) note=" (through Cloudflare)" ;; + false) note=" (straight to this server)" ;; + *) note="" ;; + esac + ok "$host → $CLOUDFLARE_ORIGIN_IPV4$note" + fi + done <<<"$plan" + while read -r zone_id zone_name; do + cf_caa_warning "$zone_id" "$zone_name" + done < <(awk 'NF && $3 != "-" { print $3 " " $4 }' <<<"$plan" | sort -u) + if [ -n "$conflicts" ]; then + printf '%s' "$conflicts" + warn "These records are not 5Stack's, so they were left alone. Run again with --replace and their names to replace them." + return 2 + fi +} + +cf_host_expression() { + jq -rn --args '"(http.host in {\($ARGS.positional | map("\"" + . + "\"") | join(" "))})"' "$@" +} + +cf_rule() { + local zone_id="$1" phase="$2" ref="$3" description="$4" action="$5" params="$6" expression="$7" response rule ruleset_id rule_id + rule="$(jq -nc --arg ref "$ref" --arg d "$description" --arg a "$action" --argjson p "$params" --arg e "$expression" \ + '{ref: $ref, description: $d, expression: $e, action: $a, action_parameters: $p, enabled: true}')" + response="$(cfa GET "/zones/$zone_id/rulesets/phases/$phase/entrypoint")" + if ! cfa_ok <<<"$response"; then + cfa PUT "/zones/$zone_id/rulesets/phases/$phase/entrypoint" "$(jq -nc --argjson r "$rule" '{rules: [$r]}')" | cfa_ok + return + fi + ruleset_id="$(jq -r '.result.id' <<<"$response")" + rule_id="$(jq -r --arg ref "$ref" '[.result.rules[]? | select(.ref == $ref)][0].id // empty' <<<"$response")" + if [ -n "$rule_id" ]; then + cfa PATCH "/zones/$zone_id/rulesets/$ruleset_id/rules/$rule_id" "$rule" | cfa_ok + else + cfa POST "/zones/$zone_id/rulesets/$ruleset_id/rules" "$rule" | cfa_ok + fi +} + +cf_acme_expression() { + printf '%s and starts_with(http.request.uri.path, "%s")' "$(cf_host_expression "$@")" "$CF_ACME_PATH" +} + +cf_https_expression() { + printf '(not ssl and not starts_with(http.request.uri.path, "%s"))' "$CF_ACME_PATH" +} + +cf_https() { + local zone_id="$1" wide=false expression params + shift + if cfa GET "/zones/$zone_id/settings/always_use_https" | jq -e '.result.value == "on"' >/dev/null 2>&1; then + wide=true + fi + if cfa GET "/zones/$zone_id/rulesets/phases/http_request_dynamic_redirect/entrypoint" | + jq -e --arg ref "$CF_REDIRECT_REF" 'any(.result.rules[]?; .ref == $ref and (.expression | startswith("(http.host in") | not))' >/dev/null 2>&1; then + wide=true + fi + expression="$(cf_https_expression)" + if [ "$wide" != true ]; then + expression="$(cf_host_expression "$@") and $expression" + fi + params='{"from_value": {"status_code": 301, "target_url": {"expression": "concat(\"https://\", http.host, http.request.uri.path)"}, "preserve_query_string": true}}' + cf_rule "$zone_id" http_request_dynamic_redirect "$CF_REDIRECT_REF" "5Stack panel: HTTPS, except Let's Encrypt's check" redirect "$params" "$expression" || { + err "Could not save the 5Stack rule that sends plain HTTP to HTTPS" + return 1 + } + if [ "$wide" = true ]; then + cfa PATCH "/zones/$zone_id/settings/always_use_https" '{"value": "off"}' | cfa_ok || { + err "Could not turn off Always Use HTTPS, which sends Let's Encrypt's check to HTTPS" + return 1 + } + fi +} + +cf_challenge_blockers() { + local zone_id="$1" response target pattern host rule + shift + response="$(cfa GET "/zones/$zone_id/rulesets/phases/http_request_dynamic_redirect/entrypoint")" + if cfa_ok <<<"$response"; then + while read -r rule; do + echo "redirect_rule: The Redirect Rule \"$rule\" can send Let's Encrypt's check to HTTPS. Leave $CF_ACME_PATH out of it." + done < <(jq -r --arg ref "$CF_REDIRECT_REF" --argjson hosts "$(jq -nc '$ARGS.positional' --args "$@")" ' + .result.rules[]? + | select(.ref != $ref and .enabled != false and .action == "redirect") + | select(.expression | contains("acme-challenge") | not) + | . as $r + | select(($r.expression | contains("http.host") | not) or any($hosts[]; . as $h | $r.expression | contains("\"" + $h + "\""))) + | .description // .ref // .id' <<<"$response") + fi + response="$(cfa GET "/zones/$zone_id/pagerules?status=active")" + cfa_ok <<<"$response" || return 0 + while IFS=$'\t' read -r rule target; do + [ -n "$target" ] || continue + if [[ "$target" == https://* ]]; then + continue + fi + pattern="${target#http://}" + pattern="${pattern%%/*}" + pattern="${pattern//./\\.}" + pattern="^${pattern//\*/.*}$" + for host in "$@"; do + if [[ "$host" =~ $pattern ]]; then + echo "page_rule: The Page Rule for $target ($rule) sends Let's Encrypt's check to HTTPS, and a Page Rule can't leave one path out. Replace it with a Redirect Rule that leaves $CF_ACME_PATH out." + break + fi + done + done < <(jq -r '.result[]? | select(any(.actions[]?; .id == "always_use_https" or .id == "forwarding_url")) | [([.actions[].id] | join(",")), (.targets[0].constraint.value // "")] | @tsv' <<<"$response") +} + +cf_edge_ip() { + curl -sS --max-time 10 -H 'accept: application/dns-json' "$CF_DOH_URL?name=$1&type=A" 2>/dev/null | + jq -r --arg o "$CLOUDFLARE_ORIGIN_IPV4" '[.Answer[]? | select(.type == 1 and .data != $o) | .data][0] // empty' 2>/dev/null +} + +cf_acme_probe() { + local host="$1" edge headers code args + args=(-sS -o /dev/null -D - --max-time 20) + edge="$(cf_edge_ip "$host")" + if [ -n "$edge" ]; then + args+=(--resolve "$host:80:$edge") + fi + headers="$(curl "${args[@]}" "http://$host$CF_ACME_CHECK" 2>/dev/null | tr -d '\r' | tr '[:upper:]' '[:lower:]')" + code="$(awk 'NR == 1 { print $2 }' <<<"$headers")" + if grep -q '^x-5stack-check:' <<<"$headers"; then + if grep -q '^cf-ray:' <<<"$headers"; then + echo ok + else + echo direct + fi + return + fi + case "$code" in + "") echo none ;; + 52[0-6] | 530) echo "unreachable $code" ;; + 3[0-9][0-9]) echo "redirect $code $(sed -n 's/^location: *//p' <<<"$headers" | head -n 1)" ;; + *) + if grep -q '^cf-mitigated:' <<<"$headers" || { [[ "$code" =~ ^(403|429|503)$ ]] && grep -q '^server: cloudflare' <<<"$headers"; }; then + echo "challenge $code" + else + echo "other $code" + fi + ;; + esac +} + +cf_acme_blocked() { + local host="$1" i result + for ((i = 0; i < CF_PROBE_WAIT; i++)); do + result="$(cf_acme_probe "$host")" + case "$result" in + ok) return 1 ;; + redirect*) + echo "Cloudflare redirects http://$host$CF_ACME_CHECK to ${result#redirect * }" + return 0 + ;; + challenge*) + echo "Cloudflare answers http://$host$CF_ACME_CHECK with a security challenge" + return 0 + ;; + unreachable*) + echo "Cloudflare can't reach this server for http://$host$CF_ACME_CHECK (HTTP ${result#unreachable })" + return 0 + ;; + other*) + echo "http://$host$CF_ACME_CHECK answers HTTP ${result#other } instead of the panel" + return 0 + ;; + esac + sleep "$CF_PROBE_PAUSE" + done + warn "http://$host$CF_ACME_CHECK didn't answer through Cloudflare yet, so Let's Encrypt's path wasn't checked. The next update checks it again." >&2 + return 1 +} + +cf_origin_status() { + local host="$1" + curl -s -o /dev/null --max-time 20 -w '%{http_code}' "https://$host/" +} + +cf_ready() { + local problems line + problems="$(cf_check)" + if [ -n "$problems" ]; then + err "Cloudflare isn't ready for the panel:" + while read -r line; do + err "${line#*: }" + done <<<"$problems" + return 1 + fi +} + +cf_ruled() { + awk '$5 == "yes" || ($1 == "GAME_STREAM_DOMAIN" && $6 == "worker")' +} + +cf_rules() { + local key host zone_id zone_name proxied why zone hosts nocache + for zone in $(cf_plan | cf_ruled | awk '{ print $3 }' | sort -u); do + hosts=() + nocache=() + while read -r key host zone_id zone_name proxied why; do + hosts+=("$host") + if [[ " $CF_NO_CACHE_KEYS " == *" $key "* ]]; then + nocache+=("$host") + fi + done < <(cf_plan | cf_ruled | awk -v z="$zone" '$3 == z') + cf_rule "$zone" http_config_settings "$CF_CONFIG_REF" "5Stack panel" set_config \ + '{"ssl": "strict", "rocket_loader": false, "email_obfuscation": false, "disable_zaraz": true, "bic": false}' "$(cf_host_expression "${hosts[@]}")" || { + err "Could not save the 5Stack configuration rule" + return 1 + } + if ! cf_reverse_proxy; then + cf_rule "$zone" http_config_settings "$CF_ACME_REF" "5Stack panel: Let's Encrypt's check" set_config \ + '{"security_level": "essentially_off", "bic": false}' "$(cf_acme_expression "${hosts[@]}")" || { + err "Could not save the 5Stack rule that keeps challenges off Let's Encrypt's check" + return 1 + } + fi + if [ "${#nocache[@]}" -gt 0 ]; then + cf_rule "$zone" http_request_cache_settings "$CF_CACHE_REF" "5Stack panel: live data" set_cache_settings \ + '{"cache": false}' "$(cf_host_expression "${nocache[@]}")" || { + err "Could not save the 5Stack cache rule" + return 1 + } + fi + if ! cf_reverse_proxy; then + cf_https "$zone" "${hosts[@]}" || return 1 + fi + done +} + +cf_zone_blockers() { + local hosts + if cf_reverse_proxy; then + return 0 + fi + mapfile -t hosts < <(cf_plan | cf_ruled | awk -v z="$1" '$3 == z { print $2 }') + cf_challenge_blockers "$1" "${hosts[@]}" +} + +cf_blockers() { + local zone + for zone in $(cf_plan | cf_ruled | awk '{ print $3 }' | sort -u); do + cf_zone_blockers "$zone" + done +} + +cf_proxy() { + local key host zone_id zone_name proxied why status code blocked line + cf_ready || return 1 + cf_public_ip || return 1 + cf_rules || return 1 + while read -r line; do + [ -n "$line" ] && warn "${line#*: }" + done < <(cf_blockers) + status=0 + while read -r key host zone_id zone_name proxied why; do + [ "$proxied" = yes ] || continue + cf_sync_host "$zone_id" "$host" true false >/dev/null || return 1 + code="$(cf_origin_status "$host")" + case "$code" in + 52[0-6] | 530) + warn "Cloudflare can't reach the panel on $host (HTTP $code), so it goes straight to this server again." + cf_sync_host "$zone_id" "$host" false false >/dev/null + status=3 + continue + ;; + esac + if ! cf_reverse_proxy && blocked="$(cf_acme_blocked "$host")"; then + warn "Let's Encrypt can't reach $host through Cloudflare: $blocked. It goes straight to this server again, so its certificate keeps renewing." + cf_sync_host "$zone_id" "$host" false false >/dev/null + status=3 + continue + fi + ok "$host through Cloudflare" + done < <(cf_plan) + return "$status" +} + +cf_origins() { + local key host zone_id zone_name proxied why code status=0 + cf_public_ip || return 1 + while read -r key host zone_id zone_name proxied why; do + [ "$proxied" = yes ] || continue + code="$(cf_origin_status "$host")" + case "$code" in + 52[0-6] | 530) + warn "Cloudflare can't reach the panel on $host (HTTP $code), so it goes straight to this server again." + cf_sync_host "$zone_id" "$host" false false >/dev/null + status=3 + ;; + esac + done < <(cf_plan) + return "$status" +} + +cf_status_json() { + local key host zone_id zone_name proxied why records zones + cf_public_ip + zones="$(cf_plan | awk '$3 != "-" { print $3 " " $4 }' | sort -u)" + { + while read -r key host zone_id zone_name proxied why; do + if [ "$zone_id" = - ]; then + jq -nc --arg k "$key" --arg h "$host" '{key: $k, host: $h, state: "not_in_account"}' + continue + fi + records="$(cf_records_json "$zone_id" "$host")" + jq -nc --arg k "$key" --arg h "$host" --arg p "$proxied" --arg w "$why" --arg ip "$CLOUDFLARE_ORIGIN_IPV4" --arg m "$CF_COMMENT" --argjson r "$records" ' + ([$r[] | select(.type == "A")][0]) as $a + | {key: $k, host: $h, want_proxied: ($p == "yes"), why: $w, ip: $ip, + content: ($a.content // null), proxied: ($a.proxied // null), modified: ($a.modified_on // null), + managed: (($a.comment // "") == $m), + state: (if $p == "skip" then "skipped" + elif $a == null then "missing" + elif $a.content != $ip then "elsewhere" + elif ($p == "yes") != ($a.proxied // false) then "cloud" + else "ok" end)}' + done < <(cf_plan) + } | jq -sc --argjson zones "$(while read -r zone_id zone_name; do + [ -n "$zone_id" ] || continue + jq -nc --arg id "$zone_id" --arg name "$zone_name" \ + --argjson ws "$(cfa GET "/zones/$zone_id/settings/websockets" | jq -c '.result.value // null')" \ + --argjson bot "$(cfa GET "/zones/$zone_id/bot_management" | jq -c '.result | if type == "object" and has("fight_mode") then .fight_mode else null end')" \ + --argjson tiered "$(cfa GET "/zones/$zone_id/cache/tiered_cache_smart_topology_enable" | jq -c '.result.value // null')" \ + --argjson rule "$(cfa GET "/zones/$zone_id/rulesets/phases/http_config_settings/entrypoint" | jq -c --arg ref "$CF_CONFIG_REF" '[.result.rules[]? | select(.ref == $ref)][0].action_parameters.ssl // null')" \ + --argjson always "$(cfa GET "/zones/$zone_id/settings/always_use_https" | jq -c '.result.value // null')" \ + --argjson redirect "$(cfa GET "/zones/$zone_id/rulesets/phases/http_request_dynamic_redirect/entrypoint" | jq -c --arg ref "$CF_REDIRECT_REF" --arg p "$CF_ACME_PATH" '[.result.rules[]? | select(.ref == $ref and .enabled != false and (.expression | contains("not starts_with(http.request.uri.path, \"" + $p + "\")")))] | length > 0')" \ + --argjson blockers "$(cf_zone_blockers "$zone_id" | sed 's/^[a-z_]*: //' | jq -Rsc 'split("\n") | map(select(length > 0))')" \ + '{id: $id, name: $name, websockets: $ws, bot_fight: $bot, tiered_cache: $tiered, ssl: $rule, always_https: $always, https_rule: $redirect, blockers: $blockers}' + done <<<"$zones" | jq -sc .)" '{zones: $zones, hosts: .}' +} + +cf_fix() { + local target key host zone_id zone_name proxied why + target="$(cf_lower "$1")" + cf_public_ip || return 1 + while read -r key host zone_id zone_name proxied why; do + if [ "$host" = "$target" ] && [ "$proxied" != skip ]; then + cf_sync_host "$zone_id" "$host" "$([ "$proxied" = yes ] && echo true || echo false)" true + return + fi + done < <(cf_plan) + err "$target isn't one of the panel's hostnames on Cloudflare." + return 1 +} + +cf_tiered_cache() { + local zone + for zone in $(cf_plan | awk '$5 == "yes" { print $3 }' | sort -u); do + cfa PATCH "/zones/$zone/argo/tiered_caching" '{"value": "on"}' | cfa_ok || return 1 + cfa PATCH "/zones/$zone/cache/tiered_cache_smart_topology_enable" '{"value": "on"}' | cfa_ok || return 1 + done +} + +cf_upload_worker() { + local account="$1" dir response + dir="$(mktemp -d)" + jq -nc --arg web "https://$(cf_lower "$WEB_DOMAIN")" \ + '{main_module: "worker.js", compatibility_date: "2025-04-29", bindings: [{type: "plain_text", name: "WEB_ORIGIN", text: $web}]}' >"$dir/metadata.json" + response="$(printf 'header = "Authorization: Bearer %s"\n' "$CLOUDFLARE_API_TOKEN" | + curl -K - -sS --max-time 60 -X PUT \ + -F "metadata=<$dir/metadata.json;type=application/json" \ + -F "worker.js=@$CF_SETUP_DIR/../cloudflare-workers/stream-relay/worker.js;type=application/javascript+module" \ + "$CLOUDFLARE_API/accounts/$account/workers/scripts/$CF_RELAY_WORKER")" + rm -rf "$dir" + if ! cfa_ok <<<"$response"; then + err "Cloudflare refused the stream relay Worker: $(cfa_errors <<<"$response")" + return 1 + fi +} + +cf_relay_route() { + local zone_id="$1" pattern="$2" routes id owner body response + routes="$(cfa GET "/zones/$zone_id/workers/routes")" + cfa_ok <<<"$routes" || { + err "Could not read the zone's Worker routes: $(cfa_errors <<<"$routes")" + return 1 + } + id="$(jq -r --arg p "$pattern" '[.result[]? | select(.pattern == $p)][0].id // empty' <<<"$routes")" + owner="$(jq -r --arg p "$pattern" '[.result[]? | select(.pattern == $p)][0].script // empty' <<<"$routes")" + if [ -n "$owner" ] && [ "$owner" != "$CF_RELAY_WORKER" ]; then + err "The route $pattern already sends requests to the Worker \"$owner\". Remove it in Cloudflare first." + return 1 + fi + body="$(jq -nc --arg p "$pattern" --arg s "$CF_RELAY_WORKER" '{pattern: $p, script: $s, request_limit_fail_open: false}')" + if [ -z "$id" ]; then + response="$(cfa POST "/zones/$zone_id/workers/routes" "$body")" + else + response="$(cfa PUT "/zones/$zone_id/workers/routes/$id" "$body")" + fi + cfa_ok <<<"$response" || { + err "Could not route $pattern to the stream relay Worker: $(cfa_errors <<<"$response")" + return 1 + } +} + +cf_relay_answers() { + local host="$1" i + for ((i = 0; i < CF_RELAY_WAIT; i++)); do + if curl -fsS --max-time 10 "https://$host/health" 2>/dev/null | grep -q "\"worker\":\"$CF_RELAY_WORKER\""; then + return 0 + fi + sleep "$CF_RELAY_PAUSE" + done + return 1 +} + +cf_stream_relay() { + local host zone zone_id zone_name account out blocked + host="$(cf_lower "$GAME_STREAM_DOMAIN")" + if [ -z "$host" ]; then + err "The panel has no stream address (GAME_STREAM_DOMAIN), so there is nothing to relay." + return 1 + fi + if [ -n "$HTTPS_PORT" ] && [ "$HTTPS_PORT" != 443 ]; then + err "The panel serves HTTPS on port $HTTPS_PORT, and a Cloudflare Worker only reaches port 443." + return 1 + fi + zone="$(cf_zone "$host")" || { + err "$host isn't in a Cloudflare zone this token can read." + return 1 + } + read -r zone_id zone_name _ <<<"$zone" + if ! cf_covered "$host" "$zone_name" "$zone_id"; then + err "$host is deeper than Cloudflare's free certificate covers, so a Worker can't serve it." + return 1 + fi + cf_ready || return 1 + cf_public_ip || return 1 + account="$(cfa GET "/zones/$zone_id" | jq -r '.result.account.id // empty')" + if [ -z "$account" ]; then + err "Could not read which Cloudflare account $zone_name is in." + return 1 + fi + cf_upload_worker "$account" || return 1 + ok "Worker $CF_RELAY_WORKER deployed" + cf_relay_route "$zone_id" "$host/*" || return 1 + ok "Route $host/* → $CF_RELAY_WORKER" + cf_rules || return 1 + out="$(cf_sync_host "$zone_id" "$host" true false)" || return 1 + if [ -n "$out" ]; then + printf '%s\n' "$out" + err "$host has a record that isn't 5Stack's, so its proxy wasn't turned on. Run ./cloudflare.sh to replace it." + return 1 + fi + if ! cf_relay_answers "$host"; then + warn "The stream relay Worker isn't answering on $host, so streams go straight to this server again." + cf_sync_host "$zone_id" "$host" false false >/dev/null + return 3 + fi + if ! cf_reverse_proxy && blocked="$(cf_acme_blocked "$host")"; then + warn "Let's Encrypt can't reach $host through Cloudflare: $blocked. Streams go straight to this server again." + cf_sync_host "$zone_id" "$host" false false >/dev/null + return 3 + fi + ok "Streams on $host go through the stream relay Worker" +} + +cf_plan_json() { + cf_plan | jq -Rsc ' + split("\n") | map(select(length > 0) | split(" ") + | {key: .[0], host: .[1], zone_id: .[2], zone: .[3], proxied: .[4], why: .[5]})' +} + +cloudflare_setup() { + local command="$1" replace="" response + shift + if ! command -v jq >/dev/null 2>&1; then + err "Setting up Cloudflare needs jq. Install it with: apt-get install -y jq" + return 1 + fi + if [ "$1" = --replace ]; then + replace="$(cf_lower "$2")" + fi + cf_load_hosts + cf_load_token || { + err "No Cloudflare API token. Run ./cloudflare.sh to add one." + return 1 + } + response="$(cfa GET "/zones?per_page=1")" + if ! cfa_ok <<<"$response"; then + err "Cloudflare refused the API token: $(cfa_errors <<<"$response")" + return 1 + fi + case "$command" in + plan) + if [ "$1" = --json ]; then + cf_plan_json + else + cf_plan + fi + ;; + check) cf_check ;; + records) cf_records "$replace" ;; + proxy) cf_proxy ;; + origins) cf_origins ;; + status) cf_status_json ;; + fix) cf_fix "$1" ;; + tiered-cache) cf_tiered_cache ;; + stream-relay) cf_stream_relay ;; + *) + err "Unknown step: $command" + return 1 + ;; + esac +} + +cloudflare_sync_records() { + local out status hosts answer + out="$(cloudflare_setup records)" + status=$? + grep -v '^conflict: ' <<<"$out" + if [ "$status" = 2 ] && [ -t 0 ] && [ "$ASSUME_YES" != true ]; then + grep '^conflict: ' <<<"$out" | sed 's/^conflict: / /' + hosts="$(grep '^conflict: ' <<<"$out" | awk '{ print $2 }' | sort -u | paste -sd, -)" + read -r -p " These records aren't 5Stack's. Replace them so the panel works? (y/n): " answer + if [ "$answer" = y ]; then + cloudflare_setup records --replace "$hosts" + status=$? + fi + fi + if [ "$status" != 0 ]; then + warn "Not every Cloudflare record was updated; the panel carries on with them as they are." + fi +} + +cloudflare_sync_proxy() { + if ! cloudflare_setup proxy; then + warn "Cloudflare isn't in front of every panel hostname yet. Run ./cloudflare.sh to see why." + fi +} + +cloudflare_sync_stream_relay() { + if ! cloudflare_setup stream-relay; then + warn "The stream relay Worker isn't set up, so streams go straight to this server. Run ./stream-relay.sh to see why." + fi +} + +cloudflare_update_records() { + if [ "$CLOUDFLARE_RECORDS_SET" = true ]; then + return 0 + fi + step "Updating the panel's Cloudflare records" + cloudflare_sync_records +} + +cloudflare_update_proxy() { + if [ "$CLOUDFLARE_RECORDS_SET" = true ]; then + cloudflare_setup origins + else + step "Putting Cloudflare in front of the panel" + cloudflare_sync_proxy + fi + if [ "$CLOUDFLARE_STREAM_RELAY" = true ]; then + step "Putting the stream relay Worker in front of streams" + cloudflare_sync_stream_relay + fi +} + +if [ "${BASH_SOURCE[0]}" = "$0" ]; then + cloudflare_setup "$@" +fi diff --git a/utils/colors.sh b/utils/colors.sh index 3239ccf..9ffa3e5 100644 --- a/utils/colors.sh +++ b/utils/colors.sh @@ -47,6 +47,19 @@ read_masked() { printf -v "$__outvar" '%s' "$__value" } +# Runs a command for at most SECONDS. --foreground keeps Ctrl+C reaching it; a +# host whose timeout lacks that (macOS has none, busybox's has no flag) runs it +# unbounded. +time_limited() { + local seconds="$1" + shift + if timeout --foreground 1 true >/dev/null 2>&1; then + timeout --foreground "$seconds" "$@" + else + "$@" + fi +} + # Bail out with a message. Sourced scripts (update.sh runs inside install.sh) # rely on this actually stopping the install rather than letting a half-applied # cluster carry on to the next step. diff --git a/utils/setup-env.sh b/utils/setup-env.sh index f8786a4..57f3f3b 100755 --- a/utils/setup-env.sh +++ b/utils/setup-env.sh @@ -11,6 +11,7 @@ DEBUG=false FIVE_STACK_ENV_SETUP=true REVERSE_PROXY="" CLOUDFLARE="" +CLOUDFLARE_MANAGED="" CLOUDFLARE_DOCS="https://docs.5stack.gg/install/cloudflare/" @@ -71,6 +72,12 @@ while [[ $# -gt 0 ]]; do fi shift ;; + --cloudflare-managed) + CLOUDFLARE=true + CLOUDFLARE_MANAGED=true + REVERSE_PROXY=false + shift + ;; --cloudflare=*) CLOUDFLARE="${1#*=}" if [ "$CLOUDFLARE" = "0" ] || [ "$CLOUDFLARE" = "n" ]; then @@ -91,9 +98,15 @@ if [ "$DEBUG" = true ]; then fi ask_yes_no() { - local prompt="$1" answer + local prompt="$1" default="${2:-}" answer hint="y/n" + if [ "$default" = y ]; then + hint="Y/n" + fi while true; do - read -p "$prompt (y/n): " answer + read -p "$prompt ($hint): " answer + if [ -z "$answer" ]; then + answer="$default" + fi if [ "$answer" = "y" ] || [ "$answer" = "n" ]; then break fi @@ -110,13 +123,104 @@ ask_cloudflare() { return fi + if ask_yes_no "Set up DNS, proxy and SSL automatically?"; then + CLOUDFLARE_MANAGED=true + REVERSE_PROXY=false + return + fi + step "Cloudflare Setup" - warn "Cloudflare needs a few 5Stack specific settings — SSL/TLS mode, an ACME" - warn "challenge rule, and tunnel routes. Follow the guide before continuing:" + warn "Cloudflare needs a few 5Stack specific settings. Follow the guide before continuing:" open_docs "$CLOUDFLARE_DOCS" "Cloudflare" echo } +cloudflare_token_url() { + local keys='[{"key":"zone","type":"read"},{"key":"dns","type":"edit"},{"key":"zone_settings","type":"edit"},{"key":"ssl_and_certificates","type":"read"},{"key":"config_settings","type":"edit"},{"key":"cache_settings","type":"edit"},{"key":"workers_routes","type":"edit"},{"key":"workers_scripts","type":"edit"},{"key":"bot_management","type":"read"},{"key":"dynamic_url_redirects","type":"edit"},{"key":"page_rules","type":"read"}]' + printf 'https://dash.cloudflare.com/profile/api-tokens?permissionGroupKeys=%s&accountId=*&zoneId=all&name=5Stack%%20panel\n' "$(jq -rn --arg k "$keys" '$k | @uri')" +} + +ask_cloudflare_token() { + local file="overlays/local-secrets/cloudflare-secrets.env" token + if [ -n "$CLOUDFLARE_API_TOKEN" ] || grep -qs '^CLOUDFLARE_API_TOKEN=..*' "$file"; then + return + fi + if [ "$ASSUME_YES" = true ] || [ ! -t 0 ]; then + warn "Cloudflare is managed for this panel but it has no API token yet. Run ./cloudflare.sh to add one." + return + fi + step "Cloudflare API token" + echo " Make a token on Cloudflare's page; the permissions 5Stack needs are filled in:" + echo " $(cloudflare_token_url)" + read -rs -p " Paste the token: " token + echo + if [ -n "$token" ]; then + update_env_var "$file" CLOUDFLARE_API_TOKEN "$token" + fi +} + +flat_host() { + printf '%s-%s.%s' "${WEB_DOMAIN%%.*}" "$1" "${WEB_DOMAIN#*.}" +} + +panel_host() { + if [ "$FLAT_HOSTS" = true ]; then + flat_host "$1" + else + printf '%s.%s' "$1" "$WEB_DOMAIN" + fi +} + +ask_cloudflare_plan() { + local answer + while true; do + read -p "Which Cloudflare plan is ${WEB_DOMAIN#*.} on? Free or Pro or Higher (f/p): " answer + case "$answer" in + f) CLOUDFLARE_PLAN=free; return ;; + p) CLOUDFLARE_PLAN=paid; return ;; + esac + echo "Please enter 'f' or 'p'" + done +} + +flat_hosts_alert() { + local parent="${WEB_DOMAIN#*.}" sub names="" + for sub in api ws tv demos hls console search; do + names="${names:+$names, }$(flat_host "$sub")" + done + warn "Cloudflare's free plan only covers one level below $parent, so the addresses become $names." + warn "Other sites on $parent will receive the panel's sign-in cookie." +} + +ask_flat_hosts() { + local parent="${WEB_DOMAIN#*.}" interactive=true + if [[ "$parent" != *.* ]]; then + FLAT_HOSTS=false + return + fi + if [ -n "$FLAT_HOSTS" ] || [ -n "$API_DOMAIN$WS_DOMAIN$RELAY_DOMAIN$DEMOS_DOMAIN$GAME_STREAM_DOMAIN$S3_CONSOLE_HOST$TYPESENSE_HOST" ]; then + return + fi + if [ "$ASSUME_YES" = true ] || [ ! -t 0 ]; then + interactive=false + fi + FLAT_HOSTS=true + if [ "$CLOUDFLARE" = true ] && [ -z "$CLOUDFLARE_PLAN" ] && [ "$interactive" = true ]; then + ask_cloudflare_plan + fi + local question + question="Keep every address one level deep ($(flat_host api) instead of api.$WEB_DOMAIN)?" + if [ "$CLOUDFLARE" = true ] && [ "$CLOUDFLARE_PLAN" = free ]; then + flat_hosts_alert + question="Keep every address one level deep? Recommended on Cloudflare's free plan; n keeps them under the panel, straight to this server" + else + echo "Sign-in then covers all of $parent, so other sites on $parent get the panel's sign-in cookie." + fi + if [ "$interactive" = true ] && ! ask_yes_no "$question" y; then + FLAT_HOSTS=false + fi +} + ask_reverse_proxy() { if [ "$CLOUDFLARE" = true ]; then if ask_yes_no "Are you using the Cloudflare proxy (orange cloud) or a Cloudflare Tunnel?"; then @@ -195,7 +299,20 @@ if [ -z "$REVERSE_PROXY" ]; then if [ -z "$CLOUDFLARE" ]; then ask_cloudflare fi - ask_reverse_proxy + if [ "$CLOUDFLARE_MANAGED" != true ]; then + ask_reverse_proxy + fi +fi + +if [ "$CLOUDFLARE_MANAGED" = true ]; then + if ! command -v jq >/dev/null 2>&1; then + echo "Error: managing Cloudflare needs jq. Install it with: apt-get install -y jq" + exit 1 + fi + if [ "$REVERSE_PROXY" != true ]; then + REVERSE_PROXY=false + fi + CLOUDFLARE=true fi if [ ! -f .5stack-env.config ]; then @@ -209,6 +326,12 @@ KUBECONFIG=$KUBECONFIG EOF fi +if [ "$CLOUDFLARE_MANAGED" = true ] && ! grep -qs '^CLOUDFLARE_MANAGED=true' .5stack-env.config; then + update_env_var .5stack-env.config CLOUDFLARE_MANAGED true + update_env_var .5stack-env.config CLOUDFLARE true + update_env_var .5stack-env.config REVERSE_PROXY "$REVERSE_PROXY" +fi + if [ -d "base/secrets" ]; then echo "base/secrets directory found, moving to overlays/local-secrets" mv base/secrets/* overlays/local-secrets @@ -236,6 +359,10 @@ copy_config_or_secrets "overlays/config" "overlays/config" copy_config_or_secrets "overlays/mediamtx" "overlays/mediamtx" copy_config_or_secrets "overlays/coturn" "overlays/coturn" +if [ "$CLOUDFLARE_MANAGED" = true ] && [ "$VAULT_MANAGER" != true ]; then + ask_cloudflare_token +fi + # Replace $(RAND32) with a random base64 encoded string in all non-example env files replace_rand32_in_env_files "overlays/local-secrets" @@ -295,29 +422,30 @@ if [ -z "$WEB_DOMAIN" ] || [ -z "$WS_DOMAIN" ] || [ -z "$API_DOMAIN" ] || [ -z " echo "WEB_DOMAIN: $WEB_DOMAIN" update_env_var "overlays/config/api-config.env" "WEB_DOMAIN" "$WEB_DOMAIN" + ask_flat_hosts if [ -z "$WS_DOMAIN" ]; then - WS_DOMAIN="ws.$WEB_DOMAIN" + WS_DOMAIN="$(panel_host ws)" update_env_var "overlays/config/api-config.env" "WS_DOMAIN" "$WS_DOMAIN" fi if [ -z "$API_DOMAIN" ]; then - API_DOMAIN="api.$WEB_DOMAIN" + API_DOMAIN="$(panel_host api)" update_env_var "overlays/config/api-config.env" "API_DOMAIN" "$API_DOMAIN" fi if [ -z "$RELAY_DOMAIN" ]; then - RELAY_DOMAIN="tv.$WEB_DOMAIN" + RELAY_DOMAIN="$(panel_host tv)" update_env_var "overlays/config/api-config.env" "RELAY_DOMAIN" "$RELAY_DOMAIN" fi if [ -z "$DEMOS_DOMAIN" ]; then - DEMOS_DOMAIN="demos.$WEB_DOMAIN" + DEMOS_DOMAIN="$(panel_host demos)" update_env_var "overlays/config/api-config.env" "DEMOS_DOMAIN" "$DEMOS_DOMAIN" fi if [ -z "$GAME_STREAM_DOMAIN" ]; then - GAME_STREAM_DOMAIN="hls.$WEB_DOMAIN" + GAME_STREAM_DOMAIN="$(panel_host hls)" update_env_var "overlays/config/api-config.env" "GAME_STREAM_DOMAIN" "$GAME_STREAM_DOMAIN" fi @@ -332,12 +460,12 @@ if [ -z "$WEB_DOMAIN" ] || [ -z "$WS_DOMAIN" ] || [ -z "$API_DOMAIN" ] || [ -z " fi if [ -z "$S3_CONSOLE_HOST" ]; then - S3_CONSOLE_HOST="console.$WEB_DOMAIN" + S3_CONSOLE_HOST="$(panel_host console)" update_env_var "overlays/config/s3-config.env" "S3_CONSOLE_HOST" "$S3_CONSOLE_HOST" fi if [ -z "$TYPESENSE_HOST" ]; then - TYPESENSE_HOST="search.$WEB_DOMAIN" + TYPESENSE_HOST="$(panel_host search)" update_env_var "overlays/config/typesense-config.env" "TYPESENSE_HOST" "$TYPESENSE_HOST" fi fi diff --git a/utils/signed_clip_links.sh b/utils/signed_clip_links.sh index 8d72661..ca82fe4 100644 --- a/utils/signed_clip_links.sh +++ b/utils/signed_clip_links.sh @@ -143,6 +143,6 @@ require_signed_clip_links() { } signed_clip_links_psql() { - kubectl --kubeconfig="$PANEL_KUBECONFIG" -n 5stack exec timescaledb-0 -c timescaledb -- \ + time_limited 120 kubectl --kubeconfig="$PANEL_KUBECONFIG" -n 5stack --request-timeout=120s exec timescaledb-0 -c timescaledb -- \ sh -c 'psql -X -v ON_ERROR_STOP=1 -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"' sh "$1" 2>/dev/null } diff --git a/utils/tailscale-api.sh b/utils/tailscale-api.sh index 5215f0c..16c6b27 100644 --- a/utils/tailscale-api.sh +++ b/utils/tailscale-api.sh @@ -4,6 +4,39 @@ # Documentation: https://tailscale.com/api TAILSCALE_API_BASE="https://api.tailscale.com/api/v2" +TAILSCALE_SHARED_TAG="tag:fivestack" +TAILSCALE_SHARED_PODS="10.42.0.0/16" + +# The shared 10.42 lines would route a panel with its own pod range into +# another panel's pods, so such a cluster is refused rather than guessed at. +tailnet_pod_range() { + local cidr + if [ -n "${PANEL_POD_RANGE:-}" ]; then + echo "$PANEL_POD_RANGE" + return 0 + fi + for cidr in $(kubectl get nodes -o jsonpath='{.items[*].spec.podCIDR}' 2>/dev/null); do + if [[ "$cidr" != 10.42.* ]]; then + echo "Error: this panel's pods are on $cidr, outside $TAILSCALE_SHARED_PODS. Set PANEL_POD_RANGE and PANEL_TAILSCALE_TAG in .5stack-env.config to the panel's own range and tag" >&2 + return 1 + fi + done + echo "$TAILSCALE_SHARED_PODS" +} + +tailnet_tag() { + echo "${PANEL_TAILSCALE_TAG:-$TAILSCALE_SHARED_TAG}" +} + +tailnet_key_tags() { + local tag + tag="$(tailnet_tag)" + if [ "$tag" = "$TAILSCALE_SHARED_TAG" ]; then + jq -nc --arg s "$TAILSCALE_SHARED_TAG" '[$s]' + else + jq -nc --arg s "$TAILSCALE_SHARED_TAG" --arg t "$tag" '[$s, $t]' + fi +} get_oauth_token() { local client_id=$1 @@ -36,30 +69,22 @@ get_oauth_token() { } create_auth_key() { - local access_token=$1 + local access_token=$1 tags if [ -z "$access_token" ]; then echo "Error: Access token is required" >&2 return 1 fi + tags="$(tailnet_key_tags)" || return 1 local response=$(curl -s -X POST "${TAILSCALE_API_BASE}/tailnet/-/keys" \ -H "Authorization: Bearer ${access_token}" \ -H "Content-Type: application/json" \ - -d '{ - "capabilities": { - "devices": { - "create": { - "reusable": false, - "ephemeral": false, - "preauthorized": true, - "tags": ["tag:fivestack"] - } - } - }, - "expirySeconds": 3600, - "description": "5stack game node auth key" - }') + -d "$(jq -nc --argjson tags "$tags" '{ + capabilities: {devices: {create: {reusable: false, ephemeral: false, preauthorized: true, tags: $tags}}}, + expirySeconds: 3600, + description: "5stack game node auth key" + }')") if echo "$response" | grep -q '"message"' && ! echo "$response" | grep -q '"key"'; then echo "Error: Failed to create auth key" >&2 @@ -78,13 +103,16 @@ create_auth_key() { } update_acl_for_fivestack() { - local access_token=$1 + local access_token=$1 pods tag if [ -z "$access_token" ]; then echo "Error: Access token is required" >&2 return 1 fi + pods="$(tailnet_pod_range)" || return 1 + tag="$(tailnet_tag)" + local acl_response=$(curl -s -D - -X GET "${TAILSCALE_API_BASE}/tailnet/-/acl" \ -H "Authorization: Bearer ${access_token}" \ -H "Accept: application/json") @@ -109,22 +137,26 @@ update_acl_for_fivestack() { return 1 fi - updated_acl=$(echo "$body" | jq ' - .autoApprovers = (.autoApprovers // {}) + updated_acl=$(echo "$body" | jq --arg pods "$pods" --arg tag "$tag" --arg shared "$TAILSCALE_SHARED_TAG" ' + (if $tag != $shared then + .tagOwners = (.tagOwners // {}) + | .tagOwners[$tag] = (.tagOwners[$tag] // ["autogroup:admin"]) + else . end) + | .autoApprovers = (.autoApprovers // {}) | .autoApprovers.routes = (.autoApprovers.routes // {}) - | .autoApprovers.routes["10.42.0.0/16"] = ( - (.autoApprovers.routes["10.42.0.0/16"] // []) | - if index("tag:fivestack") == null then . + ["tag:fivestack"] else . end + | .autoApprovers.routes[$pods] = ( + (.autoApprovers.routes[$pods] // []) | + if index($tag) == null then . + [$tag] else . end ) | .grants = (.grants // []) | .grants = ( - [ - { src: ["tag:fivestack", "10.42.0.0/16"], dst: ["tag:fivestack", "10.42.0.0/16"], ip: ["*"] } + [ + { src: [$tag, $pods], dst: [$tag, $pods], ip: ["*"] } ] + ( .grants | map(select( - .src != ["tag:fivestack", "10.42.0.0/16"] - or .dst != ["tag:fivestack", "10.42.0.0/16"] + .src != [$tag, $pods] + or .dst != [$tag, $pods] or .ip != ["*"] )) ) diff --git a/utils/update_timescaledb_extension.sh b/utils/update_timescaledb_extension.sh index 9d7b3ef..d40ec5d 100644 --- a/utils/update_timescaledb_extension.sh +++ b/utils/update_timescaledb_extension.sh @@ -8,14 +8,18 @@ update_timescaledb_extension() { local before available after + echo " Waiting for TimescaleDB to start (the first start pulls its image)" if ! output_redirect kubectl --kubeconfig="$KUBECONFIG" -n 5stack \ rollout status statefulset/timescaledb --timeout=300s; then warn "timescaledb never became ready; its extension was not updated" return 1 fi - before="$(timescaledb_psql "SELECT extversion FROM pg_extension WHERE extname = 'timescaledb'")" - available="$(timescaledb_psql "SELECT default_version FROM pg_available_extensions WHERE name = 'timescaledb'")" + if ! before="$(timescaledb_psql "SELECT extversion FROM pg_extension WHERE extname = 'timescaledb'")" \ + || ! available="$(timescaledb_psql "SELECT default_version FROM pg_available_extensions WHERE name = 'timescaledb'")"; then + warn "timescaledb did not answer; its extension was not updated" + return 1 + fi if [ -z "$before" ] || [ -z "$available" ]; then return 0 fi @@ -35,7 +39,14 @@ update_timescaledb_extension() { ok "timescaledb extension updated from $before to $after" } +timescaledb_exec() { + local seconds="$1" + shift + time_limited "$seconds" kubectl --kubeconfig="$KUBECONFIG" -n 5stack --request-timeout="${seconds}s" \ + exec timescaledb-0 -c timescaledb -- "$@" +} + timescaledb_psql() { - kubectl --kubeconfig="$KUBECONFIG" -n 5stack exec timescaledb-0 -c timescaledb -- \ + timescaledb_exec "${TIMESCALEDB_EXEC_TIMEOUT:-120}" \ sh -c 'psql -X -v ON_ERROR_STOP=1 -At -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"' sh "$1" } diff --git a/utils/upgrade_timescaledb_major.sh b/utils/upgrade_timescaledb_major.sh index 9578f0b..236545d 100644 --- a/utils/upgrade_timescaledb_major.sh +++ b/utils/upgrade_timescaledb_major.sh @@ -13,6 +13,7 @@ TIMESCALEDB_UPGRADE_JOB=timescaledb-major-upgrade TIMESCALEDB_UPGRADE_MARKER=timescaledb-major-upgraded TIMESCALEDB_UPGRADE_FROM="" TIMESCALEDB_UPGRADE_ATTACH="" +TIMESCALEDB_FRESH_INSTALL="" timescaledb_kubectl() { kubectl --kubeconfig="$KUBECONFIG" -n 5stack "$@" @@ -309,6 +310,7 @@ plan_timescaledb_major_upgrade() { local pvc job active data_major live_image free_kb need_kb minutes TIMESCALEDB_UPGRADE_FROM="" TIMESCALEDB_UPGRADE_ATTACH="" + TIMESCALEDB_FRESH_INSTALL="" timescaledb_manifest_image || return 1 @@ -317,6 +319,7 @@ plan_timescaledb_major_upgrade() { return 1 fi if [ -z "$pvc" ]; then + export TIMESCALEDB_FRESH_INSTALL=true return 0 fi if [ "$pvc" != persistentvolumeclaim/timescaledb-pvc ]; then @@ -358,7 +361,7 @@ plan_timescaledb_major_upgrade() { return 0 fi - data_major="$(timescaledb_kubectl exec timescaledb-0 -c timescaledb -- cat /var/lib/postgresql/data/PG_VERSION 2>/dev/null)" + data_major="$(timescaledb_exec "${TIMESCALEDB_EXEC_TIMEOUT:-120}" cat /var/lib/postgresql/data/PG_VERSION 2>/dev/null)" if [ -z "$data_major" ]; then timescaledb_probe_volume || return 1 data_major="$TIMESCALEDB_PROBE_MAJOR" @@ -667,7 +670,7 @@ finish_timescaledb_major_upgrade() { return 0 fi - major="$(timescaledb_kubectl exec timescaledb-0 -c timescaledb -- cat /var/lib/postgresql/data/PG_VERSION 2>/dev/null)" + major="$(timescaledb_exec "${TIMESCALEDB_EXEC_TIMEOUT:-120}" cat /var/lib/postgresql/data/PG_VERSION 2>/dev/null)" if [ "$major" = "$from" ]; then if [ "$TIMESCALEDB_MAJOR" = "$from" ]; then timescaledb_kubectl delete configmap "$TIMESCALEDB_UPGRADE_MARKER" --ignore-not-found >/dev/null 2>&1 @@ -684,7 +687,7 @@ finish_timescaledb_major_upgrade() { # pg_upgrade from 18 on carries most statistics over, so only what is # missing needs analysing. # shellcheck disable=SC2016 - if output_redirect timescaledb_kubectl exec timescaledb-0 -c timescaledb -- sh -c ' + if output_redirect timescaledb_exec 3600 sh -c ' set -- --all --analyze-in-stages if vacuumdb --help | grep -q -- --missing-stats-only; then set -- "$@" --missing-stats-only diff --git a/utils/utils.sh b/utils/utils.sh index da8d410..6fff096 100755 --- a/utils/utils.sh +++ b/utils/utils.sh @@ -39,5 +39,6 @@ source "$SCRIPT_DIR/tailscale-api.sh" source "$SCRIPT_DIR/interactive_select.sh" source "$SCRIPT_DIR/open_docs.sh" source "$SCRIPT_DIR/print_domains_and_hosts.sh" +source "$SCRIPT_DIR/cloudflare_setup.sh" source "$SCRIPT_DIR/setup-env.sh" "$@" \ No newline at end of file diff --git a/utils/write_overlays.sh b/utils/write_overlays.sh index d282f4e..2e0f443 100644 --- a/utils/write_overlays.sh +++ b/utils/write_overlays.sh @@ -101,12 +101,12 @@ resources: - ../config $STREAMING_RESOURCES $AI_RESOURCES -$(if [[ "$PROTOCOL" == "https" ]]; then echo "- ../cert-manager"; echo "- https-redirect.yaml"; else echo "- gateway-tls.yaml"; fi) +$(if [[ "$PROTOCOL" == "https" ]]; then echo "- ../cert-manager"; echo "- https-redirect.yaml"; echo "- acme-check.yaml"; else echo "- gateway-tls.yaml"; fi) $HASURA_REPLICAS_ENTRY $STORAGE_COMPONENTS EOF if [ "$PROTOCOL" = "https" ]; then - cp "$PANEL_DIR/overlays/http/gateway-https-patch.yaml" "$PANEL_DIR/overlays/http/https-redirect.yaml" "$OVERLAY/" + cp "$PANEL_DIR/overlays/http/gateway-https-patch.yaml" "$PANEL_DIR/overlays/http/https-redirect.yaml" "$PANEL_DIR/overlays/http/acme-check.yaml" "$OVERLAY/" cat "$PANEL_DIR/overlays/http/https-replacements.yaml" >> "$OVERLAY/kustomization.yaml" else ensure_reverse_proxy_certificate "$OVERLAY" || return 1