From 1871b88ab3d007b2190837e9e14cb826346d0909 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 11:25:43 -0400 Subject: [PATCH 01/17] feat(operator): reserve auxiliary resources and expose management placement Signed-off-by: Baylem --- .github/workflows/ci.yaml | 12 +- charts/gameplane/templates/_helpers.tpl | 23 ++ charts/gameplane/templates/api.yaml | 1 + .../templates/audit-syslog-bridge.yaml | 1 + .../gameplane/templates/crd-apply-hook.yaml | 1 + charts/gameplane/templates/gateway.yaml | 1 + charts/gameplane/templates/mcp-server.yaml | 1 + charts/gameplane/templates/operator.yaml | 8 + .../templates/poddisruptionbudgets.yaml | 47 ++++ .../templates/telemetry-receiver.yaml | 1 + charts/gameplane/templates/validate.yaml | 3 + charts/gameplane/templates/web.yaml | 1 + .../gameplane/values-production.example.yaml | 36 +++ charts/gameplane/values.yaml | 65 ++++- charts/tests/check_workloads.py | 78 ++++++ deploy/kind/e2e.sh | 13 +- docs/production-placement.md | 140 ++++++++++ docs/release-notes.md | 21 ++ operator/cmd/main.go | 33 +++ .../controller/auxiliary_resources.go | 65 +++++ .../controller/auxiliary_resources_test.go | 105 ++++++++ .../internal/controller/backup_controller.go | 19 +- operator/internal/controller/backup_forget.go | 26 +- .../controller/gameserver_controller.go | 26 +- .../controller/gameserver_sentinel.go | 5 +- .../internal/controller/gameserver_tunnel.go | 1 + .../internal/controller/restore_controller.go | 7 +- operator/specs.md | 14 + test/e2e/buckets.sh | 8 +- test/e2e/management_placement_e2e_test.go | 251 ++++++++++++++++++ test/e2e/specs.md | 11 + 31 files changed, 979 insertions(+), 45 deletions(-) create mode 100644 charts/gameplane/templates/poddisruptionbudgets.yaml create mode 100644 charts/gameplane/values-production.example.yaml create mode 100644 charts/tests/check_workloads.py create mode 100644 docs/production-placement.md create mode 100644 docs/release-notes.md create mode 100644 operator/internal/controller/auxiliary_resources.go create mode 100644 operator/internal/controller/auxiliary_resources_test.go create mode 100644 test/e2e/management_placement_e2e_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1c9aecdd..a902708f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1115,6 +1115,8 @@ jobs: run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders run: python3 charts/tests/check_standalone.py + - name: auxiliary resources and management placement chart renders + run: python3 charts/tests/check_workloads.py - name: sanity-check rendered manifests run: | grep -q 'kind: Deployment' /tmp/rendered.yaml @@ -1664,13 +1666,17 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement] arch: [amd64] include: - bucket: operator parallel: 8 test_timeout: 20m job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: management-placement + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: api-auth parallel: 4 test_timeout: 15m @@ -1718,7 +1724,9 @@ jobs: with: arch: ${{ matrix.arch }} - name: bring up cluster + helm install - run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e + env: + GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }} + run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) run: | RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')" diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index bec3e1ffb..8085e8ac0 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -135,3 +135,26 @@ true true {{- end -}} {{- end -}} + +{{- /* Shared PodSpec placement fields; absent keys preserve older reuse-values. */ -}} +{{- define "gameplane.placement" -}} +{{- range $key := list "nodeSelector" "tolerations" "affinity" "topologySpreadConstraints" "priorityClassName" -}} +{{- with index $ $key }} +{{ $key }}: {{ toYaml . | nindent 2 }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- /* Complete auxiliary overrides; empty objects must not become defaults. */ -}} +{{- define "gameplane.auxiliaryResourceArgs" -}} +{{- range $name := list "agent" "sentinel" "tunnel" "backup" -}} +{{- $key := printf "%sResources" $name -}} +{{- $value := index $ $key -}} +{{- if and (hasKey $ $key) (not (kindIs "invalid" $value)) -}} +{{- if not (kindIs "map" $value) -}} +{{- fail (printf "operator.%s must be a resources object or null" $key) -}} +{{- end }} +- {{ printf "--%s-resources=%s" $name (toJson $value) | quote }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/gameplane/templates/api.yaml b/charts/gameplane/templates/api.yaml index a562d7c7b..f04fa99ec 100644 --- a/charts/gameplane/templates/api.yaml +++ b/charts/gameplane/templates/api.yaml @@ -258,6 +258,7 @@ spec: checksum/mtls: {{ include "gameplane.mtlsChecksum" . }} {{- end }} spec: + {{- include "gameplane.placement" .Values.api | nindent 6 }} serviceAccountName: gameplane-api {{- if .Values.api.standalone }} automountServiceAccountToken: false diff --git a/charts/gameplane/templates/audit-syslog-bridge.yaml b/charts/gameplane/templates/audit-syslog-bridge.yaml index 11bb8b279..c030d0150 100644 --- a/charts/gameplane/templates/audit-syslog-bridge.yaml +++ b/charts/gameplane/templates/audit-syslog-bridge.yaml @@ -27,6 +27,7 @@ spec: app.kubernetes.io/name: gameplane-audit-syslog-bridge {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" $b | nindent 6 }} securityContext: runAsNonRoot: true runAsUser: 65532 diff --git a/charts/gameplane/templates/crd-apply-hook.yaml b/charts/gameplane/templates/crd-apply-hook.yaml index b767b4e28..4ad3925cb 100644 --- a/charts/gameplane/templates/crd-apply-hook.yaml +++ b/charts/gameplane/templates/crd-apply-hook.yaml @@ -194,6 +194,7 @@ spec: app.kubernetes.io/name: gameplane-crd-apply {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" .Values.crds.autoApply | nindent 6 }} serviceAccountName: {{ .Release.Name }}-crd-apply restartPolicy: Never securityContext: diff --git a/charts/gameplane/templates/gateway.yaml b/charts/gameplane/templates/gateway.yaml index 82c46613e..aeab6bafc 100644 --- a/charts/gameplane/templates/gateway.yaml +++ b/charts/gameplane/templates/gateway.yaml @@ -113,6 +113,7 @@ spec: app.kubernetes.io/name: gameplane-gateway {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" $gateway | nindent 6 }} serviceAccountName: gameplane-gateway securityContext: runAsNonRoot: true diff --git a/charts/gameplane/templates/mcp-server.yaml b/charts/gameplane/templates/mcp-server.yaml index 3858878a3..40051e993 100644 --- a/charts/gameplane/templates/mcp-server.yaml +++ b/charts/gameplane/templates/mcp-server.yaml @@ -66,6 +66,7 @@ spec: app.kubernetes.io/name: gameplane-mcp-server {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" .Values.mcpServer | nindent 6 }} serviceAccountName: gameplane-mcp-server securityContext: runAsNonRoot: true diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 772e0b10e..444a60936 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -239,6 +239,12 @@ metadata: labels: {{- include "gameplane.labels" . | nindent 4 }} spec: replicas: {{ .Values.operator.replicas }} + {{- if not .Values.operator.leaderElect }} + # Without leader election, prevent a rolling update from overlapping writers. + strategy: + type: Recreate + rollingUpdate: null + {{- end }} selector: matchLabels: app.kubernetes.io/name: gameplane-operator @@ -250,6 +256,7 @@ spec: annotations: checksum/mtls: {{ include "gameplane.mtlsChecksum" . }} spec: + {{- include "gameplane.placement" .Values.operator | nindent 6 }} serviceAccountName: gameplane-operator securityContext: runAsNonRoot: true @@ -261,6 +268,7 @@ spec: image: {{ include "gameplane.operatorImage" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: + {{- include "gameplane.auxiliaryResourceArgs" .Values.operator | nindent 12 }} {{- if .Values.operator.leaderElect }} - --leader-elect {{- end }} diff --git a/charts/gameplane/templates/poddisruptionbudgets.yaml b/charts/gameplane/templates/poddisruptionbudgets.yaml new file mode 100644 index 000000000..35667e143 --- /dev/null +++ b/charts/gameplane/templates/poddisruptionbudgets.yaml @@ -0,0 +1,47 @@ +{{- $apiEnabled := eq (include "gameplane.apiEnabled" .) "true" -}} +{{- $bridge := .Values.api.audit.webhook.syslogBridge -}} +{{- $receiver := .Values.api.telemetry.receiver -}} +{{- $gateway := .Values.gateway | default dict -}} +{{- $components := list + (dict "name" "operator" "config" .Values.operator "enabled" (eq (include "gameplane.operatorEnabled" .) "true")) + (dict "name" "api" "config" .Values.api "enabled" $apiEnabled) + (dict "name" "web" "config" .Values.web "enabled" (and $apiEnabled .Values.web.enabled)) + (dict "name" "mcp-server" "config" .Values.mcpServer "enabled" .Values.mcpServer.enabled) + (dict "name" "gateway" "config" $gateway "enabled" $gateway.enabled) + (dict "name" "audit-syslog-bridge" "config" $bridge "enabled" (and $apiEnabled $bridge.enabled (not .Values.api.audit.webhook.url))) + (dict "name" "telemetry-receiver" "config" $receiver "enabled" (and $apiEnabled $receiver.enabled (not .Values.api.telemetry.endpoint))) -}} +{{- range $components -}} +{{- $pdb := .config.pdb | default dict -}} +{{- if and .enabled $pdb.enabled -}} +{{- $minimum := not (kindIs "invalid" $pdb.minAvailable) -}} +{{- $maximum := not (kindIs "invalid" $pdb.maxUnavailable) -}} +{{- if eq .name "api" -}} +{{- if or $minimum (and $maximum (ne (toString $pdb.maxUnavailable) "1")) -}} +{{- fail "api.pdb must allow the singleton API to be evicted: use maxUnavailable: 1 without minAvailable" -}} +{{- end -}} +{{- else if and $minimum $maximum -}} +{{- fail (printf "%s.pdb must set only one of minAvailable or maxUnavailable" .name) -}} +{{- end }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: gameplane-{{ .name }} + namespace: {{ $.Release.Namespace }} + labels: {{- include "gameplane.labels" $ | nindent 4 }} +spec: + {{- if eq .name "api" }} + maxUnavailable: 1 + {{- else if $minimum }} + minAvailable: {{ toYaml $pdb.minAvailable }} + {{- else if $maximum }} + maxUnavailable: {{ toYaml $pdb.maxUnavailable }} + {{- else }} + maxUnavailable: 1 + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: gameplane-{{ .name }} + app.kubernetes.io/instance: {{ $.Release.Name }} +{{- end -}} +{{- end -}} diff --git a/charts/gameplane/templates/telemetry-receiver.yaml b/charts/gameplane/templates/telemetry-receiver.yaml index 4e8255fbc..18a084d32 100644 --- a/charts/gameplane/templates/telemetry-receiver.yaml +++ b/charts/gameplane/templates/telemetry-receiver.yaml @@ -88,6 +88,7 @@ spec: app.kubernetes.io/name: gameplane-telemetry-receiver {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" $r | nindent 6 }} securityContext: runAsNonRoot: true runAsUser: 65532 diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index 79b5e17b6..c6df65ede 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,3 +1,6 @@ +{{- if and (eq (include "gameplane.operatorEnabled" .) "true") (gt (int .Values.operator.replicas) 1) (not .Values.operator.leaderElect) }} +{{- fail "operator.leaderElect must be enabled with more than one replica" }} +{{- end }} {{- if .Values.api.standalone }} {{- if ne (include "gameplane.apiEnabled" .) "true" }} {{- fail "api.standalone requires api.enabled=true" }} diff --git a/charts/gameplane/templates/web.yaml b/charts/gameplane/templates/web.yaml index 4429fc5e4..736fd7136 100644 --- a/charts/gameplane/templates/web.yaml +++ b/charts/gameplane/templates/web.yaml @@ -25,6 +25,7 @@ spec: app.kubernetes.io/name: gameplane-web {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" .Values.web | nindent 6 }} automountServiceAccountToken: false securityContext: runAsNonRoot: true diff --git a/charts/gameplane/values-production.example.yaml b/charts/gameplane/values-production.example.yaml new file mode 100644 index 000000000..229448ac3 --- /dev/null +++ b/charts/gameplane/values-production.example.yaml @@ -0,0 +1,36 @@ +# Example policy, not a capacity recommendation. Label/taint worker nodes before +# applying this profile; see docs/production-placement.md. Never remove or +# tolerate control-plane taints just to satisfy these selectors. +operator: + replicas: 2 + leaderElect: true + nodeSelector: &managementNodes { gameplane.local/pool: management } + tolerations: &managementTolerations + - { key: gameplane.local/pool, operator: Equal, value: management, effect: NoSchedule } + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: { app.kubernetes.io/name: gameplane-operator } + pdb: { enabled: true, maxUnavailable: 1 } +api: + replicas: 1 + nodeSelector: *managementNodes + tolerations: *managementTolerations + pdb: { enabled: true, maxUnavailable: 1 } +web: + replicas: 2 + nodeSelector: *managementNodes + tolerations: *managementTolerations + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: { app.kubernetes.io/name: gameplane-web } + pdb: { enabled: true, maxUnavailable: 1 } +crds: + autoApply: + nodeSelector: *managementNodes + tolerations: *managementTolerations diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 66d95af0b..9346f8e00 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -39,6 +39,12 @@ image: crds: autoApply: enabled: true + # The upgrade hook must be schedulable before new Deployments roll out. + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" # kubectl image the apply Job runs. Any image with kubectl on PATH works; # retag to a private mirror for air-gapped installs (needed at upgrade # time, and also on an install over CRDs an earlier, uninstalled release @@ -53,17 +59,28 @@ operator: # Disable the operator and bundled native CRDs for a standalone central panel. enabled: true replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } # optional minAvailable OR maxUnavailable; defaults to maxUnavailable: 1 logLevel: info # zap log level: debug|info|error (warn not supported by zap's --zap-log-level) agentLogLevel: "" # injected into every agent sidecar as GAMEPLANE_LOG_LEVEL (debug|info|warn|error); empty injects nothing, avoiding StatefulSet rollouts on upgrade - # Leader election guards against two operators reconciling at once. With a - # single replica it's unnecessary, and on a resource-starved node a missed - # lease renewal makes controller-runtime exit (the operator then crash-loops - # and stops reconciling). Leave on for HA installs; turn off for single-replica - # or constrained clusters (the e2e install sets it false). + # Required with multiple replicas and recommended during rolling upgrades. + # Disabling it with one replica selects Recreate to prevent overlapping writers. leaderElect: true resources: requests: { cpu: 100m, memory: 64Mi } limits: { cpu: 500m, memory: 256Mi } + # Complete auxiliary ResourceRequirements overrides. Null/missing uses binary + # defaults, including on --reuse-values upgrades. {} (or requests: {}, limits: + # {}) clears defaults. No new CPU/memory limits are set by default. + agentResources: null # requests 25m CPU / 64Mi memory; also config-init + sentinelResources: null # requests 25m CPU / 32Mi memory + tunnelResources: null # requests 25m CPU / 64Mi memory (every provider) + backupResources: null # backup/init/restore requests 100m CPU / 128Mi memory; + # cleanup retains 50m / 128Mi, memory limit 512Mi agentImage: "" # defaults to {image.registry}/agent:{image.tag} sentinelImage: "" # defaults to {image.registry}/sentinel:{image.tag} # Which load-balancer address manager runs in this cluster, so the operator @@ -133,6 +150,14 @@ api: # Standalone mode also skips the CRD apply hook and its RBAC. standalone: false replicas: 1 # Keep at 1. SQLite is single-writer on a RWO volume; with db.driver=postgres the user-management lock and audit hash chain are still per-process + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + # The API is a singleton: eviction causes downtime. A blocking budget would + # hang voluntary node drains, so only maxUnavailable: 1 is supported here. + pdb: { enabled: false, maxUnavailable: 1 } logLevel: info # API log verbosity (debug|info|warn|error), passed as GAMEPLANE_LOG_LEVEL # Port of the API's Prometheus metrics listener. It is separate from the # public API port (the one the ingress and the web front end route to), so @@ -241,6 +266,12 @@ api: facility: local0 # RFC 5424 facility: kern/user/daemon/auth/.../local0..local7 severity: info # emerg/alert/crit/err/warning/notice/info/debug replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: {} # Native S3-compatible audit sink. When endpoint and bucket are set, the API # batches audit events as NDJSON and uploads them to S3 (or MinIO, Backblaze, @@ -314,6 +345,12 @@ api: # Must stay 1 while persistence is enabled: the SQLite file on the PVC # has a single writer, so the template fails the render otherwise. replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: {} # Aggregate storage (SQLite on a PVC mounted at /data). With # persistence disabled an emptyDir is used instead and every @@ -422,6 +459,12 @@ api: gateway: enabled: false replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } # Must match the Cluster resource name in the central Gameplane installation. clusterID: "" # Exact URI SAN trusted in the central API client certificate. @@ -465,6 +508,12 @@ gateway: web: enabled: true replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: requests: { cpu: 10m, memory: 16Mi } limits: { cpu: 200m, memory: 128Mi } @@ -602,6 +651,12 @@ clusterOps: mcpServer: enabled: false replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: requests: { cpu: 50m, memory: 32Mi } limits: { cpu: 200m, memory: 128Mi } diff --git a/charts/tests/check_workloads.py b/charts/tests/check_workloads.py new file mode 100644 index 000000000..33fd6f89f --- /dev/null +++ b/charts/tests/check_workloads.py @@ -0,0 +1,78 @@ +"""CI-only resource override, management placement and disruption checks.""" +import json +from pathlib import Path +import shutil +import tempfile + +import yaml + +from check_gateway import CHART, one, render, settings + + +def main(): + defaults = render() + assert not any(obj["kind"] == "PodDisruptionBudget" for obj in defaults) + args = one(defaults, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert not any(arg.startswith("--tunnel-resources=") for arg in args) + placement = { + "nodeSelector": {"gameplane.local/pool": "management"}, + "tolerations": [{"key": "gameplane.local/pool", "operator": "Equal", "value": "management", "effect": "NoSchedule"}], + "affinity": {"podAntiAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [{"weight": 100, "podAffinityTerm": {"topologyKey": "kubernetes.io/hostname", "labelSelector": {"matchLabels": {"app.kubernetes.io/instance": "gameplane"}}}}]}}, + "topologySpreadConstraints": [{"maxSkew": 1, "topologyKey": "kubernetes.io/hostname", "whenUnsatisfiable": "DoNotSchedule", "labelSelector": {"matchLabels": {"app.kubernetes.io/instance": "gameplane"}}}], + "priorityClassName": "management", + } + component = dict(placement, pdb={"enabled": True, "maxUnavailable": 1}) + gateway = dict(settings(), **component) + values = { + "operator": dict(component, replicas=2, tunnelResources={"requests": {}, "limits": {}}, agentResources={"requests": {"cpu": "80m"}}), + "api": dict(component, audit={"webhook": {"syslogBridge": dict(component, enabled=True, syslog={"addr": "logs:514"})}}, telemetry={"receiver": dict(component, enabled=True)}), + "web": component, "mcpServer": dict(component, enabled=True), "gateway": gateway, + "crds": {"autoApply": placement}, + } + production = render(values, upgrade=True) + for name in ["operator", "api", "web", "mcp-server", "gateway", "audit-syslog-bridge", "telemetry-receiver"]: + deployment = one(production, "Deployment", "gameplane-" + name) + spec = deployment["spec"]["template"]["spec"] + for key, expected in placement.items(): + assert spec[key] == expected, (name, key) + pdb = one(production, "PodDisruptionBudget", "gameplane-" + name)["spec"] + assert pdb["maxUnavailable"] == 1 and "minAvailable" not in pdb + assert pdb["selector"]["matchLabels"].items() <= deployment["spec"]["template"]["metadata"]["labels"].items() + hook = one(production, "Job", "gameplane-crd-apply")["spec"]["template"]["spec"] + for key, expected in placement.items(): + assert hook[key] == expected + args = one(production, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + overrides = {arg.split("=", 1)[0]: json.loads(arg.split("=", 1)[1]) for arg in args if arg.startswith(("--agent-resources=", "--tunnel-resources="))} + assert overrides["--tunnel-resources"] == {"requests": {}, "limits": {}} + assert overrides["--agent-resources"] == {"requests": {"cpu": "80m"}} + render({"api": {"pdb": {"enabled": True, "minAvailable": 1}}}, failure="api.pdb must allow the singleton API to be evicted") + render({"api": {"pdb": {"enabled": True, "maxUnavailable": 0}}}, failure="api.pdb must allow the singleton API to be evicted") + render({"operator": {"replicas": 2, "leaderElect": False}}, failure="operator.leaderElect must be enabled") + single = render({"operator": {"leaderElect": False}}) + assert one(single, "Deployment", "gameplane-operator")["spec"]["strategy"] == {"type": "Recreate", "rollingUpdate": None} + removed = render({"operator": {"agentResources": None, "tunnelResources": {}}}) + args = one(removed, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert not any(arg.startswith("--agent-resources=") for arg in args) + assert "--tunnel-resources={}" in args + disabled = render({"api": {"enabled": False, "pdb": {"enabled": True}}, "web": {"pdb": {"enabled": True}}}) + assert not any(obj["kind"] == "PodDisruptionBudget" for obj in disabled) + profile = yaml.safe_load((CHART / "values-production.example.yaml").read_text()) + example = render(profile, upgrade=True) + assert one(example, "Deployment", "gameplane-operator")["spec"]["replicas"] == 2 + assert one(example, "PodDisruptionBudget", "gameplane-api")["spec"]["maxUnavailable"] == 1 + # Helm --reuse-values may not contain new keys. Omission must retain binary + # resource defaults and empty placement without indexing a missing map. + with tempfile.TemporaryDirectory() as directory: + old_chart = Path(directory) / "chart" + shutil.copytree(CHART, old_chart) + old_values = yaml.safe_load((old_chart / "values.yaml").read_text()) + for key in ["agentResources", "sentinelResources", "tunnelResources", "backupResources", "pdb", *placement]: + old_values["operator"].pop(key, None) + (old_chart / "values.yaml").write_text(yaml.safe_dump(old_values)) + old = render(chart=old_chart) + args = one(old, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert not any(arg.startswith("--tunnel-resources=") for arg in args) + + +if __name__ == "__main__": + main() diff --git a/deploy/kind/e2e.sh b/deploy/kind/e2e.sh index 3c2f7b12f..8f5df5211 100755 --- a/deploy/kind/e2e.sh +++ b/deploy/kind/e2e.sh @@ -6,7 +6,7 @@ # deploy/kind/e2e.sh down [cluster-name] # # Diffs from dev-up (deploy/kind/up.sh): -# - Single-node cluster (faster boot, sufficient for E2E coverage). +# - Single-node by default; management profile adds two scheduling workers. # - Skips ingress-nginx (the dashboard isn't exercised here). # - Loads pre-built gameplane-test/{operator,api,agent,sentinel,capture-sidecar,telemetry-receiver,fakeoidc}: # images (and gameplane-test/gameprobe: when present). @@ -28,9 +28,9 @@ CLUSTER="${2:-gameplane-e2e}" TAG="${3:-${GAMEPLANE_E2E_TAG:-e2e}}" PROFILE="${4:-default}" case "${PROFILE}" in -default | postgres) ;; +default | postgres | management) ;; *) - echo "unknown profile '${PROFILE}' (want default or postgres)" >&2 + echo "unknown profile '${PROFILE}' (want default, postgres or management)" >&2 exit 2 ;; esac @@ -188,12 +188,17 @@ up) echo "removing pre-existing cluster ${CLUSTER} for a clean slate" kind delete cluster --name "${CLUSTER}" || true fi - echo "creating kind cluster ${CLUSTER} (single-node)" + echo "creating kind cluster ${CLUSTER} (${PROFILE})" + extra_nodes="" + if [[ "${PROFILE}" == "management" ]]; then + extra_nodes=$' - role: worker\n - role: worker' + fi kind create cluster --name "${CLUSTER}" --wait 90s --config - </dev/null diff --git a/docs/production-placement.md b/docs/production-placement.md new file mode 100644 index 000000000..40bdb41b0 --- /dev/null +++ b/docs/production-placement.md @@ -0,0 +1,140 @@ +# Resource reservations and management placement + +New installs and upgrades reserve resources for supporting containers. These +requests are scheduling floors, not measured capacity targets: + +| Container | CPU request | Memory request | +| --- | --- | --- | +| Agent and config initializer | 25m | 64Mi | +| Wake sentinel | 25m | 32Mi | +| Tunnel, every provider | 25m | 64Mi | +| Restic backup, repository initializer and restore | 100m | 128Mi | + +These defaults add no CPU or memory limits. CPU limits can throttle bursts; +memory limits can terminate the process when its working set exceeds them. +Repository forget/prune retains its existing 50m/128Mi requests and 512Mi memory +limit unless overridden. Init containers receive requests too; Kubernetes +accounts for their peak requirements when scheduling the Pod. + +## Overrides and upgrades + +The new chart keys `operator.agentResources`, `operator.sentinelResources`, +`operator.tunnelResources` and `operator.backupResources` each accept a complete +Kubernetes ResourceRequirements object. Backup resources also apply to restore +and forget/prune workers; agent resources also apply to the config initializer. +The matching operator flags are `--agent-resources`, `--sentinel-resources`, +`--tunnel-resources`, and `--backup-resources`, with JSON values. + +Null or a missing chart key uses the binary defaults, including older +`helm upgrade --reuse-values` configurations. A supplied object replaces defaults +wholesale. Clear reservations explicitly: + +```yaml +operator: + agentResources: { requests: {}, limits: {} } + sentinelResources: { requests: {}, limits: {} } + tunnelResources: { requests: {}, limits: {} } + backupResources: { requests: {}, limits: {} } +``` + +An empty object `{}` has the same effect. Setting a key back to `null` restores +the defaults. No chart-side default map is merged into the supplied object. +For example `agentResources: {requests: {memory: 96Mi}}` reserves only memory. + +Helm still merges user-supplied maps. When replacing a previously populated +override with empty maps, export the release's values, edit that complete file, +and upgrade with `--reset-values -f `. An empty nested map +combined with `--reuse-values` can retain the release's old resource entries. +Keep all unrelated install values in the edited file; resetting to chart defaults +alone is not a resource-only change. + +Existing `GameTemplate.spec.agent.resources` takes precedence when it declares +requests or limits, replacing the install agent object wholesale. An image-only +agent override keeps install defaults. Kubernetes may omit empty maps in a +template, so use explicit zero CPU/memory requests for a per-template zero +reservation, or the install-level empty object to clear all defaults. Game +container resources continue to use the existing template defaults and complete +`GameServer.spec.resources` override. Game placement fields are unchanged. + +Changing resources changes generated Pod templates. Existing game StatefulSets, +sentinels and tunnels roll according to their normal reconciliation policy; +new Restic Jobs use the new configuration. Schedule this upgrade when game pod +restarts are acceptable. Insufficient capacity can leave replacement pods or +Jobs Pending. Check requested resources and concurrent backups before upgrading. + +## Management workers + +The chart now exposes `nodeSelector`, `tolerations`, `affinity`, +`topologySpreadConstraints`, and `priorityClassName` beneath: + +- `operator`, `api`, `web`, `gateway`, and `mcpServer`. +- `api.audit.webhook.syslogBridge` and `api.telemetry.receiver`. +- `crds.autoApply` for the upgrade hook Job. + +Each Deployment also supports `pdb.enabled` and either `pdb.minAvailable` or +`pdb.maxUnavailable`; enabled budgets default to `maxUnavailable: 1`. PDBs select +both the component name and Helm release. Disabled components get no PDB. Hook +Jobs have placement settings but no disruption budget. + +The API remains one process for both SQLite and PostgreSQL. Its optional PDB +only permits `maxUnavailable: 1`, so a voluntary node drain can evict it and will +cause API downtime. A PDB is not an availability guarantee and does not prevent +node failures. The persistent telemetry receiver is also a singleton; avoid a +zero-unavailable budget unless intentionally accepting a blocked drain. + +Keep operator leader election enabled with redundant replicas and rolling +updates. The chart rejects multiple replicas with leader election disabled; +with one replica and election disabled it uses Recreate to avoid overlapping +reconcilers during upgrades. API replica/process restrictions remain unchanged. + +The [example profile](../charts/gameplane/values-production.example.yaml) places +management services on labeled, tainted workers and spreads redundant operator +and dashboard pods. First label and taint the selected **worker** nodes: + +```sh +kubectl label node management-a management-b gameplane.local/pool=management +kubectl taint node management-a management-b gameplane.local/pool=management:NoSchedule +kubectl label node game-a game-b gameplane.local/pool=games +kubectl taint node game-a game-b gameplane.local/pool=games:NoSchedule +``` + +Use existing GameServer placement fields for game workers: + +```yaml +spec: + nodeSelector: { gameplane.local/pool: games } + tolerations: + - { key: gameplane.local/pool, operator: Equal, value: games, effect: NoSchedule } +``` + +Preserve control-plane taints. Avoid broad `operator: Exists` tolerations. +Optional management components need their corresponding placement values too. +Per-game sentinel and tunnel Deployments are operator-generated and are not +covered by these chart management placement keys. A priorityClassName must name +an existing PriorityClass. Unsatisfied hard selectors, taints or topology +constraints correctly leave pods Pending; do not silently relax them. + +Local or node-bound volumes cannot move just because a pod has a new selector. +Check API/receiver PVC node affinity before selecting management workers, and +game PVC topology before backups or moving game pods. Restic scheduling and +source preparation are separate from management placement. + +## Sizing and controlled-drain checks + +Collect per-container CPU/memory, pod requests, Pending events, game tick/query +latency, storage throughput and backup duration during busy periods and actual +backup concurrency. Use `kubectl top pods --containers`, scheduler events and +the available Prometheus metrics. Include headroom for one failed management +node, game startup, repository maintenance and replacement pods. No fixed server +count follows from these request values. + +In a disposable multi-worker environment, apply the example and verify every +management pod's node labels. Temporarily set a nonexistent selector and verify +Pending/FailedScheduling, then restore it. Cordon one management worker, evict +its redundant operator/web pods through the eviction API, and verify their PDB +allows only one unavailable replica while the remaining replica stays Ready. +Uncordon after checking replacements. A full `kubectl drain` also evicts the +singleton API and can interrupt access; account for local PVCs and validate +recovery before doing this in production. CI chart checks verify selectors, +PDB gates, resource clearing and the production profile; cluster behavior needs +the multi-node integration scenario, not just a successful render. diff --git a/docs/release-notes.md b/docs/release-notes.md new file mode 100644 index 000000000..f3da01f11 --- /dev/null +++ b/docs/release-notes.md @@ -0,0 +1,21 @@ +# Unreleased operational changes + +## Auxiliary resource requests and management placement (#724) + +Agent/config-init (25m CPU, 64Mi), sentinel (25m, 32Mi), tunnel (25m, 64Mi), and +Restic backup/init/restore (100m, 128Mi) now request resources on every install. +These are initial scheduling reservations, not workload capacity estimates. +No new CPU or memory limits are imposed. Existing forget/prune limits remain. + +Upgrades change generated Pod templates and can restart game/auxiliary pods or +leave them Pending when capacity is insufficient. Review capacity and schedule +the upgrade accordingly. The four `operator.*Resources` chart values provide +complete overrides; explicit empty objects clear defaults, while null/missing +keys use the defaults even with `--reuse-values`. + +Chart-managed pods now accept placement, topology-spread and priority settings, +with optional PDBs for Deployments. API remains a singleton and its PDB uses +`maxUnavailable: 1`; node drains can cause API downtime. Operator replicas greater +than one require leader election. Disabling election on a singleton uses Recreate. +See [production placement](production-placement.md) for upgrade controls and a +worker-pool example. diff --git a/operator/cmd/main.go b/operator/cmd/main.go index 85263e901..9625068fb 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -150,10 +150,18 @@ func main() { captureDefaultMaxSizeBytes int64 backupJobBackoffLimit int64 backupJobActiveDeadlineSeconds int64 + agentResourcesJSON string + sentinelResourcesJSON string + tunnelResourcesJSON string + backupResourcesJSON string ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "Address the metrics endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "Address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election.") + flag.StringVar(&agentResourcesJSON, "agent-resources", "", "JSON ResourceRequirements for agents and config init containers; default requests 25m/64Mi, {} clears defaults.") + flag.StringVar(&sentinelResourcesJSON, "sentinel-resources", "", "JSON ResourceRequirements for sentinels; default requests 25m/32Mi, {} clears defaults.") + flag.StringVar(&tunnelResourcesJSON, "tunnel-resources", "", "JSON ResourceRequirements for tunnel relays; default requests 25m/64Mi, {} clears defaults.") + flag.StringVar(&backupResourcesJSON, "backup-resources", "", "JSON ResourceRequirements for restic backup/init, restore and cleanup workers; default backup/restore requests 100m/128Mi, {} clears defaults.") flag.StringVar(&agentImage, "agent-image", "ghcr.io/gameplanepanel/gameplane/agent:dev", "Image to use for the Gameplane agent sidecar injected into game pods.") flag.StringVar(&agentImagePullPolicy, "agent-image-pull-policy", "", @@ -263,6 +271,26 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) setupLog := ctrl.Log.WithName("setup") + agentResources, err := controller.ParseAuxiliaryResources(agentResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --agent-resources") + os.Exit(1) + } + sentinelResources, err := controller.ParseAuxiliaryResources(sentinelResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --sentinel-resources") + os.Exit(1) + } + tunnelResources, err := controller.ParseAuxiliaryResources(tunnelResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --tunnel-resources") + os.Exit(1) + } + backupResources, err := controller.ParseAuxiliaryResources(backupResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --backup-resources") + os.Exit(1) + } backupJobBackoff32, err := boundedInt32(backupJobBackoffLimit) // rejects <0 and >MaxInt32 if err != nil { setupLog.Error(err, "invalid --backup-job-backoff-limit value (must be >= 0)", "value", backupJobBackoffLimit) @@ -380,6 +408,9 @@ func main() { APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), AgentImage: agentImage, + AgentResources: agentResources, + SentinelResources: sentinelResources, + TunnelResources: tunnelResources, AgentImagePullPolicy: agentImagePullPolicy, ConfigInitImage: configInitImage, SentinelImage: sentinelImage, @@ -419,6 +450,7 @@ func main() { Clientset: kubernetes.NewForConfigOrDie(mgr.GetConfig()), AgentClient: agentClient, ResticImage: resticImage, + BackupResources: backupResources, JobBackoffLimit: &backupJobBackoff32, JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds, EventRecorder: mgr.GetEventRecorder("backup-controller"), @@ -438,6 +470,7 @@ func main() { APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), ResticImage: resticImage, + BackupResources: backupResources, JobBackoffLimit: &backupJobBackoff32, JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds, }).SetupWithManager(mgr); err != nil { diff --git a/operator/internal/controller/auxiliary_resources.go b/operator/internal/controller/auxiliary_resources.go new file mode 100644 index 000000000..8f7bc7901 --- /dev/null +++ b/operator/internal/controller/auxiliary_resources.go @@ -0,0 +1,65 @@ +package controller + +import ( + "encoding/json" + "fmt" + "io" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation" +) + +// ParseAuxiliaryResources parses an install-level resource override. An absent +// flag returns nil (use defaults); an explicit object, including {}, replaces +// defaults wholesale. Reject typos before the operator starts creating pods. +func ParseAuxiliaryResources(value string) (*corev1.ResourceRequirements, error) { + if value == "" { + return nil, nil + } + var configured *corev1.ResourceRequirements + decoder := json.NewDecoder(strings.NewReader(value)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&configured); err != nil { + return nil, fmt.Errorf("decode resources: %w", err) + } + if configured == nil { + return nil, fmt.Errorf("resources must be a JSON object; use {} to clear defaults") + } + if err := decoder.Decode(new(any)); err != io.EOF { + return nil, fmt.Errorf("resources must contain exactly one JSON object") + } + if len(configured.Claims) != 0 { + return nil, fmt.Errorf("resource claims are not supported for auxiliary containers") + } + for _, values := range []corev1.ResourceList{configured.Requests, configured.Limits} { + for name, quantity := range values { + if len(validation.IsQualifiedName(string(name))) != 0 || quantity.Sign() < 0 { + return nil, fmt.Errorf("invalid resource %q or negative quantity", name) + } + } + } + for name, request := range configured.Requests { + if limit, exists := configured.Limits[name]; exists && request.Cmp(limit) > 0 { + return nil, fmt.Errorf("request for %s exceeds its limit", name) + } + } + return configured, nil +} + +func auxiliaryResources(configured *corev1.ResourceRequirements, cpu, memory string) corev1.ResourceRequirements { + if configured != nil { + return *configured.DeepCopy() + } + return corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(cpu), corev1.ResourceMemory: resource.MustParse(memory), + }} +} + +func optionalResources(values []*corev1.ResourceRequirements) *corev1.ResourceRequirements { + if len(values) == 0 { + return nil + } + return values[0] +} diff --git a/operator/internal/controller/auxiliary_resources_test.go b/operator/internal/controller/auxiliary_resources_test.go new file mode 100644 index 000000000..f63cef5f8 --- /dev/null +++ b/operator/internal/controller/auxiliary_resources_test.go @@ -0,0 +1,105 @@ +package controller + +import ( + "reflect" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestAuxiliaryResourceConfiguration(t *testing.T) { + for _, bad := range []string{"null", `{"request":{"cpu":"1"}}`, `{"requests":{"cpu":"-1"}}`, `{"requests":{"memory":"128Mi"},"limits":{"memory":"64Mi"}}`, `{} {}`} { + if _, err := ParseAuxiliaryResources(bad); err == nil { + t.Errorf("accepted invalid resources %s", bad) + } + } + unset, err := ParseAuxiliaryResources("") + if err != nil || unset != nil { + t.Fatalf("absent configuration: %v %v", unset, err) + } + cleared, err := ParseAuxiliaryResources(`{"requests":{},"limits":{}}`) + if err != nil || cleared == nil { + t.Fatalf("explicit clear: %v %v", cleared, err) + } + if got := auxiliaryResources(cleared, "25m", "64Mi"); len(got.Requests) != 0 || len(got.Limits) != 0 { + t.Fatal("clear restored default reservations") + } + defaults := auxiliaryResources(nil, "25m", "64Mi") + if defaults.Requests.Cpu().String() != "25m" || defaults.Requests.Memory().String() != "64Mi" || len(defaults.Limits) != 0 { + t.Fatalf("unexpected scheduling defaults: %+v", defaults) + } +} + +func TestAuxiliaryResourcesReachAgentAndResticContainers(t *testing.T) { + configured := &corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("73m")}} + gs := &gameplanev1alpha1.GameServer{} + tmpl := &gameplanev1alpha1.GameTemplate{} + agent := buildAgentContainer(gs, tmpl, nil, "agent", "", "", configured) + if !reflect.DeepEqual(agent.Resources, *configured) { + t.Fatalf("agent lost install resources: %+v", agent.Resources) + } + tmpl.Spec.Agent = &gameplanev1alpha1.AgentSpec{Image: "custom-agent"} + agent = buildAgentContainer(gs, tmpl, nil, "agent", "", "", configured) + if !reflect.DeepEqual(agent.Resources, *configured) { + t.Fatal("image-only template override removed resource defaults") + } + tmpl.Spec.Agent.Resources = corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("91Mi")}} + agent = buildAgentContainer(gs, tmpl, nil, "agent", "", "", configured) + if !reflect.DeepEqual(agent.Resources, tmpl.Spec.Agent.Resources) { + t.Fatal("template agent resources did not replace the install defaults") + } + b := &gameplanev1alpha1.Backup{Spec: gameplanev1alpha1.BackupSpec{RepoRef: &gameplanev1alpha1.SecretKeySelector{Name: "repo"}}} + br := &BackupReconciler{BackupResources: configured} + backup := br.buildBackupPodSpec(b, tmpl) + forget := br.buildForgetPodSpec(b) + rr := &RestoreReconciler{BackupResources: configured} + restore := rr.buildRestorePodSpec(&gameplanev1alpha1.Restore{}, b, tmpl) + for _, container := range append(append(append(backup.Containers, backup.InitContainers...), forget.Containers...), restore.Containers...) { + if !reflect.DeepEqual(container.Resources, *configured) { + t.Errorf("container %s lost resource configuration: %+v", container.Name, container.Resources) + } + } + if init := buildConfigInitContainer("init", tmpl, configured); !reflect.DeepEqual(init.Resources, *configured) { + t.Fatal("config init lost install resource configuration") + } + // Builders must not return the shared maps: one container's customization + // cannot mutate the defaults for another server or operation. + agent.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1Mi") + if tmpl.Spec.Agent.Resources.Requests.Memory().String() != "91Mi" { + t.Fatal("agent resources alias the template") + } +} + +func TestAuxiliaryDeploymentResourceOverrideAndRemoval(t *testing.T) { + s := wipeScheme(t) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "games", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "frp", Frp: &gameplanev1alpha1.FrpTunnelSpec{ServerAddr: "relay.example"}}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + custom := &corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("81m")}} + for _, configured := range []*corev1.ResourceRequirements{nil, custom, {}} { + r.SentinelResources, r.TunnelResources = configured, configured + if err := r.reconcileSentinel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + if err := r.reconcileTunnel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + for _, component := range []struct{ suffix, memory string }{{"waker", "32Mi"}, {"tunnel", "64Mi"}} { + var dep appsv1.Deployment + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-" + component.suffix}, &dep); err != nil { + t.Fatal(err) + } + want := auxiliaryResources(configured, "25m", component.memory) + if !reflect.DeepEqual(dep.Spec.Template.Spec.Containers[0].Resources, want) { + t.Fatalf("%s did not apply/clear resources", component.suffix) + } + } + } +} diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index b350a5448..3e5a82d2b 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -119,6 +119,9 @@ type BackupReconciler struct { // operator flag so air-gapped installs can point it at a private // registry mirror. Empty falls back to DefaultResticImage. ResticImage string + // BackupResources replaces resources for backup/init and repository cleanup + // workers. Nil uses defaults; an explicit empty object clears them. + BackupResources *corev1.ResourceRequirements // JobBackoffLimit / JobActiveDeadlineSeconds bound the restic backup Job. // Set from operator flags; nil / zero fall back to // DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds. @@ -965,9 +968,10 @@ func (r *BackupReconciler) buildBackupPodSpec(b *gameplanev1alpha1.Backup, tmpl return corev1.PodSpec{ SecurityContext: buildBackupPodSecurityContext(tmpl), InitContainers: []corev1.Container{{ - Name: "restic-init", - Image: resticImageOrDefault(r.ResticImage), - Command: []string{"/bin/sh", "-c"}, + Name: "restic-init", + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), + Command: []string{"/bin/sh", "-c"}, // `cat config` is the canonical "does the repo exist" probe; // `init` is idempotent against an empty target. Args: []string{"restic cat config --no-lock >/dev/null 2>&1 || restic init"}, @@ -978,10 +982,11 @@ func (r *BackupReconciler) buildBackupPodSpec(b *gameplanev1alpha1.Backup, tmpl SecurityContext: containerSec, }}, Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), - Args: args, - Env: env, + Name: "restic", + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), + Args: args, + Env: env, VolumeMounts: []corev1.VolumeMount{ {Name: "data", MountPath: "/data", ReadOnly: true}, {Name: "cache", MountPath: "/tmp"}, diff --git a/operator/internal/controller/backup_forget.go b/operator/internal/controller/backup_forget.go index 86f1c53cf..2d97d9410 100644 --- a/operator/internal/controller/backup_forget.go +++ b/operator/internal/controller/backup_forget.go @@ -420,6 +420,12 @@ func forgetJobName(backupName string) string { func (r *BackupReconciler) buildForgetPodSpec(b *gameplanev1alpha1.Backup) corev1.PodSpec { nonRoot := true uid := int64(65532) + resources := auxiliaryResources(r.BackupResources, "50m", "128Mi") + if r.BackupResources == nil { + // Preserve the existing bounded repository-cleanup default. Explicit + // install overrides replace it along with other restic worker resources. + resources.Limits = corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")} + } env := resticEnv(b.Spec.RepoRef.Name) env = append(env, corev1.EnvVar{Name: forgetSnapshotIDEnv, Value: b.Status.SnapshotID}) @@ -433,20 +439,12 @@ func (r *BackupReconciler) buildForgetPodSpec(b *gameplanev1alpha1.Backup) corev SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, }, Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), - Command: []string{"/bin/sh", "-c"}, - Args: []string{forgetScript}, - Env: env, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("50m"), - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - }, + Name: "restic", + Image: resticImageOrDefault(r.ResticImage), + Command: []string{"/bin/sh", "-c"}, + Args: []string{forgetScript}, + Env: env, + Resources: resources, VolumeMounts: []corev1.VolumeMount{ {Name: "cache", MountPath: "/tmp"}, }, diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 67e8be4e3..4093959d8 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -68,6 +68,12 @@ type GameServerReconciler struct { // to info and existing StatefulSets don't roll on operator upgrade. AgentLogLevel string + // Auxiliary resources are complete install-level overrides. Nil uses the + // built-in requests; a non-nil empty object explicitly clears them. + AgentResources *corev1.ResourceRequirements + SentinelResources *corev1.ResourceRequirements + TunnelResources *corev1.ResourceRequirements + // ConfigInitImage is the image for the init container that copies // operator-rendered config files onto the data volume. Set from an // operator flag so air-gapped installs can point it at a private @@ -1509,7 +1515,7 @@ func (r *GameServerReconciler) reconcileStatefulSet( ss.Spec.Template.Annotations = ann ss.Spec.Template.Spec.Containers = []corev1.Container{ buildGameContainer(gs, tmpl, image, ver, mc), - buildAgentContainer(gs, tmpl, ver, r.AgentImage, r.AgentLogLevel, r.AgentImagePullPolicy), + buildAgentContainer(gs, tmpl, ver, r.AgentImage, r.AgentLogLevel, r.AgentImagePullPolicy, r.AgentResources), } volumes := []corev1.Volume{ { @@ -1569,7 +1575,7 @@ func (r *GameServerReconciler) reconcileStatefulSet( }, }, }) - ss.Spec.Template.Spec.InitContainers = []corev1.Container{buildConfigInitContainer(r.ConfigInitImage, tmpl)} + ss.Spec.Template.Spec.InitContainers = []corev1.Container{buildConfigInitContainer(r.ConfigInitImage, tmpl, r.AgentResources)} } else { ss.Spec.Template.Spec.InitContainers = nil } @@ -1983,7 +1989,7 @@ func configInitImageOrDefault(image string) string { // When the template sets security.fsGroup, config-init makes each file // and directory it copied group-writable so a non-root game sharing that // fsGroup can update them. -func buildConfigInitContainer(image string, tmpl *gameplanev1alpha1.GameTemplate) corev1.Container { +func buildConfigInitContainer(image string, tmpl *gameplanev1alpha1.GameTemplate, defaults ...*corev1.ResourceRequirements) corev1.Container { image = configInitImageOrDefault(image) mountPath := effectiveMountPath(tmpl) cpCmd := "cp -RL " + configFilesStagingPath + "/* '" + mountPath + "/'" @@ -1996,9 +2002,10 @@ func buildConfigInitContainer(image string, tmpl *gameplanev1alpha1.GameTemplate } return corev1.Container{ - Name: "config-init", - Image: image, - Command: []string{"/bin/sh", "-c"}, + Name: "config-init", + Image: image, + Resources: auxiliaryResources(optionalResources(defaults), "25m", "64Mi"), + Command: []string{"/bin/sh", "-c"}, // -L dereferences the kubelet's per-key symlinks; the * glob // skips the ..data/.. dot-entries of the Secret mount. Args: []string{cpCmd}, @@ -2242,14 +2249,17 @@ func gamePodSecurityContext(tmpl *gameplanev1alpha1.GameTemplate) *corev1.PodSec func buildAgentContainer( gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate, ver *gameplanev1alpha1.GameVersion, fallbackImage, logLevel, pullPolicy string, + defaults ...*corev1.ResourceRequirements, ) corev1.Container { image := fallbackImage - res := corev1.ResourceRequirements{} + res := auxiliaryResources(optionalResources(defaults), "25m", "64Mi") if tmpl.Spec.Agent != nil { if tmpl.Spec.Agent.Image != "" { image = tmpl.Spec.Agent.Image } - res = tmpl.Spec.Agent.Resources + if tmpl.Spec.Agent.Resources.Requests != nil || tmpl.Spec.Agent.Resources.Limits != nil || len(tmpl.Spec.Agent.Resources.Claims) != 0 { + res = *tmpl.Spec.Agent.Resources.DeepCopy() + } } mountPath := effectiveMountPath(tmpl) nonRoot := true diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index b2c8a9edc..5415fbbf8 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -278,8 +278,9 @@ func (r *GameServerReconciler) reconcileSentinel( dep.Spec.Template.Spec.ServiceAccountName = sentinelSAName(gs) dep.Spec.Template.Spec.Containers = []corev1.Container{{ - Name: "sentinel", - Image: image, + Name: "sentinel", + Image: image, + Resources: auxiliaryResources(r.SentinelResources, "25m", "32Mi"), Env: []corev1.EnvVar{ {Name: "GAMESERVER_NAME", Value: gs.Name}, {Name: "GAMESERVER_NAMESPACE", Value: gs.Namespace}, diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index a950292b2..e5712d96b 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -392,6 +392,7 @@ func (r *GameServerReconciler) reconcileTunnel( dep.Spec.Template.Spec.Containers = []corev1.Container{{ Name: "tunnel", Image: image, + Resources: auxiliaryResources(r.TunnelResources, "25m", "64Mi"), Env: envVars, Ports: ports, VolumeMounts: volumeMounts, diff --git a/operator/internal/controller/restore_controller.go b/operator/internal/controller/restore_controller.go index 4cbf94e10..fe46011ef 100644 --- a/operator/internal/controller/restore_controller.go +++ b/operator/internal/controller/restore_controller.go @@ -36,6 +36,8 @@ type RestoreReconciler struct { // operator flag so air-gapped installs can point it at a private // registry mirror. Empty falls back to DefaultResticImage. ResticImage string + // BackupResources shares the backup worker resource configuration. + BackupResources *corev1.ResourceRequirements // JobBackoffLimit / JobActiveDeadlineSeconds bound the restic restore Job. // Set from operator flags; nil / zero fall back to // DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds. @@ -327,8 +329,9 @@ func (r *RestoreReconciler) buildRestorePodSpec( return corev1.PodSpec{ SecurityContext: buildBackupPodSecurityContext(tmpl), Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), + Name: "restic", + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), // --target / restores each snapshot entry at its original // absolute path. The companion backup runs `restic backup // /data`, so the snapshot tree is rooted at /data/...; with diff --git a/operator/specs.md b/operator/specs.md index c7aea8225..2db4f71cf 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -131,6 +131,20 @@ Nine CRD kinds under `gameplane.local/v1alpha1`: ## Reconcilers +### Auxiliary resource reservations + +Install flags `--agent-resources`, `--sentinel-resources`, `--tunnel-resources` +and `--backup-resources` accept complete JSON ResourceRequirements objects. +Absent flags use requests of 25m/64Mi for agent/config-init, 25m/32Mi for sentinel, +25m/64Mi for each tunnel provider, and 100m/128Mi for Restic backup/init/restore. +No new limits are set. Forget/prune retains 50m/128Mi and its existing 512Mi +memory limit when no override is configured. Explicit `{}` clears defaults. +Builders deep-copy resource maps. Template agent requests/limits, when declared, +replace install agent defaults; image-only overrides keep those defaults. +Game resources and GameServer placement retain their existing semantics. +Chart placement and optional PDBs are documented in +[`docs/production-placement.md`](../docs/production-placement.md). + Primary reconcilers register with the manager in `cmd/main.go` and handle CRD lifecycle: ### GameServerReconciler diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d60a26f14..4ea08f43e 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -392,10 +392,15 @@ EOF # Tests that exist in the suite but deliberately run in NO bucket. Every # entry needs a reason; `verify` fails on any unlisted stray so additions # here are a conscious, reviewed act. Currently empty. +bucket_management_placement() { cat <<'EOF' +TestManagementPlacementAndDisruption +EOF +} + unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components management-placement } list_bucket() { @@ -414,6 +419,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + management-placement) bucket_management_placement ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/management_placement_e2e_test.go b/test/e2e/management_placement_e2e_test.go new file mode 100644 index 000000000..c212de9ea --- /dev/null +++ b/test/e2e/management_placement_e2e_test.go @@ -0,0 +1,251 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// This bucket has no API logins and runs alone on the three-node management +// profile. It uses chart-rendered placement/PDBs with sleeping fixture containers +// to exercise the real scheduler and eviction controller without game downloads. +func TestManagementPlacementAndDisruption(t *testing.T) { + e := envInstance + nodes, err := e.K8s.CoreV1().Nodes().List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + var candidates []string + for _, node := range nodes.Items { + eligible := !node.Spec.Unschedulable + for _, taint := range node.Spec.Taints { + if taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute { + eligible = false + } + } + ready := false + for _, condition := range node.Status.Conditions { + ready = ready || (condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue) + } + if eligible && ready { + candidates = append(candidates, node.Name) + } + } + if len(candidates) < 2 { + t.Fatal("management placement requires two schedulable workers; use the management e2e profile") + } + candidates = candidates[:2] + ns := fmt.Sprintf("placement-%d", time.Now().UnixNano()) + if _, err := e.K8s.CoreV1().Namespaces().Create(t.Context(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = e.K8s.CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}) }) + poolLabel := "gameplane.local/" + ns + for _, name := range candidates { + patchNode := func(value any) error { + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"labels": map[string]any{poolLabel: value}}}) + if err != nil { + return err + } + _, err = e.K8s.CoreV1().Nodes().Patch(context.Background(), name, types.MergePatchType, patch, metav1.PatchOptions{}) + return err + } + if err := patchNode("management"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = patchNode(nil) }) + } + + chart, err := findRepoRel("charts/gameplane") + if err != nil { + t.Fatal(err) + } + selector := map[string]any{"app.kubernetes.io/name": "gameplane-web", "app.kubernetes.io/instance": ns} + values := map[string]any{ + "operator": map[string]any{"enabled": false}, + "api": map[string]any{"pdb": map[string]any{"enabled": true, "maxUnavailable": 1}}, + "web": map[string]any{ + "enabled": true, "replicas": 2, "nodeSelector": map[string]any{poolLabel: "management"}, + "pdb": map[string]any{"enabled": true, "maxUnavailable": 1}, + "affinity": map[string]any{"podAntiAffinity": map[string]any{"requiredDuringSchedulingIgnoredDuringExecution": []any{map[string]any{"topologyKey": "kubernetes.io/hostname", "labelSelector": map[string]any{"matchLabels": selector}}}}}, + "topologySpreadConstraints": []any{map[string]any{"maxSkew": 1, "topologyKey": "kubernetes.io/hostname", "whenUnsatisfiable": "DoNotSchedule", "labelSelector": map[string]any{"matchLabels": selector}}}, + }, + } + data, err := json.Marshal(values) + if err != nil { + t.Fatal(err) + } + valuesPath := filepath.Join(t.TempDir(), "values.json") + if err := os.WriteFile(valuesPath, data, 0o600); err != nil { + t.Fatal(err) + } + rendered, err := exec.CommandContext(t.Context(), "helm", "template", ns, chart, "--namespace", ns, "--values", valuesPath, "--show-only", "templates/web.yaml", "--show-only", "templates/poddisruptionbudgets.yaml").CombinedOutput() + if err != nil { + t.Fatalf("render placement chart: %v: %s", err, rendered) + } + fixture := corev1.Container{Name: "fixture", Image: "busybox:1.38.0", Command: []string{"sleep", "3600"}} + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 4096) + for { + var object unstructured.Unstructured + if err := decoder.Decode(&object); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + switch object.GetKind() { + case "Deployment": + var deployment appsv1.Deployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.Object, &deployment); err != nil { + t.Fatal(err) + } + deployment.Spec.Template.Spec.Containers = []corev1.Container{fixture} + if _, err := e.K8s.AppsV1().Deployments(ns).Create(t.Context(), &deployment, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + case "PodDisruptionBudget": + var pdb policyv1.PodDisruptionBudget + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.Object, &pdb); err != nil { + t.Fatal(err) + } + if _, err := e.K8s.PolicyV1().PodDisruptionBudgets(ns).Create(t.Context(), &pdb, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + } + webPods := func() ([]corev1.Pod, error) { + list, err := e.K8s.CoreV1().Pods(ns).List(t.Context(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gameplane-web"}) + if err != nil { + return nil, err + } + return list.Items, nil + } + waitReady := func() { + e.Eventually(t, 3*time.Minute, func() (bool, string) { + pods, err := webPods() + if err != nil || len(pods) != 2 { + return false, fmt.Sprintf("pods=%d error=%v", len(pods), err) + } + seen := map[string]bool{} + for _, pod := range pods { + ready, err := e.PodIsReady(t.Context(), ns, pod.Name) + if err != nil || !ready || !pod.DeletionTimestamp.IsZero() { + return false, "web fixture not ready" + } + if pod.Spec.NodeName != candidates[0] && pod.Spec.NodeName != candidates[1] { + t.Fatalf("pod escaped management selector to %s", pod.Spec.NodeName) + } + seen[pod.Spec.NodeName] = true + } + return len(seen) == 2, "web replicas were not spread across two workers" + }) + } + waitReady() + setSelector := func(value string) { + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{"nodeSelector": map[string]any{poolLabel: value}}}}}) + if err != nil { + t.Fatal(err) + } + if _, err := e.K8s.AppsV1().Deployments(ns).Patch(t.Context(), "gameplane-web", types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + } + setSelector("absent-pool") + e.Eventually(t, time.Minute, func() (bool, string) { + pods, err := webPods() + if err != nil { + return false, err.Error() + } + for _, pod := range pods { + for _, condition := range pod.Status.Conditions { + if pod.Spec.NodeSelector[poolLabel] == "absent-pool" && condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && condition.Reason == "Unschedulable" { + return true, "" + } + } + } + return false, "missing unschedulable replacement" + }) + setSelector("management") + waitReady() + cordon := func(unschedulable bool) error { + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"unschedulable": unschedulable}}) + if err != nil { + return err + } + _, err = e.K8s.CoreV1().Nodes().Patch(context.Background(), candidates[0], types.MergePatchType, patch, metav1.PatchOptions{}) + return err + } + if err := cordon(true); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cordon(false) }) + waitBudget := func(name string, allowed int32) { + e.Eventually(t, time.Minute, func() (bool, string) { + pdb, err := e.K8s.PolicyV1().PodDisruptionBudgets(ns).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return pdb.Status.ObservedGeneration >= pdb.Generation && pdb.Status.DisruptionsAllowed == allowed, fmt.Sprintf("budget status: %+v", pdb.Status) + }) + } + waitBudget("gameplane-web", 1) + pods, err := webPods() + if err != nil { + t.Fatal(err) + } + var remaining string + for _, pod := range pods { + if pod.Spec.NodeName == candidates[0] { + if err := e.K8s.CoreV1().Pods(ns).EvictV1(t.Context(), &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Name: pod.Name, Namespace: ns}}); err != nil { + t.Fatal(err) + } + } else { + remaining = pod.Name + } + } + waitBudget("gameplane-web", 0) + if err := e.K8s.CoreV1().Pods(ns).EvictV1(t.Context(), &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Name: remaining, Namespace: ns}}); !apierrors.IsTooManyRequests(err) { + t.Fatalf("second eviction bypassed web availability budget: %v", err) + } + if ready, err := e.PodIsReady(t.Context(), ns, remaining); err != nil || !ready { + t.Fatalf("remaining web replica lost readiness: %v", err) + } + if err := cordon(false); err != nil { + t.Fatal(err) + } + waitReady() + // The chart's API budget must permit the only API pod to be evicted. This + // isolated labeled fixture tests the PDB without stopping the suite's API. + apiLabels := map[string]string{"app.kubernetes.io/name": "gameplane-api", "app.kubernetes.io/instance": ns} + one := int32(1) + apiDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "gameplane-api", Namespace: ns}, Spec: appsv1.DeploymentSpec{Replicas: &one, Selector: &metav1.LabelSelector{MatchLabels: apiLabels}, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: apiLabels}, Spec: corev1.PodSpec{Containers: []corev1.Container{fixture}}}}} + if _, err := e.K8s.AppsV1().Deployments(ns).Create(t.Context(), apiDeployment, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + waitBudget("gameplane-api", 1) + apiPods, err := e.K8s.CoreV1().Pods(ns).List(t.Context(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gameplane-api"}) + if err != nil || len(apiPods.Items) != 1 { + t.Fatalf("API fixture pods: %v", err) + } + if err := e.K8s.CoreV1().Pods(ns).EvictV1(t.Context(), &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Name: apiPods.Items[0].Name, Namespace: ns}}); err != nil { + t.Fatalf("API singleton PDB blocked drain: %v", err) + } +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index abf7cda91..75fff7eb7 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -40,6 +40,17 @@ are removed through test cleanup. CI sets `GAMEPLANE_E2E_REQUIRE_GATEWAY=1`, so missing second cluster fails instead of skipping the scenario. The multicluster Go invocation has a 45-minute deadline within the existing 60-minute job budget. +### Management placement and disruption acceptance + +The `management-placement` bucket uses the three-node `management` Kind profile +and performs no API login. `TestManagementPlacementAndDisruption` applies the +chart's web placement and API/web PDBs in an isolated namespace, with sleeping +fixture containers. It verifies real scheduling across labeled workers, +Unschedulable status for an impossible selector, one-at-a-time eviction during +a cordon, readiness of the surviving replica, and singleton API eviction. Node +labels and cordons are restored in cleanup. This validates scheduler/PDB behavior, +not application HA or storage migration. + ## Non-goals / boundaries - Does **not** replace the unit or envtest tiers for any other module — it exercises integration behavior, not the exhaustive branch coverage those tiers own (this module carries no `.testcoverage.yml` coverage gate). From 4ae4173230440693cb42f3815a995fa340ba861c Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 11:25:58 -0400 Subject: [PATCH 02/17] feat(network): add template sentinel isolation and strict policy profile Signed-off-by: Baylem --- .github/workflows/ci.yaml | 10 +- .../crds/gameplane.local_backups.yaml | 2 +- .../crds/gameplane.local_backupschedules.yaml | 2 +- .../crds/gameplane.local_clusters.yaml | 2 +- .../crds/gameplane.local_gameservers.yaml | 2 +- .../crds/gameplane.local_gametemplates.yaml | 8 +- .../crds/gameplane.local_modules.yaml | 2 +- .../crds/gameplane.local_modulesources.yaml | 2 +- .../crds/gameplane.local_networkcaptures.yaml | 2 +- .../crds/gameplane.local_restores.yaml | 2 +- .../gameplane.local_backups.yaml | 2 +- .../gameplane.local_backupschedules.yaml | 2 +- .../gameplane.local_clusters.yaml | 2 +- .../gameplane.local_gameservers.yaml | 2 +- .../gameplane.local_gametemplates.yaml | 8 +- .../gameplane.local_modules.yaml | 2 +- .../gameplane.local_modulesources.yaml | 2 +- .../gameplane.local_networkcaptures.yaml | 2 +- .../gameplane.local_restores.yaml | 2 +- .../gameplane/templates/networkpolicies.yaml | 20 +- charts/gameplane/templates/validate.yaml | 28 +++ charts/gameplane/values.yaml | 3 + charts/tests/check_network_isolation.py | 49 +++++ deploy/kind/e2e.sh | 37 +++- docs/module-authoring.md | 9 + docs/networking.md | 60 ++++++ operator/api/v1alpha1/gametemplate_types.go | 6 + .../crd/gameplane.local_gametemplates.yaml | 6 + .../controller/gameserver_controller.go | 11 +- .../controller/gameserver_sentinel.go | 3 + .../internal/controller/sentinel_isolation.go | 66 ++++++ .../controller/sentinel_isolation_test.go | 106 ++++++++++ operator/specs.md | 1 + test/e2e/buckets.sh | 5 +- test/e2e/network_isolation_e2e_test.go | 193 ++++++++++++++++++ test/e2e/specs.md | 7 + 36 files changed, 639 insertions(+), 29 deletions(-) create mode 100644 charts/tests/check_network_isolation.py create mode 100644 operator/internal/controller/sentinel_isolation.go create mode 100644 operator/internal/controller/sentinel_isolation_test.go create mode 100644 test/e2e/network_isolation_e2e_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1c9aecdd..402afbe08 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1115,6 +1115,8 @@ jobs: run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders run: python3 charts/tests/check_standalone.py + - name: strict network profile and sentinel policy selectors + run: python3 charts/tests/check_network_isolation.py - name: sanity-check rendered manifests run: | grep -q 'kind: Deployment' /tmp/rendered.yaml @@ -1664,9 +1666,13 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, network-isolation] arch: [amd64] include: + - bucket: network-isolation + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: operator parallel: 8 test_timeout: 20m @@ -1718,6 +1724,8 @@ jobs: with: arch: ${{ matrix.arch }} - name: bring up cluster + helm install + env: + GAMEPLANE_E2E_CNI: ${{ matrix.bucket == 'network-isolation' && 'cilium' || 'kindnet' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e - name: run go e2e (${{ matrix.bucket }}) run: | diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index d902ea995..e75f625df 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index d85406090..d57e9005b 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 4d6aadcbb..49f176cbb 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index 95178e56e..e98f8ebe8 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index e5e3e1e3b..3d107f6fa 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: @@ -2114,6 +2114,12 @@ spec: minimum: 0 type: integer type: object + sentinelIsolation: + description: |- + SentinelIsolation restricts a wake sentinel to its own server's advertised + ports. Templates opt in after module connectivity validation; absent or + false retains the legacy namespace-wide sentinel allowances. + type: boolean storage: description: Storage describes the default persistent storage layout. properties: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 5ade28cee..fd54ac3e8 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 6c5dcbf36..6605d3fbc 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index cc113e7e8..39a6dd873 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index b77cd5c25..13d3185a4 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index d902ea995..e75f625df 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index d85406090..d57e9005b 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 4d6aadcbb..49f176cbb 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index 95178e56e..e98f8ebe8 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index e5e3e1e3b..3d107f6fa 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: @@ -2114,6 +2114,12 @@ spec: minimum: 0 type: integer type: object + sentinelIsolation: + description: |- + SentinelIsolation restricts a wake sentinel to its own server's advertised + ports. Templates opt in after module connectivity validation; absent or + false retains the legacy namespace-wide sentinel allowances. + type: boolean storage: description: Storage describes the default persistent storage layout. properties: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 5ade28cee..fd54ac3e8 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 6c5dcbf36..6605d3fbc 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index cc113e7e8..39a6dd873 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index b77cd5c25..13d3185a4 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 0cf9b2a9fa0d548476df615efb31f6c362eb68abeb3e85ba34798d7a6c431b23 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/templates/networkpolicies.yaml b/charts/gameplane/templates/networkpolicies.yaml index 81f963f4d..612fc5071 100644 --- a/charts/gameplane/templates/networkpolicies.yaml +++ b/charts/gameplane/templates/networkpolicies.yaml @@ -331,8 +331,9 @@ spec: - { protocol: TCP, port: 443 } - { protocol: TCP, port: 6443 } --- -# Allow the sentinel to egress to game pods on advertised ports for the proxy path. -# This is the INGRESS half only, on the game pod's own selector. +# Legacy sentinel proxy allowance, ingress half. This grants ALL ports between +# non-isolated sentinels and games. NetworkPolicies are additive; per-server +# player ingress does not narrow it. Opted-in templates use operator policies. # NetworkPolicy requires both sides to agree: the destination (this policy) # admits the sentinel, but default-deny-egress's podSelector: {} still # governs the sentinel as a SOURCE and only opens DNS. allow-sentinel-to-game-egress @@ -349,14 +350,17 @@ spec: podSelector: matchLabels: app.kubernetes.io/name: gameplane-game + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } policyTypes: [Ingress] ingress: - from: - podSelector: matchLabels: app.kubernetes.io/name: gameplane-waker - # All ports; the per-server ingress policy covers which ports are - # actually advertised for each server. + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } + # Intentionally unrestricted for templates that have not opted in. --- # The Egress half of allow-sentinel-to-game (see its comment above): admits # the sentinel, as a traffic SOURCE, to reach game pods. Selects the @@ -374,15 +378,17 @@ spec: podSelector: matchLabels: app.kubernetes.io/name: gameplane-waker + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } policyTypes: [Egress] egress: - to: - podSelector: matchLabels: app.kubernetes.io/name: gameplane-game - # All ports; allow-sentinel-to-game above (the matching ingress half) - # and the per-server ingress policy cover which ports are actually - # advertised for each server. + matchExpressions: + - { key: gameplane.local/sentinel-isolation, operator: NotIn, values: ["true"] } + # Intentionally unrestricted for templates that have not opted in. {{- end }} {{- end }} diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index 79b5e17b6..bf4a4580f 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,3 +1,31 @@ +{{- $network := .Values.networkPolicies | default dict }} +{{- if $network.strict }} +{{- if not $network.enabled }} +{{- fail "networkPolicies.strict requires networkPolicies.enabled=true" }} +{{- end }} +{{- range $key := list "kubeletCIDRs" "apiServerCIDRs" "probePorts" }} +{{- $entries := index $network $key }} +{{- if or (not (kindIs "slice" $entries)) (eq (len $entries) 0) }} +{{- fail (printf "networkPolicies.strict requires a non-empty networkPolicies.%s list" $key) }} +{{- end }} +{{- end }} +{{- $octet := "(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])" }} +{{- $ipv4 := printf "^%s(\\.%s){3}/([0-9]|[12][0-9]|3[0-2])$" $octet $octet }} +{{- $h := "[0-9A-Fa-f]{1,4}" }} +{{- $ipv6 := printf "^((%s:){7}%s|(%s:){1,7}:|(%s:){1,6}:%s|(%s:){1,5}(:%s){1,2}|(%s:){1,4}(:%s){1,3}|(%s:){1,3}(:%s){1,4}|(%s:){1,2}(:%s){1,5}|%s:((:%s){1,6})|:((:%s){1,7}|:))/([0-9]|[1-9][0-9]|1[01][0-9]|12[0-8])$" $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h $h }} +{{- range $key := list "kubeletCIDRs" "apiServerCIDRs" }} +{{- range (index $network $key) }} +{{- if or (not (kindIs "string" .)) (not (or (regexMatch $ipv4 (toString .)) (regexMatch $ipv6 (toString .)))) }} +{{- fail (printf "networkPolicies.%s entries must be explicit IPv4/IPv6 CIDRs" $key) }} +{{- end }} +{{- end }} +{{- end }} +{{- range $network.probePorts }} +{{- if or (not (has (.protocol | default "") (list "TCP" "UDP" "SCTP"))) (not (regexMatch "^[0-9]+$" (toString .port))) (lt (int .port) 1) (gt (int .port) 65535) }} +{{- fail "networkPolicies.probePorts requires protocol TCP/UDP/SCTP and an integer port from 1 to 65535" }} +{{- end }} +{{- end }} +{{- end }} {{- if .Values.api.standalone }} {{- if ne (include "gameplane.apiEnabled" .) "true" }} {{- fail "api.standalone requires api.enabled=true" }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 66d95af0b..1c400384a 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -493,6 +493,9 @@ gamesNamespace: gameplane-games networkPolicies: enabled: true + # Fail closed unless actual kubelet source CIDRs/probe ports and apiserver + # endpoint CIDRs are supplied. Defaults retain the legacy broad allowances. + strict: false # CIDRs that hold kubelet source IPs for liveness/readiness probes. # If unset, the chart falls back to RFC1918 + link-local ranges, # which cover most clusters without opening the public internet. diff --git a/charts/tests/check_network_isolation.py b/charts/tests/check_network_isolation.py new file mode 100644 index 000000000..7c180d143 --- /dev/null +++ b/charts/tests/check_network_isolation.py @@ -0,0 +1,49 @@ +"""CI-only rendered effective policy tuple and strict-profile regressions.""" +from check_standalone import render, one + + +def main(): + values = {"networkPolicies": {"strict": True, + "kubeletCIDRs": ["192.0.2.10/32"], + "probePorts": [{"protocol": "TCP", "port": 8090}], + "apiServerCIDRs": ["198.51.100.20/32"]}} + objects = render(values) + probe = one(objects, "NetworkPolicy", "allow-kubelet-probes")["spec"]["ingress"] + assert probe == [{"from": [{"ipBlock": {"cidr": "192.0.2.10/32"}}], + "ports": [{"protocol": "TCP", "port": 8090}]}] + for name in ("allow-agent-to-apiserver", "allow-sentinel-to-apiserver"): + egress = one(objects, "NetworkPolicy", name)["spec"]["egress"] + assert egress == [{"to": [{"ipBlock": {"cidr": "198.51.100.20/32"}}], + "ports": [{"protocol": "TCP", "port": 443}, {"protocol": "TCP", "port": 6443}]}] + excluded = [{"key": "gameplane.local/sentinel-isolation", "operator": "NotIn", "values": ["true"]}] + for name, direction, peer in (("allow-sentinel-to-game", "ingress", "from"), + ("allow-sentinel-to-game-egress", "egress", "to")): + policy = one(objects, "NetworkPolicy", name)["spec"] + assert policy["podSelector"]["matchExpressions"] == excluded + assert policy[direction][0][peer][0]["podSelector"]["matchExpressions"] == excluded + import copy + for key in ("kubeletCIDRs", "apiServerCIDRs", "probePorts"): + invalid = copy.deepcopy(values) + invalid["networkPolicies"][key] = [] + render(invalid, failure="requires a non-empty networkPolicies." + key) + for cidr in ("", "10.0.0.1", "999.0.0.1/32", "10.0.0.1/33", "2001:::1/64", "2001:db8/64", "::/129"): + invalid = copy.deepcopy(values) + invalid["networkPolicies"]["apiServerCIDRs"] = [cidr] + render(invalid, failure="must be explicit IPv4/IPv6 CIDRs") + for cidr in ("::/0", "::1/128", "2001:db8::/64", "2001:db8:1:2:3:4:5:6/128"): + valid = copy.deepcopy(values) + valid["networkPolicies"]["apiServerCIDRs"] = [cidr] + render(valid) + for port in (0, 65536, 1.5, "all"): + invalid = copy.deepcopy(values) + invalid["networkPolicies"]["probePorts"][0]["port"] = port + render(invalid, failure="integer port") + render({"networkPolicies": {"strict": True, "enabled": False}}, failure="requires networkPolicies.enabled=true") + # Legacy/default installs still render the documented broad fallback. + fallback = one(render(), "NetworkPolicy", "allow-kubelet-probes")["spec"]["ingress"][0] + assert "ports" not in fallback and {p["ipBlock"]["cidr"] for p in fallback["from"]} == { + "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "169.254.0.0/16"} + + +if __name__ == "__main__": + main() diff --git a/deploy/kind/e2e.sh b/deploy/kind/e2e.sh index 3c2f7b12f..01c169798 100755 --- a/deploy/kind/e2e.sh +++ b/deploy/kind/e2e.sh @@ -27,6 +27,11 @@ ACTION="${1:-up}" CLUSTER="${2:-gameplane-e2e}" TAG="${3:-${GAMEPLANE_E2E_TAG:-e2e}}" PROFILE="${4:-default}" +CNI="${GAMEPLANE_E2E_CNI:-kindnet}" +case "${CNI}" in +kindnet | cilium) ;; +*) echo "unsupported E2E CNI: ${CNI}" >&2; exit 2 ;; +esac case "${PROFILE}" in default | postgres) ;; *) @@ -189,15 +194,33 @@ up) kind delete cluster --name "${CLUSTER}" || true fi echo "creating kind cluster ${CLUSTER} (single-node)" - kind create cluster --name "${CLUSTER}" --wait 90s --config - </dev/null + if [ "${CNI}" = cilium ]; then + # Pin the enforcing CNI; kindnet does not implement NetworkPolicy. + # https://docs.cilium.io/en/stable/installation/kind/ + helm upgrade --install cilium cilium --repo https://helm.cilium.io/ \ + --version 1.20.3 --namespace kube-system \ + --set image.pullPolicy=IfNotPresent --set ipam.mode=kubernetes \ + --set policyCIDRMatchMode=nodes \ + --set operator.replicas=1 --wait --timeout 5m + kubectl wait nodes --all --for=condition=Ready --timeout=120s + fi + install_metallb apply_metallb_pools @@ -358,6 +381,17 @@ EOF deployment/gameplane-test-fakeoidc --timeout=120s DB_SETS=() + NETWORK_SETS=() + if [ "${CNI}" = cilium ]; then + # Single-node IPv4 fixture: use actual endpoint/source addresses. + node_ip="$(kubectl get nodes -o jsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')" + api_ip="$(kubectl get endpoints kubernetes -n default -o jsonpath='{.subsets[0].addresses[0].ip}')" + NETWORK_SETS=(--set networkPolicies.strict=true + --set-string "networkPolicies.kubeletCIDRs[0]=${node_ip}/32" + --set-string "networkPolicies.apiServerCIDRs[0]=${api_ip}/32" + --set networkPolicies.probePorts[0].protocol=TCP + --set networkPolicies.probePorts[0].port=25566) + fi if [ "${PROFILE}" = postgres ]; then dsn="$(kubectl get secret gameplane-postgres --namespace gameplane-system \ -o jsonpath='{.data.dsn}' | base64 -d)" @@ -403,6 +437,7 @@ EOF --set "defaultModuleSource.enabled=false" \ --set "api.telemetry.receiver.enabled=true" \ ${DB_SETS[@]+"${DB_SETS[@]}"} \ + ${NETWORK_SETS[@]+"${NETWORK_SETS[@]}"} \ --wait --timeout 5m echo diff --git a/docs/module-authoring.md b/docs/module-authoring.md index 1141965bb..55b2436aa 100644 --- a/docs/module-authoring.md +++ b/docs/module-authoring.md @@ -12,6 +12,15 @@ materializes into an in-cluster `GameTemplate`. ## Quickstart: Authoring with `gp-module` CLI & Web Builder +For a validated wake proxy, add `sentinelIsolation: true` under the template's +`spec`. The operator permits that server's sentinel to reach only its own +advertised container ports, preserving each port's TCP/UDP/SCTP protocol. +Leave management ports unadvertised. A template with no advertised ports grants +no sentinel access. Omission preserves legacy behavior; adoption in the +GameplanePanel/module repository is separate from the operator upgrade. +Test player joins, status queries and sleep/wake cycles with an enforcing CNI +before publishing the opt-in. See [network policy migration](networking.md#networkpolicy-isolation-and-migration). + Gameplane provides a unified toolkit for creating, verifying, previewing, and packaging modules: - **Command-Line Interface (`gp-module`)**: First-class Go CLI with Makefile targets. diff --git a/docs/networking.md b/docs/networking.md index a8415fb72..fd3259012 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -2,6 +2,66 @@ Gameplane exposes game servers through Kubernetes Services, with optional integration with your cluster's load-balancer address manager to assign specific public addresses or address pools to each server. +## NetworkPolicy isolation and migration + +NetworkPolicies are additive. An allowance in one policy cannot be narrowed by +another policy. Enforcement also requires a CNI that implements NetworkPolicy. + +Modules may set `GameTemplate.spec.sentinelIsolation: true`. The operator then +labels the game and sentinel pod templates and creates two owned policies: +`-sentinel-ingress` and `-sentinel-egress`. They allow a sentinel +to reach only its own server's advertised container ports, with the declared +protocol. Zero advertised ports creates no allowance. Management ports must +remain `advertise: false`. The legacy chart policies exclude opted-in sources +and destinations. Templates without the field retain the existing broad +sentinel-to-game allowance; module maintainers can adopt it after validating +their proxy traffic. + +Upgrade the CRDs, operator and chart together before opting in a template. +Existing game pods roll out to acquire the isolation label; StatefulSet +selectors stay unchanged. Do not claim isolation until both game and sentinel +pods have the label and the per-server policies exist. Other administrator-added +policies can still grant wider access. Disabling chart NetworkPolicies or using +a non-enforcing CNI does not provide this boundary. + +Probe and API-server narrowing is a separate installation option: + +```yaml +networkPolicies: + enabled: true + strict: true + kubeletCIDRs: ["192.0.2.10/32"] # Replace with observed kubelet sources. + probePorts: + - {protocol: TCP, port: 25565} # Replace with all configured probe ports. + apiServerCIDRs: ["198.51.100.20/32"] # Replace with actual API endpoints. +``` + +Strict mode rejects missing/empty lists, malformed CIDRs and invalid probe +ports. It never supplies broad fallback ranges. API egress is limited to the +listed addresses on TCP 443/6443. Choose addresses based on your CNI's policy +evaluation point: service DNAT or source SNAT can change which address it sees. +Include every control-plane endpoint and kubelet source that can host a server; +update these lists when nodes or endpoints change. CNI host-traffic exceptions +may still apply. + +On Cilium, CIDR rules exclude node identities by default. For a self-hosted +API server on a node, configure `policyCIDRMatchMode=nodes` before using these +plain NetworkPolicy endpoint CIDRs, or provide equivalent Cilium-specific +API-server policy. The CI fixture uses nodes mode. See +[Cilium CIDR policy semantics](https://docs.cilium.io/en/stable/security/policy/layer3/#selecting-pods-or-nodes-with-cidr-ipblock) +and validate kubelet source handling for your Cilium configuration as well. + +Before enabling strict mode, inventory the actual endpoints and configured +game probes, observe their traffic in the CNI, then validate game readiness, +agent heartbeats, wake requests and API/operator agent access in a staging +namespace. The default profile retains RFC1918/link-local API egress and +unrestricted probe ports for compatibility; it does not isolate those paths. +Narrowing sentinel traffic alone cannot remove those separate grants. + +The dedicated CI `network-isolation` bucket installs pinned Cilium, exercises +real allowed/denied connections, and upgrades existing legacy templates. +The default kindnet fixture only verifies policy shape. + ## Address Pool Overview When a GameServer is exposed as a LoadBalancer (`spec.networking.expose: LoadBalancer`), it can request a specific external address or address pool. The operator translates this request to your cluster's load-balancer flavor so the same CRD works whether you run MetalLB, Cilium, or no address manager at all. diff --git a/operator/api/v1alpha1/gametemplate_types.go b/operator/api/v1alpha1/gametemplate_types.go index bea234aea..38a906125 100644 --- a/operator/api/v1alpha1/gametemplate_types.go +++ b/operator/api/v1alpha1/gametemplate_types.go @@ -89,6 +89,12 @@ type GameTemplateSpec struct { // +optional Ports []GamePort `json:"ports,omitempty"` + // SentinelIsolation restricts a wake sentinel to its own server's advertised + // ports. Templates opt in after module connectivity validation; absent or + // false retains the legacy namespace-wide sentinel allowances. + // +optional + SentinelIsolation bool `json:"sentinelIsolation,omitempty"` + // Storage describes the default persistent storage layout. // +optional Storage GameStorageSpec `json:"storage,omitempty"` diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index 13d8a1fa5..0b9cf37a0 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -2113,6 +2113,12 @@ spec: minimum: 0 type: integer type: object + sentinelIsolation: + description: |- + SentinelIsolation restricts a wake sentinel to its own server's advertised + ports. Templates opt in after module connectivity validation; absent or + false retains the legacy namespace-wide sentinel allowances. + type: boolean storage: description: Storage describes the default persistent storage layout. properties: diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 67e8be4e3..028beb0d1 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -298,6 +298,9 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "reconcile ingress NetworkPolicy") return ctrl.Result{}, err } + if err := r.reconcileSentinelIsolation(ctx, &gs, &tmpl); err != nil { + return ctrl.Result{}, err + } if err := r.reconcileAgentTLS(ctx, &gs); err != nil { logger.Error(err, "reconcile agent TLS") return ctrl.Result{}, err @@ -1494,7 +1497,13 @@ func (r *GameServerReconciler) reconcileStatefulSet( ss.Spec.Replicas = &actualReplicas ss.Spec.ServiceName = gs.Name ss.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} - ss.Spec.Template.Labels = labels + ss.Spec.Template.Labels = make(map[string]string, len(labels)+1) + for key, value := range labels { + ss.Spec.Template.Labels[key] = value + } + if tmpl.Spec.SentinelIsolation { + ss.Spec.Template.Labels[sentinelIsolationLabel] = "true" + } // Stamp (or clear) the config fingerprint without touching // annotations other actors may have set on the pod template. ann := ss.Spec.Template.Annotations diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index b2c8a9edc..694c9710b 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -230,6 +230,9 @@ func (r *GameServerReconciler) reconcileSentinel( sentinelLabel: sentinelValue, "app.kubernetes.io/instance": gs.Name, } + if tmpl.Spec.SentinelIsolation { + dep.Spec.Template.Labels[sentinelIsolationLabel] = "true" + } // Container ports: advertised ports only, mirroring buildGameContainer. ports := make([]corev1.ContainerPort, 0) diff --git a/operator/internal/controller/sentinel_isolation.go b/operator/internal/controller/sentinel_isolation.go new file mode 100644 index 000000000..6cd2969bd --- /dev/null +++ b/operator/internal/controller/sentinel_isolation.go @@ -0,0 +1,66 @@ +package controller + +import ( + "context" + "fmt" + + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const sentinelIsolationLabel = "gameplane.local/sentinel-isolation" + +// The chart's legacy policies exclude opted-in pods on both sides. Policies +// are additive: these per-server grants cannot subtract a namespace-wide grant. +func (r *GameServerReconciler) reconcileSentinelIsolation(ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate) error { + ports := networkPolicyPortsFromTemplate(tmpl) + for _, ingress := range []bool{true, false} { + suffix, target, peer := "egress", "gameplane-waker", "gameplane-game" + if ingress { + suffix, target, peer = "ingress", "gameplane-game", "gameplane-waker" + } + np := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-sentinel-" + suffix, Namespace: gs.Namespace}} + if !tmpl.Spec.SentinelIsolation || len(ports) == 0 { + if err := r.Get(ctx, client.ObjectKeyFromObject(np), np); err != nil { + if client.IgnoreNotFound(err) != nil { + return err + } + continue + } + if metav1.IsControlledBy(np, gs) { + uid := np.UID + if err := client.IgnoreNotFound(r.Delete(ctx, np, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}})); err != nil { + return err + } + } + continue + } + _, err := controllerutil.CreateOrUpdate(ctx, r.Client, np, func() error { + if np.ResourceVersion != "" && !metav1.IsControlledBy(np, gs) { + return fmt.Errorf("sentinel policy %s is not owned by this GameServer", np.Name) + } + np.Spec = networkingv1.NetworkPolicySpec{PodSelector: metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/name": target, "app.kubernetes.io/instance": gs.Name, + }}} + peers := []networkingv1.NetworkPolicyPeer{{PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "app.kubernetes.io/name": peer, "app.kubernetes.io/instance": gs.Name, + }}}} + if ingress { + np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeIngress} + np.Spec.Ingress = []networkingv1.NetworkPolicyIngressRule{{From: peers, Ports: ports}} + } else { + np.Spec.PolicyTypes = []networkingv1.PolicyType{networkingv1.PolicyTypeEgress} + np.Spec.Egress = []networkingv1.NetworkPolicyEgressRule{{To: peers, Ports: ports}} + } + return controllerutil.SetControllerReference(gs, np, r.Scheme) + }) + if err != nil { + return err + } + } + return nil +} diff --git a/operator/internal/controller/sentinel_isolation_test.go b/operator/internal/controller/sentinel_isolation_test.go new file mode 100644 index 000000000..1db7f3587 --- /dev/null +++ b/operator/internal/controller/sentinel_isolation_test.go @@ -0,0 +1,106 @@ +package controller + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestSentinelIsolationPolicies(t *testing.T) { + scheme := runtime.NewScheme() + if err := networkingv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := gameplanev1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &GameServerReconciler{Client: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + tmpl.Spec.Ports = append(tmpl.Spec.Ports, + gameplanev1alpha1.GamePort{Name: "query", ContainerPort: 19132, Protocol: corev1.ProtocolUDP, Advertise: true}, + gameplanev1alpha1.GamePort{Name: "management", ContainerPort: 25575, Protocol: corev1.ProtocolTCP}) + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, target, peer string + ingress bool + }{ + {"alpha-sentinel-ingress", "gameplane-game", "gameplane-waker", true}, + {"alpha-sentinel-egress", "gameplane-waker", "gameplane-game", false}, + } { + var policy networkingv1.NetworkPolicy + if err := c.Get(t.Context(), client.ObjectKey{Namespace: "games", Name: tc.name}, &policy); err != nil { + t.Fatal(err) + } + if !metav1.IsControlledBy(&policy, gs) { + t.Fatal("missing server ownership") + } + selector := policy.Spec.PodSelector.MatchLabels + if selector["app.kubernetes.io/name"] != tc.target || selector["app.kubernetes.io/instance"] != "alpha" { + t.Fatalf("wrong target: %v", selector) + } + var peers []networkingv1.NetworkPolicyPeer + var ports []networkingv1.NetworkPolicyPort + if tc.ingress { + if len(policy.Spec.Ingress) != 1 || len(policy.Spec.Egress) != 0 { + t.Fatal("wrong ingress policy shape") + } + peers, ports = policy.Spec.Ingress[0].From, policy.Spec.Ingress[0].Ports + } else { + if len(policy.Spec.Egress) != 1 || len(policy.Spec.Ingress) != 0 { + t.Fatal("wrong egress policy shape") + } + peers, ports = policy.Spec.Egress[0].To, policy.Spec.Egress[0].Ports + } + if len(peers) != 1 || peers[0].NamespaceSelector != nil || peers[0].IPBlock != nil || peers[0].PodSelector.MatchLabels["app.kubernetes.io/name"] != tc.peer || peers[0].PodSelector.MatchLabels["app.kubernetes.io/instance"] != "alpha" { + t.Fatalf("cross-server peer grant: %+v", peers) + } + if len(ports) != 2 || ports[0].Port.IntVal != 25565 || *ports[0].Protocol != corev1.ProtocolTCP || ports[1].Port.IntVal != 19132 || *ports[1].Protocol != corev1.ProtocolUDP { + t.Fatalf("wrong protocol/port tuples: %+v", ports) + } + } + // Zero advertised ports removes both grants instead of opening all ports. + tmpl.Spec.Ports = nil + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 0 { + t.Fatalf("zero-port grants remain: %v %+v", err, policies.Items) + } + // Absent/default false preserves legacy behavior and creates no policies. + tmpl = singlePortTemplate(true, "minecraft") + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 0 { + t.Fatalf("legacy template changed: %v %+v", err, policies.Items) + } +} + +func TestSentinelIsolationDoesNotDeleteForeignPolicy(t *testing.T) { + scheme := runtime.NewScheme() + _ = networkingv1.AddToScheme(scheme) + _ = gameplanev1alpha1.AddToScheme(scheme) + foreign := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "alpha-sentinel-ingress", Namespace: "games"}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(foreign).Build() + r := &GameServerReconciler{Client: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + if err := r.reconcileSentinelIsolation(t.Context(), gs, singlePortTemplate(false, "none")); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(foreign), foreign); err != nil { + t.Fatal("deleted foreign policy", err) + } +} diff --git a/operator/specs.md b/operator/specs.md index c7aea8225..61ea99618 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -149,6 +149,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Graceful stop: if template has Lifecycle.Stop sequence, orchestrate via agent before scaling down. - Node affinity: honor spec.nodeSelector and spec.affinity preferences. - Ingress NetworkPolicy: enforce per-template advertised ports, admit CIDRs from `--game-ingress-from-cidr`. + - Sentinel NetworkPolicies: `GameTemplate.spec.sentinelIsolation` defaults false. When true, reconcile owned `-sentinel-ingress` and `-sentinel-egress` policies joining only the same namespace/server instance on advertised protocol/port tuples. Zero advertised ports removes owned grants. Set `gameplane.local/sentinel-isolation=true` on both pod templates, without changing immutable workload selectors. The matching chart excludes opted-in sources and destinations from legacy grants; other policies remain additive. - Fixed-name child cleanup: Delete fixed-name objects (`-config` Secret, `-files` Secret, `-rcon` Secret, `-auto` BackupSchedule, `-wipe` Job) only when the GameServer controls them via OwnerReference; unowned objects with these names stay in place (same rule as game-ingress NetworkPolicy). Wipe cleanup carries the verified Job UID as a delete precondition; a foreign Job cannot acknowledge a pending wipe or resume its server. - **Conflict requeue:** When reconcileStatefulSet or reconcileWipe encounters an apierrors.IsConflict error (concurrent update races), requeue without error logging — conflicts are expected and transient. The Module, NetworkCapture and Backup reconcilers apply the same rule to every error they return, via `requeueOnConflict` (`conflict.go`). - Load-balancer address management: translate spec.networking.addressPool / .address preferences onto the Service based on the cluster's address-manager flavor (MetalLB, Cilium, or none), report assignment status via the AddressAssignment condition. diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d60a26f14..c9591505f 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -394,8 +394,10 @@ EOF # here are a conscious, reviewed act. Currently empty. unbucketed() { :; } +bucket_network_isolation() { echo TestNetworkIsolation_EnforcedAndUpgrade; } + bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components network-isolation } list_bucket() { @@ -414,6 +416,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + network-isolation) bucket_network_isolation ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go new file mode 100644 index 000000000..436a192ee --- /dev/null +++ b/test/e2e/network_isolation_e2e_test.go @@ -0,0 +1,193 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// This bucket uses Cilium, not kindnet. Exercise the effective union of all +// installed policies, including an existing legacy server's opt-in rollout. +func TestNetworkIsolation_EnforcedAndUpgrade(t *testing.T) { + ctx := t.Context() + const ns = "gameplane-games" + if _, err := envInstance.K8s.AppsV1().DaemonSets("kube-system").Get(ctx, "cilium", metav1.GetOptions{}); err != nil { + t.Fatalf("network-isolation bucket requires the enforcing Cilium fixture: %v", err) + } + stamp := time.Now().UnixNano() + legacyTemplate := fmt.Sprintf("net-legacy-%d", stamp) + isolatedTemplate := fmt.Sprintf("net-isolated-%d", stamp) + isolationTemplate(t, legacyTemplate, false) + isolationTemplate(t, isolatedTemplate, true) + a, b, c := fmt.Sprintf("net-a-%d", stamp), fmt.Sprintf("net-b-%d", stamp), fmt.Sprintf("net-c-%d", stamp) + for _, pair := range [][2]string{{a, legacyTemplate}, {b, legacyTemplate}, {c, isolatedTemplate}} { + applyBusyboxGameServer(t, ns, pair[0], pair[1]) + } + aIP := isolationGameIP(t, a, false) + bIP := isolationGameIP(t, b, false) + cIP := isolationGameIP(t, c, true) + probeA := isolationProbe(t, a, false) + probeC := isolationProbe(t, c, true) + + // Compatibility: absent-field templates keep their existing broad grant. + isolationHTTP(t, probeA, bIP, 25575, true) + isolationHTTP(t, probeC, cIP, 25566, true) + isolationHTTP(t, probeC, cIP, 25575, false) + isolationHTTP(t, probeC, aIP, 25566, false) + isolationHTTP(t, probeA, cIP, 25566, false) + + ss, err := envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + oldSelector := ss.Spec.Selector.DeepCopy() + _, err = envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, legacyTemplate, types.MergePatchType, + []byte(`{"spec":{"sentinelIsolation":true}}`), metav1.PatchOptions{}) + if err != nil { + t.Fatal(err) + } + aIP, bIP = isolationGameIP(t, a, true), isolationGameIP(t, b, true) + ss, err = envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, a, metav1.GetOptions{}) + if err != nil || !reflect.DeepEqual(oldSelector, ss.Spec.Selector) { + t.Fatalf("opt-in must preserve the immutable StatefulSet selector: %v", err) + } + _, err = envInstance.K8s.CoreV1().Pods(ns).Patch(ctx, probeA, types.MergePatchType, + []byte(`{"metadata":{"labels":{"gameplane.local/sentinel-isolation":"true"}}}`), metav1.PatchOptions{}) + if err != nil { + t.Fatal(err) + } + isolationHTTP(t, probeA, aIP, 25566, true) + isolationHTTP(t, probeA, aIP, 25575, false) + isolationHTTP(t, probeA, bIP, 25566, false) + isolationHTTP(t, probeA, bIP, 25575, false) + + // Kubelet HTTP readiness and agent heartbeats still work with strict CIDRs. + // Also exercise an actual API -> agent mTLS call through its Service. + envInstance.BootstrapAdmin(t, adminUsername, adminPassword) + cli := envInstance.APIClient(t, adminUsername, adminPassword) + defer cli.Close() + requireAgentReady(t, ns, a) + waitAgentReachable(t, cli, a) + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, a, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + hb, _, _ := unstructured.NestedString(gs.Object, "status", "agent", "lastHeartbeat") + stamp, err := time.Parse(time.RFC3339, hb) + return err == nil && time.Since(stamp) < 45*time.Second, "waiting for fresh agent heartbeat" + }) +} + +func isolationTemplate(t *testing.T, name string, isolated bool) { + t.Helper() + spec := map[string]any{ + "displayName": name, "game": "busybox", "version": "1", "image": "busybox:1.36", + "command": []any{"sh", "-c", "mkdir -p /tmp/http; echo reachable >/tmp/http/index.html; httpd -p 25575 -h /tmp/http; exec httpd -f -p 25566 -h /tmp/http"}, + "ports": []any{ + map[string]any{"name": "game", "containerPort": int64(25566), "protocol": "TCP", "advertise": true}, + map[string]any{"name": "management", "containerPort": int64(25575), "protocol": "TCP", "advertise": false}, + }, + "probes": map[string]any{"readiness": map[string]any{ + "httpGet": map[string]any{"path": "/", "port": int64(25566)}, "periodSeconds": int64(2), + }}, + } + if isolated { + spec["sentinelIsolation"] = true + } + obj := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "GameTemplate", + "metadata": map[string]any{"name": name}, "spec": spec, + }} + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Create(t.Context(), obj, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameTemplateGVR).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) +} + +func isolationGameIP(t *testing.T, name string, isolated bool) string { + t.Helper() + var ip string + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods("gameplane-games").Get(t.Context(), name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + if pod.DeletionTimestamp != nil || (pod.Labels["gameplane.local/sentinel-isolation"] == "true") != isolated { + return false, "waiting for rollout with expected isolation label" + } + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue && pod.Status.PodIP != "" { + ip = pod.Status.PodIP + return true, "" + } + } + return false, "waiting for HTTP readiness probe" + }) + return ip +} + +// A synthetic client carries the exact real sentinel selectors. It exposes no +// listener and owns no Service/Deployment, so the running game is unaffected. +func isolationProbe(t *testing.T, server string, isolated bool) string { + t.Helper() + const ns = "gameplane-games" + name := server + "-probe" + labels := map[string]string{"app.kubernetes.io/name": "gameplane-waker", "app.kubernetes.io/instance": server} + if isolated { + labels["gameplane.local/sentinel-isolation"] = "true" + } + noToken := false + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: labels}, + Spec: corev1.PodSpec{AutomountServiceAccountToken: &noToken, + Containers: []corev1.Container{{Name: "probe", Image: "busybox:1.36", Command: []string{"sleep", "3600"}}}}, + } + if _, err := envInstance.K8s.CoreV1().Pods(ns).Create(t.Context(), pod, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.CoreV1().Pods(ns).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { + got, err := envInstance.K8s.CoreV1().Pods(ns).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return got.Status.Phase == corev1.PodRunning, "waiting for probe pod" + }) + return name +} + +func isolationHTTP(t *testing.T, source, ip string, port int, allowed bool) { + t.Helper() + // Every exec must succeed regardless of network outcome. Otherwise a broken + // kubectl/CNI fixture could masquerade as the expected denied connection. + target := fmt.Sprintf("http://%s:%d/", ip, port) + envInstance.Eventually(t, 45*time.Second, func() (bool, string) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + out, err := envInstance.Kubectl(ctx, "exec", "-n", "gameplane-games", source, "--", + "sh", "-c", `if wget -q -T 2 -O /dev/null "$1"; then echo ALLOWED; else echo DENIED; fi`, "probe", target) + if err != nil { + return false, "probe execution failed: " + err.Error() + out + } + got := strings.TrimSpace(out) + want := "DENIED" + if allowed { + want = "ALLOWED" + } + return got == want, fmt.Sprintf("%s -> %s: got %q, want %s", source, target, got, want) + }) +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index abf7cda91..d560531a5 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -6,6 +6,13 @@ ## Purpose +The amd64 `network-isolation` bucket requires the Cilium fixture +(`GAMEPLANE_E2E_CNI=cilium`) and strict chart CIDRs derived from the single-node +cluster. It tests actual legacy versus isolated sentinel connections, an +existing template's opt-in rollout without selector changes, HTTP kubelet +readiness, fresh agent heartbeats, and API-to-agent mTLS access. It uses one +admin login and runs separately from the kindnet shape-only checks. + The `test/e2e` module is Gameplane's end-to-end test suite: it stands up a real (kind-based) Kubernetes cluster, installs the Helm chart, and drives the operator/api/agent/sentinel stack as real components rather than mocks — CRDs applied through the typed/dynamic client, HTTP calls through a real port-forward, RCON/wire-protocol joins against real game server containers. Where the other modules' test tiers (unit, envtest) prove a component's own logic in isolation, this module proves the whole product works together on a cluster shaped like the ones users actually run. It was one of three `go.work` modules without CI lint coverage until this feature (`api`, `agent`, and `test/e2e` — see FR-001); all 13 `go.work` members (`netguard`, `gameaction`, `gameproto`, `operator`, `agent`, `api`, `sentinel`, `audit-syslog-bridge`, `telemetry-receiver`, `mcp-server`, `svcutil`, `tunnel`, and `test/e2e`) are golangci-lint gated in CI now — see the `lint` job's matrix in `.github/workflows/ci.yaml`. `test/e2e` lints with `--build-tags=e2e`, same tier as `operator`/`api`'s `--build-tags=envtest`, run as their own step so the catch-all lint step excludes all three and nothing is linted twice. From 9f2ce750e57bb30736c91dd1ad1ab8313990fadc Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 11:44:30 -0400 Subject: [PATCH 03/17] fix(tunnel): forward advertised Tailscale TCP and UDP traffic Signed-off-by: Baylem --- .../crds/gameplane.local_backups.yaml | 2 +- .../crds/gameplane.local_backupschedules.yaml | 2 +- .../crds/gameplane.local_clusters.yaml | 2 +- .../crds/gameplane.local_gameservers.yaml | 17 +- .../crds/gameplane.local_gametemplates.yaml | 2 +- .../crds/gameplane.local_modules.yaml | 2 +- .../crds/gameplane.local_modulesources.yaml | 2 +- .../crds/gameplane.local_networkcaptures.yaml | 2 +- .../crds/gameplane.local_restores.yaml | 2 +- .../gameplane.local_backups.yaml | 2 +- .../gameplane.local_backupschedules.yaml | 2 +- .../gameplane.local_clusters.yaml | 2 +- .../gameplane.local_gameservers.yaml | 17 +- .../gameplane.local_gametemplates.yaml | 2 +- .../gameplane.local_modules.yaml | 2 +- .../gameplane.local_modulesources.yaml | 2 +- .../gameplane.local_networkcaptures.yaml | 2 +- .../gameplane.local_restores.yaml | 2 +- docs/tailscale-forwarding-integration.md | 150 +++++ docs/tunnels.md | 36 +- go.work | 2 +- go.work.sum | 9 +- i18n/locales/en/app.json | 4 + operator/api/v1alpha1/gameserver_types.go | 15 +- .../crd/gameplane.local_gameservers.yaml | 15 +- operator/hack/condition-ids/main.go | 1 + .../internal/controller/gameserver_status.go | 36 +- .../controller/gameserver_tailscale_test.go | 48 ++ .../internal/controller/gameserver_tunnel.go | 45 +- .../gameserver_tunnel_envtest_test.go | 26 + operator/specs.md | 1 + .../OPEN-DECISIONS.md | 6 + tunnel/Dockerfile.frp | 2 +- tunnel/Dockerfile.playit | 2 +- tunnel/Dockerfile.tailscale | 10 +- tunnel/go.mod | 50 +- tunnel/go.sum | 236 +++++++ tunnel/main.go | 475 +------------ tunnel/main_test.go | 630 +----------------- tunnel/specs.md | 85 ++- tunnel/tailscale.go | 197 ++++++ tunnel/tailscale_forward.go | 253 +++++++ tunnel/tailscale_forward_test.go | 140 ++++ tunnel/tailscale_test.go | 214 ++++++ 44 files changed, 1519 insertions(+), 1235 deletions(-) create mode 100644 docs/tailscale-forwarding-integration.md create mode 100644 operator/internal/controller/gameserver_tailscale_test.go create mode 100644 tunnel/go.sum create mode 100644 tunnel/tailscale.go create mode 100644 tunnel/tailscale_forward.go create mode 100644 tunnel/tailscale_forward_test.go create mode 100644 tunnel/tailscale_test.go diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index d902ea995..0c4fd731f 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index d85406090..9ad791743 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 4d6aadcbb..51cb145f9 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index 95178e56e..427266f81 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1546,15 +1546,12 @@ spec: tags: description: |- Tags are Tailscale ACL tags requested for the device when it - registers, via `tailscale up --advertise-tags` (tailscaled's - declarative config has no field for tags, and changing the tags of a - registered node needs a fresh login). Each entry is "tag:" or a bare - name, which gets the "tag:" prefix. The tailnet ACL must grant - tagOwners for these tags to the auth key's owner; if the request is - refused, the tunnel pod logs the error and the tunnel runs untagged. - If any entry is invalid, the whole list is logged and ignored and the - device registers untagged. See - tunnel/specs.md. + registers through embedded tsnet. Each entry is "tag:" or a + bare name, which gets the "tag:" prefix. The tailnet policy must grant + tagOwners to the auth key's owner. Invalid or refused tags keep the + tunnel not ready; it never silently registers without requested tags. + Changing tags replaces the Pod and requires a reusable auth key. + See tunnel/specs.md for identity persistence and registration details. items: type: string maxItems: 8 diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index e5e3e1e3b..e3b8bdb31 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 5ade28cee..6d641633d 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 6c5dcbf36..d76adbe5b 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index cc113e7e8..365cac11f 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index b77cd5c25..adcbfd0f8 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index d902ea995..0c4fd731f 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index d85406090..9ad791743 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 4d6aadcbb..51cb145f9 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index 95178e56e..427266f81 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1546,15 +1546,12 @@ spec: tags: description: |- Tags are Tailscale ACL tags requested for the device when it - registers, via `tailscale up --advertise-tags` (tailscaled's - declarative config has no field for tags, and changing the tags of a - registered node needs a fresh login). Each entry is "tag:" or a bare - name, which gets the "tag:" prefix. The tailnet ACL must grant - tagOwners for these tags to the auth key's owner; if the request is - refused, the tunnel pod logs the error and the tunnel runs untagged. - If any entry is invalid, the whole list is logged and ignored and the - device registers untagged. See - tunnel/specs.md. + registers through embedded tsnet. Each entry is "tag:" or a + bare name, which gets the "tag:" prefix. The tailnet policy must grant + tagOwners to the auth key's owner. Invalid or refused tags keep the + tunnel not ready; it never silently registers without requested tags. + Changing tags replaces the Pod and requires a reusable auth key. + See tunnel/specs.md for identity persistence and registration details. items: type: string maxItems: 8 diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index e5e3e1e3b..e3b8bdb31 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 5ade28cee..6d641633d 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 6c5dcbf36..d76adbe5b 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index cc113e7e8..365cac11f 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index b77cd5c25..adcbfd0f8 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 338f68484c2e1e46db1da4f9bdb53f8a8bf1851aa30e8a744458208044aca6f0 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/docs/tailscale-forwarding-integration.md b/docs/tailscale-forwarding-integration.md new file mode 100644 index 000000000..2874177b5 --- /dev/null +++ b/docs/tailscale-forwarding-integration.md @@ -0,0 +1,150 @@ +# Tailscale forwarding integration + +This procedure needs a disposable Kubernetes installation and a disposable +tailnet with two client devices. It creates real Tailscale devices and sends real +TCP and UDP traffic. It is an integration environment for #722, not evidence +that a live tailnet run has passed. Ordinary CI runs the local socket and +controller regressions without provider credentials. Follow repository policy: +execute this in an authorized integration environment, not a local test run. + +Use the operator and tunnel image built from the same revision. Record their +digests, the revision, client versions, tailnet device IDs, conditions and traffic +results. Do not record auth keys, login URLs or private node state. + +## Provision the disposable environment + +Enable MagicDNS. Create a reusable auth key permitted to request +`tag:gameplane-test`. Configure a tailnet policy with no broad pre-existing allow +rules: permit the approved client to reach that tag on TCP 25565 and UDP 19132. +The second client must have no grant to the tag. For example, replacing the +example email with the approved test user's identity: + +```json +{ + "tagOwners": {"tag:gameplane-test": ["autogroup:admin"]}, + "grants": [{ + "src": ["approved-test-user@example.com"], + "dst": ["tag:gameplane-test"], + "ip": ["tcp:25565", "udp:19132"] + }] +} +``` + +Install this fixture in the disposable Gameplane cluster. Its third TCP port is +deliberately unadvertised. The backend prefixes replies so replacing it can be +distinguished from continuing to reach an old process. + +```yaml +apiVersion: gameplane.local/v1alpha1 +kind: GameTemplate +metadata: + name: tailnet-echo +spec: + displayName: Tailnet echo integration + game: tailnet-echo + version: "1" + image: python:3.14-alpine + consoleMode: none + command: [python, -u, -c] + args: + - | + import socketserver, threading + class TCP(socketserver.BaseRequestHandler): + def handle(self): + while True: + data = self.request.recv(65536) + if not data: return + self.request.sendall(b'backend-v1:' + data) + class UDP(socketserver.BaseRequestHandler): + def handle(self): + data, sock = self.request + sock.sendto(b'backend-v1:' + data, self.client_address) + for port in (25565, 25575): + server = socketserver.ThreadingTCPServer(('0.0.0.0', port), TCP) + threading.Thread(target=server.serve_forever, daemon=True).start() + socketserver.ThreadingUDPServer(('0.0.0.0', 19132), UDP).serve_forever() + ports: + - {name: game, containerPort: 25565, protocol: TCP, advertise: true} + - {name: query, containerPort: 19132, protocol: UDP, advertise: true} + - {name: private, containerPort: 25575, protocol: TCP, advertise: false} +--- +apiVersion: gameplane.local/v1alpha1 +kind: GameServer +metadata: + name: tailnet-echo + namespace: gameplane-games +spec: + templateRef: {name: tailnet-echo} + networking: + expose: ClusterIP + tunnel: + enabled: true + provider: tailscale + credentialsSecretRef: {name: tailnet-echo-tunnel-auth} + tailscale: + hostname: gameplane-forward-test + tags: ["tag:gameplane-test"] +``` + +Provide the reusable auth key through the dashboard's tunnel credential form or +the credential API; these create the required GameServer ownership reference. +Wait for the tunnel Deployment's ready replica and `TunnelReady=True`. Confirm +the admin console shows the requested tag and exact hostname, with no advertised +subnet routes. Record the node ID and tailnet IPs. Remove conflicting stale test +devices if the hostname has acquired a suffix; that state must remain not ready. + +## Exercise the datapath + +On the approved tailnet client, run this with the fixture's MagicDNS name: + +```python +import socket +host = 'gameplane-forward-test' +for kind, port in [(socket.SOCK_STREAM, 25565), (socket.SOCK_DGRAM, 19132)]: + with socket.socket(socket.AF_INET, kind) as client: + client.settimeout(5) + client.connect((socket.gethostbyname(host), port)) + client.sendall(b'game-payload') + assert client.recv(65536) == b'backend-v1:game-payload' +``` + +Repeat with the node's IPv6 tailnet address and `AF_INET6` when the environment +supports IPv6. Run two UDP clients concurrently with distinct payloads; each +must receive only its own reply. TCP 25575 and any other unadvertised port must +fail even from the approved client. Repeat the advertised-port probes from the +denied client; both must time out/fail while the tunnel remains ready. Removing +the approved grant must block new flows too; restore it before continuing. + +Create an unrelated echo Service with a distinguishable prefix in the cluster. +It must not be reachable through this tailnet device on any port, and no route +to its ClusterIP or the cluster's Service/Pod CIDRs may appear on either client. +The local regression also rejects unrelated backing DNS names before listeners +are opened; no incoming packet contains a selectable destination. + +Change the fixture backend prefix to `backend-v2:` and wait for its game Pod +rollout. New TCP connections must receive v2. Stop UDP traffic for over 30 seconds +to expire the old session, then send again and require v2. In the disposable +cluster, delete only the fixture's game Service and wait for the operator to +recreate it; repeat both probes. Existing connections may fail during replacement; +new sessions must resolve the Service again and recover without replacing the +tunnel Pod. + +## Identity, authentication and shutdown outcomes + +| Change | Required outcome | +|---|---| +| Restart the tunnel **container** while retaining the Pod and EmptyDir (use the integration runner's container runtime controls) | Same node ID, hostname and tags reconnect; TCP/UDP recover. | +| Delete the tunnel **Pod**, remove its stale device from the admin console, allow replacement | New identity registers using the reusable key; exact hostname/tags recover and both protocols work. | +| Replace Pod with an expired/revoked key in its credential Secret | Registration never makes the Pod ready and no Tailscale endpoint is advertised. Store a valid reusable key, replace the Pod and require recovery. | +| Request a tag not permitted by tagOwners | Not ready; there must be no untagged fallback device serving traffic. Restore permitted tags and repair credentials/device state, then verify recovery. | +| Rename an existing device to collide with the configured hostname | A suffix assigned to the replacement device cannot make Gameplane advertise the wrong hostname as ready. Resolve the collision and re-register. | +| Disconnect the node from the tailnet/control plane | Readiness turns false after the status/probe intervals; restore connectivity and require recovery. Do not use client ACL denial as a simulation of a disconnected server. | +| Delete/terminate tunnel Pod during active TCP and UDP traffic | Listeners and sessions close, worker shutdown does not hang, and clients reconnect only after the replacement is ready. | + +Remove the GameServer and fixture template, revoke the disposable auth key, and +delete all test devices from the tailnet. Retain only the sanitized result record. + +The implementation uses the official [tsnet APIs](https://tailscale.com/docs/reference/tsnet-server-api). +Its Go 1.26.10 minimum and Go 1.27.3 image builders include the `crypto/x509` +security fix described in the [Go release history](https://go.dev/doc/devel/release) +and [Tailscale security bulletin TS-2026-013](https://tailscale.com/security-bulletins#ts-2026-013). diff --git a/docs/tunnels.md b/docs/tunnels.md index b8ce3fd2a..66f965301 100644 --- a/docs/tunnels.md +++ b/docs/tunnels.md @@ -197,21 +197,27 @@ specify in `spec.networking.tunnel.tailscale.hostname` (or defaults to the GameServer name if left empty). Port 25565 is available to devices on your tailnet. -> **`tags` requires an ACL grant.** `spec.networking.tunnel.tailscale.tags` -> is requested when the device registers with your tailnet: the tunnel pod -> runs `tailscale up --advertise-tags=` once, over the local daemon's -> socket. Tailscale's headless daemon config has no field for tags, and -> changing the tags of a registered device needs a fresh login (see -> `tunnel/specs.md`). The tailnet's ACL must grant `tagOwners` for the -> requested tags to the auth key's owner. If it doesn't, Tailscale refuses -> the request and the tunnel pod logs the error. If the device is not -> running afterwards, the pod tries once more without tags. That untagged -> attempt can fail too (for example with a single-use auth key), in which -> case the error is logged and the device stays logged out. Configure `tagOwners` in your tailnet's ACL before setting -> `tags` here. Each tag is `tag:` or a bare ``, where the name -> starts with a letter and holds only letters, digits and `-`. If any tag is -> invalid, the whole list is logged and ignored and the device registers -> untagged. +The embedded Tailscale node forwards every advertised TCP and UDP port to this +GameServer's backing Service. It does not provide a subnet route or expose +unadvertised ports. `TunnelReady` requires an online device and active forwarding +listeners. Tailnet grants still govern each player's access; readiness is not a +test of an individual grant or of the sleeping game's application health. + +Requested tags require a matching `tagOwners` policy. A bare tag name receives +the `tag:` prefix. Invalid or denied tags keep the tunnel not ready; it never +silently falls back to an untagged identity. + +Container restarts retain identity in a Pod volume. Replacing the Pod, changing +its tags, or upgrading from the older tailscaled implementation re-registers the +device and requires a valid reusable auth key. Remove stale devices from the +tailnet admin console so the configured MagicDNS hostname remains available. +If Tailscale assigns a suffixed name because of a conflict, Gameplane keeps the +advertised endpoint not ready until the conflict is resolved. Enable MagicDNS. + +Upgrade the operator and tunnel images together: the forwarding contract now +carries each port's TCP/UDP protocol. See the +[disposable tailnet integration procedure](tailscale-forwarding-integration.md) +for traffic, access-policy and restart checks. The `credentialsSecretRef.name` is the name that the dashboard or API write after you provide the credential. diff --git a/go.work b/go.work index fa6b1bb44..7ccf88062 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.26.0 +go 1.26.10 use ( ./agent diff --git a/go.work.sum b/go.work.sum index 2da1910fe..1a92b2fe8 100644 --- a/go.work.sum +++ b/go.work.sum @@ -148,6 +148,7 @@ github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e/go.mod h1:V284 github.com/cavaliercoder/go-rpm v0.0.0-20200122174316-8cb9fd9c31a8/go.mod h1:AZIh1CCnMrcVm6afFf96PBvE2MRpWFco91z8ObJtgDY= github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/chainguard-dev/clog v1.7.0/go.mod h1:4+WFhRMsGH79etYXY3plYdp+tCz/KCkU8fAr0HoaPvs= github.com/chainguard-dev/clog v1.8.0/go.mod h1:5MQOZi+Iu7fV7GcJG8ag8rCB5elEOpqRMKEASgnGVdo= @@ -218,6 +219,8 @@ github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJP github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= +github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= +github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/firefart/nonamedreturns v1.0.6/go.mod h1:R8NisJnSIpvPWheCq0mNRXJok6D8h7fagJTF8EMEwCo= github.com/flynn/go-docopt v0.0.0-20140912013429-f6dd2ebbb31e/go.mod h1:HyVoz1Mz5Co8TFO8EupIdlcpwShBmY98dkT2xeHkvEI= @@ -306,7 +309,6 @@ github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba/go.mod h1:sC github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e/go.mod h1:Vrn4B5oR9qRwM+f54koyeH3yzphlecwERs0el27Fr/s= github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= github.com/google/addlicense v1.1.1/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= -github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= @@ -405,6 +407,8 @@ github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGq github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= @@ -721,6 +725,7 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= @@ -756,7 +761,6 @@ golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY= @@ -809,7 +813,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index 059470fd2..187cf2b88 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -595,6 +595,10 @@ "defaultMessage": "Unknown tunnel provider ''{provider}''.", "description": "TunnelReady condition when the provider name is not supported." }, + "condition.tunnelready.unsupportedprotocol": { + "defaultMessage": "Tailscale forwards only advertised TCP and UDP ports. Remove unsupported protocols from the advertised ports.", + "description": "TunnelReady condition when a Tailscale template advertises a protocol other than TCP or UDP." + }, "condition.unquiesced.failed": { "defaultMessage": "The server could not be resumed after the backup. Its world may still have auto-save turned off.", "description": "Unquiesced condition when the unfreeze failed." diff --git a/operator/api/v1alpha1/gameserver_types.go b/operator/api/v1alpha1/gameserver_types.go index 654cba6ed..df2a65660 100644 --- a/operator/api/v1alpha1/gameserver_types.go +++ b/operator/api/v1alpha1/gameserver_types.go @@ -460,15 +460,12 @@ type TailscaleTunnelSpec struct { Hostname string `json:"hostname,omitempty"` // Tags are Tailscale ACL tags requested for the device when it - // registers, via `tailscale up --advertise-tags` (tailscaled's - // declarative config has no field for tags, and changing the tags of a - // registered node needs a fresh login). Each entry is "tag:" or a bare - // name, which gets the "tag:" prefix. The tailnet ACL must grant - // tagOwners for these tags to the auth key's owner; if the request is - // refused, the tunnel pod logs the error and the tunnel runs untagged. - // If any entry is invalid, the whole list is logged and ignored and the - // device registers untagged. See - // tunnel/specs.md. + // registers through embedded tsnet. Each entry is "tag:" or a + // bare name, which gets the "tag:" prefix. The tailnet policy must grant + // tagOwners to the auth key's owner. Invalid or refused tags keep the + // tunnel not ready; it never silently registers without requested tags. + // Changing tags replaces the Pod and requires a reusable auth key. + // See tunnel/specs.md for identity persistence and registration details. // +kubebuilder:validation:MaxItems=8 // +optional Tags []string `json:"tags,omitempty"` diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index 4be670ebc..b323bcd1d 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -1545,15 +1545,12 @@ spec: tags: description: |- Tags are Tailscale ACL tags requested for the device when it - registers, via `tailscale up --advertise-tags` (tailscaled's - declarative config has no field for tags, and changing the tags of a - registered node needs a fresh login). Each entry is "tag:" or a bare - name, which gets the "tag:" prefix. The tailnet ACL must grant - tagOwners for these tags to the auth key's owner; if the request is - refused, the tunnel pod logs the error and the tunnel runs untagged. - If any entry is invalid, the whole list is logged and ignored and the - device registers untagged. See - tunnel/specs.md. + registers through embedded tsnet. Each entry is "tag:" or a + bare name, which gets the "tag:" prefix. The tailnet policy must grant + tagOwners to the auth key's owner. Invalid or refused tags keep the + tunnel not ready; it never silently registers without requested tags. + Changing tags replaces the Pod and requires a reusable auth key. + See tunnel/specs.md for identity persistence and registration details. items: type: string maxItems: 8 diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index e644cf041..41cefbf94 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -121,6 +121,7 @@ var rows = []row{ {types: []string{"AddressAssignment"}, reason: "Assigned", english: "{assignedFrom, select, pool {Address {address} assigned from pool ''{pool}''.} other {Address {address} assigned.}}", description: "AddressAssignment condition once an address is assigned. assignedFrom is pool or default."}, // GameServer: tunnel and other + {types: []string{"TunnelReady"}, reason: "UnsupportedProtocol", english: "Tailscale forwards only advertised TCP and UDP ports. Remove unsupported protocols from the advertised ports.", description: "TunnelReady condition when a Tailscale template advertises a protocol other than TCP or UDP."}, {types: []string{"TunnelReady"}, reason: "InvalidConfig", english: "{provider, select, none {The tunnel is enabled but the server has no tunnel configuration.} other {The ''{provider}'' tunnel provider is selected but its configuration is missing.}}", description: "TunnelReady condition for an incomplete tunnel setup. provider is the selected provider, or none."}, {types: []string{"TunnelReady"}, reason: "DeploymentNotReady", english: "Waiting for the tunnel deployment to become ready.", description: "TunnelReady condition while the tunnel pod is not ready."}, {types: []string{"TunnelReady"}, reason: "PortNotMapped", english: "No frp remote port is configured for: {ports}.", description: "TunnelReady condition for frp when advertised ports have no remote port. ports is a comma-separated list of port names."}, diff --git a/operator/internal/controller/gameserver_status.go b/operator/internal/controller/gameserver_status.go index 54393de0b..f0b33fb4d 100644 --- a/operator/internal/controller/gameserver_status.go +++ b/operator/internal/controller/gameserver_status.go @@ -260,8 +260,19 @@ func (r *GameServerReconciler) reconcileStatus( // Connection card reads the address players actually use. if svcExists { gs.Status.Endpoints = endpointsFromService(&svc, addrPlan) + } else { + // A deleted/replacing Service must not preserve a stale tailnet endpoint + // when the forwarding deployment has since lost readiness. + var retained []gameplanev1alpha1.GameServerEndpoint + for _, endpoint := range gs.Status.Endpoints { + if endpoint.TunnelProvider != "tailscale" { + retained = append(retained, endpoint) + } + } + gs.Status.Endpoints = retained } - if tunnelPlan.wantTunnel && len(tunnelPlan.endpoints) > 0 { + if tunnelPlan.wantTunnel && len(tunnelPlan.endpoints) > 0 && + (gs.Spec.Networking.Tunnel.Provider != "tailscale" || meta.IsStatusConditionTrue(gs.Status.Conditions, "TunnelReady")) { gs.Status.Endpoints = append(tunnelPlan.endpoints, gs.Status.Endpoints...) } if phase == gameplanev1alpha1.GameServerPhaseRunning { @@ -1105,6 +1116,12 @@ func computeTunnelConditions( } // Tunnel Deployment not yet created or not ready. + if tunnel.Provider == "tailscale" && len(plan.unsupportedPorts) > 0 { + tunnelReady.Status = metav1.ConditionFalse + tunnelReady.Reason = "UnsupportedProtocol" + tunnelReady.Message = "Tailscale supports only TCP and UDP; unsupported advertised ports: " + strings.Join(plan.unsupportedPorts, ", ") + return upsertCondition(conds, tunnelReady) + } if tunnelDep == nil { tunnelReady.Status = metav1.ConditionFalse tunnelReady.Reason = "DeploymentNotReady" @@ -1167,10 +1184,23 @@ func computeTunnelConditions( return conds } - // Deployment is ready and credentials are configured. + // A legacy image merely registering a device cannot satisfy readiness. + forwardingProbe := false + for _, container := range tunnelDep.Spec.Template.Spec.Containers { + if container.Name == "tunnel" && container.ReadinessProbe != nil && container.ReadinessProbe.HTTPGet != nil { + probe := container.ReadinessProbe.HTTPGet + forwardingProbe = probe.Path == "/readyz" && probe.Port.IntValue() == 8091 + } + } + if !forwardingProbe || tunnelDep.Status.ObservedGeneration < tunnelDep.Generation || tunnelDep.Status.UpdatedReplicas == 0 { + tunnelReady.Status = metav1.ConditionFalse + tunnelReady.Reason = "DeploymentNotReady" + tunnelReady.Message = "waiting for the Tailscale forwarding deployment and its readiness probe" + return upsertCondition(conds, tunnelReady) + } tunnelReady.Status = metav1.ConditionTrue tunnelReady.Reason = "Ready" - tunnelReady.Message = "tunnel deployment ready and authenticated" + tunnelReady.Message = "tailnet online and all advertised TCP/UDP forwarding listeners active; client access remains subject to tailnet policy" case "playit": if tunnel.Playit == nil { diff --git a/operator/internal/controller/gameserver_tailscale_test.go b/operator/internal/controller/gameserver_tailscale_test.go new file mode 100644 index 000000000..1653fccf2 --- /dev/null +++ b/operator/internal/controller/gameserver_tailscale_test.go @@ -0,0 +1,48 @@ +package controller + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestTailscaleAdvertisedProtocolsAndReadiness(t *testing.T) { + tmpl := &gameplanev1alpha1.GameTemplate{Spec: gameplanev1alpha1.GameTemplateSpec{Ports: []gameplanev1alpha1.GamePort{ + {Name: "game", ContainerPort: 25565, Advertise: true}, + {Name: "query", ContainerPort: 19132, Protocol: corev1.ProtocolUDP, Advertise: true}, + {Name: "rcon", ContainerPort: 25575, Protocol: corev1.ProtocolTCP}, + }}} + if got := buildTailscalePortsConfig(tmpl); got != "game:25565:tcp,query:19132:udp" { + t.Fatalf("advertised mapping: %s", got) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "tailscale", Tailscale: &gameplanev1alpha1.TailscaleTunnelSpec{}, CredentialsSecretRef: &gameplanev1alpha1.SecretNameRef{Name: "auth"}}}}} + r := &GameServerReconciler{} + plan := r.planTunnel(context.Background(), gs, tmpl) + dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Status: appsv1.DeploymentStatus{ReadyReplicas: 1, UpdatedReplicas: 1, ObservedGeneration: 2}} + assert := func(want metav1.ConditionStatus) { + t.Helper() + c := meta.FindStatusCondition(computeTunnelConditions(gs, plan, dep), "TunnelReady") + if c == nil || c.Status != want { + t.Fatalf("TunnelReady=%+v, want %s", c, want) + } + } + assert(metav1.ConditionFalse) // A registered legacy deployment has no forwarding probe. + dep.Spec.Template.Spec.Containers = []corev1.Container{{Name: "tunnel", ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt(8091)}}}}} + assert(metav1.ConditionTrue) + dep.Status.ObservedGeneration = 1 + assert(metav1.ConditionFalse) + dep.Status.ObservedGeneration = 2 + dep.Status.ReadyReplicas = 0 + assert(metav1.ConditionFalse) + dep.Status.ReadyReplicas = 1 + tmpl.Spec.Ports[0].Protocol = corev1.ProtocolSCTP + plan = r.planTunnel(context.Background(), gs, tmpl) + assert(metav1.ConditionFalse) +} diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index a950292b2..0168e9e88 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -52,7 +52,8 @@ type tunnelPlan struct { endpoints []gameplanev1alpha1.GameServerEndpoint // noMapping is the set of advertised port names that have no frp remote-port mapping. // Used only by frp to report a helpful condition reason. - noMapping []string + noMapping []string + unsupportedPorts []string } // planTunnel is the single source of truth for the tunnel pod's existence @@ -74,6 +75,7 @@ func (r *GameServerReconciler) planTunnel( tunnel := gs.Spec.Networking.Tunnel endpoints := make([]gameplanev1alpha1.GameServerEndpoint, 0) var noMapping []string + var unsupportedPorts []string switch tunnel.Provider { case "frp": @@ -128,6 +130,10 @@ func (r *GameServerReconciler) planTunnel( if !p.Advertise { continue } + if p.Protocol != "" && p.Protocol != corev1.ProtocolTCP && p.Protocol != corev1.ProtocolUDP { + unsupportedPorts = append(unsupportedPorts, p.Name) + continue + } ep := gameplanev1alpha1.GameServerEndpoint{ Name: p.Name, @@ -152,9 +158,10 @@ func (r *GameServerReconciler) planTunnel( } return tunnelPlan{ - wantTunnel: true, - endpoints: endpoints, - noMapping: noMapping, + wantTunnel: true, + endpoints: endpoints, + noMapping: noMapping, + unsupportedPorts: unsupportedPorts, } } @@ -199,6 +206,11 @@ func (r *GameServerReconciler) reconcileTunnel( refusedSecret = "" replicas := int32(1) dep.Spec.Replicas = &replicas + dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RollingUpdateDeploymentStrategyType} + if tunnel.Provider == "tailscale" { + // Avoid two tailnet devices racing for the same hostname on rollout. + dep.Spec.Strategy.Type = appsv1.RecreateDeploymentStrategyType + } dep.Spec.Selector = &metav1.LabelSelector{ MatchLabels: map[string]string{ tunnelLabel: tunnelValue, @@ -388,13 +400,20 @@ func (r *GameServerReconciler) reconcileTunnel( } } + var readiness *corev1.Probe + if tunnel.Provider == "tailscale" { + volumes = append(volumes, corev1.Volume{Name: "tailscale-state", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "tailscale-state", MountPath: "/var/lib/gameplane-tailscale"}) + readiness = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt(8091)}}, PeriodSeconds: 5, TimeoutSeconds: 2, FailureThreshold: 1} + } dep.Spec.Template.Spec.Volumes = volumes dep.Spec.Template.Spec.Containers = []corev1.Container{{ - Name: "tunnel", - Image: image, - Env: envVars, - Ports: ports, - VolumeMounts: volumeMounts, + Name: "tunnel", + Image: image, + Env: envVars, + Ports: ports, + VolumeMounts: volumeMounts, + ReadinessProbe: readiness, SecurityContext: &corev1.SecurityContext{ RunAsNonRoot: &nonRoot, RunAsUser: &uid, @@ -694,14 +713,18 @@ func buildFrpRemotePortsConfig(frp *gameplanev1alpha1.FrpTunnelSpec, tmpl *gamep } // buildTailscalePortsConfig constructs the BACKING_SERVICE_PORTS env var for Tailscale. -// Format: "port_name:port,..." e.g. "java:25565,bedrock:19133" +// Format: "port_name:port:protocol,..." e.g. "java:25565:tcp,bedrock:19133:udp". func buildTailscalePortsConfig(tmpl *gameplanev1alpha1.GameTemplate) string { var entries []string for _, p := range tmpl.Spec.Ports { if !p.Advertise { continue } - entries = append(entries, fmt.Sprintf("%s:%d", p.Name, p.ContainerPort)) + protocol := strings.ToLower(string(p.Protocol)) + if protocol == "" { + protocol = "tcp" + } + entries = append(entries, fmt.Sprintf("%s:%d:%s", p.Name, p.ContainerPort, protocol)) } if len(entries) == 0 { return "" diff --git a/operator/internal/controller/gameserver_tunnel_envtest_test.go b/operator/internal/controller/gameserver_tunnel_envtest_test.go index d86582fb9..2447b2ee1 100644 --- a/operator/internal/controller/gameserver_tunnel_envtest_test.go +++ b/operator/internal/controller/gameserver_tunnel_envtest_test.go @@ -109,6 +109,32 @@ func TestTunnelReconciliation_CredentialKeyProjection(t *testing.T) { } // Verify the Secret volume has Items projection with only "authKey" + if dep.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType || dep.Spec.Strategy.RollingUpdate != nil { + t.Fatalf("Tailscale must recreate its single identity: %+v", dep.Spec.Strategy) + } + container := dep.Spec.Template.Spec.Containers[0] + if container.ReadinessProbe == nil || container.ReadinessProbe.HTTPGet == nil || container.ReadinessProbe.HTTPGet.Path != "/readyz" || container.ReadinessProbe.HTTPGet.Port.IntValue() != 8091 { + t.Fatalf("Tailscale lacks forwarding readiness probe: %+v", container.ReadinessProbe) + } + stateVolume, stateMount, protocolEnv := false, false, false + for _, volume := range dep.Spec.Template.Spec.Volumes { + if volume.Name == "tailscale-state" && volume.EmptyDir != nil { + stateVolume = true + } + } + for _, mount := range container.VolumeMounts { + if mount.Name == "tailscale-state" && mount.MountPath == "/var/lib/gameplane-tailscale" { + stateMount = true + } + } + for _, env := range container.Env { + if env.Name == "BACKING_SERVICE_PORTS" && env.Value == "game:25565:tcp" { + protocolEnv = true + } + } + if !stateVolume || !stateMount || !protocolEnv { + t.Fatal("Tailscale lost persistent pod state or advertised protocol contract") + } if len(dep.Spec.Template.Spec.Volumes) == 0 { t.Fatal("no volumes in tunnel pod template") } diff --git a/operator/specs.md b/operator/specs.md index c7aea8225..8759da92d 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -134,6 +134,7 @@ Nine CRD kinds under `gameplane.local/v1alpha1`: Primary reconcilers register with the manager in `cmd/main.go` and handle CRD lifecycle: ### GameServerReconciler +- **Tailscale forwarding (#722):** the tunnel receives advertised `name:port:protocol` mappings and forwards only to `..svc` through embedded tsnet. Its Deployment uses Recreate and a per-Pod EmptyDir for identity across container restarts. `/readyz` on 8091 requires an online tailnet identity and every TCP/UDP listener. TunnelReady additionally requires the forwarding probe, current Deployment observedGeneration and an updated ready replica; unsupported protocols report InvalidConfig. Tailscale endpoints are withheld while not ready. Pod replacement requires a reusable auth key and an available MagicDNS hostname. No extra privileges or Kubernetes RBAC are added. - **Responsibility:** Reconcile GameServer → StatefulSet, Service, Config ConfigMap, PVC, NetworkPolicy. - **Template recovery:** A missing template reports Failed and retries every 15 seconds. Template creation or spec changes enqueue referencing servers immediately, so a GameServer applied before its Module/template recovers without a manual edit; template status updates do not enqueue servers. - **Key functions:** diff --git a/specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md b/specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md index 5401d5a14..effd4cb5b 100644 --- a/specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md +++ b/specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md @@ -1,5 +1,11 @@ # Open Decisions: v0.3 Release Readiness Audit +### OD-031: Tailscale game forwarding (F-173) — RESOLVED 2026-10-21 + +Decision (maintainer reply on #722): embed `tailscale.com/tsnet` and forward both TCP and UDP from advertised tailnet listeners to the GameServer's backing Service. This explicitly ends the tunnel module's stdlib-only dependency policy. Device registration alone cannot establish readiness; the same change must provide actual tailnet/listener health. The previous Known Gaps entry referred to a decision that had never been recorded. + +The bridge opens only advertised protocol/port pairs, never a subnet route. Tailnet grants/ACLs govern peer access. The node uses the existing unprivileged container context, with no NET_ADMIN or TUN device. A per-Pod EmptyDir retains identity across container restarts; Pod replacement re-registers with a reusable auth key and requires an available MagicDNS hostname. No durable PVC is introduced. Invalid or refused requested tags remain not ready rather than falling back to an untagged device. UDP queues, session counts, idle lifetimes and shutdown are bounded. See `tunnel/specs.md` and `docs/tailscale-forwarding-integration.md` for the implementation contract and integration procedure. + These values are unsettled. They are not contracts; resolve them during `/speckit-clarify` or `/speckit-plan`. ## OD-001: How the release candidate is deployed onto kubelab — RESOLVED 2026-09-23 diff --git a/tunnel/Dockerfile.frp b/tunnel/Dockerfile.frp index a97ce8f8c..120e4e135 100644 --- a/tunnel/Dockerfile.frp +++ b/tunnel/Dockerfile.frp @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS build +FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine AS build WORKDIR /src # tunnel's go.mod currently has no gameaction requirement (no local diff --git a/tunnel/Dockerfile.playit b/tunnel/Dockerfile.playit index 92cfb32ba..54c68af95 100644 --- a/tunnel/Dockerfile.playit +++ b/tunnel/Dockerfile.playit @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS build +FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine AS build WORKDIR /src # tunnel's go.mod currently has no gameaction requirement (no local diff --git a/tunnel/Dockerfile.tailscale b/tunnel/Dockerfile.tailscale index 21ad097a2..fc74b830b 100644 --- a/tunnel/Dockerfile.tailscale +++ b/tunnel/Dockerfile.tailscale @@ -1,4 +1,4 @@ -FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS build +FROM --platform=$BUILDPLATFORM golang:1.27.3-alpine AS build WORKDIR /src # tunnel's go.mod currently has no gameaction requirement (no local @@ -24,16 +24,10 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ -ldflags="-s -w -X main.Version=${VERSION}" \ -o /out/tunnel . -# Copy tailscale binaries from the official upstream image pinned by digest. -# tailscale is distributed as official OCI images; pinning by digest -# ensures reproducible builds. The image ships both tailscaled and tailscale CLI. -FROM tailscale/tailscale:v1.104.1@sha256:87c55d6be56ad85b3d8bceca5f87f73de288c609131460dd1be1874c250cc41d AS tailscale-stage - FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=build /out/tunnel /tunnel -COPY --from=tailscale-stage /usr/local/bin/tailscaled /usr/local/bin/tailscaled -COPY --from=tailscale-stage /usr/local/bin/tailscale /usr/local/bin/tailscale USER 65532:65532 EXPOSE 25565/tcp 25565/udp +EXPOSE 8091/tcp ENTRYPOINT ["/tunnel"] diff --git a/tunnel/go.mod b/tunnel/go.mod index da205c4d2..549ed1893 100644 --- a/tunnel/go.mod +++ b/tunnel/go.mod @@ -1,3 +1,51 @@ module github.com/GameplanePanel/gameplane/tunnel -go 1.26.0 +go 1.26.10 + +require tailscale.com v1.102.5 + +require ( + filippo.io/edwards25519 v1.2.0 // indirect + github.com/akutz/memconn v0.1.0 // indirect + github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa // indirect + github.com/coder/websocket v1.8.14 // indirect + github.com/creachadair/msync v0.8.1 // indirect + github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/gaissmai/bart v0.26.1 // indirect + github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 // indirect + github.com/godbus/dbus/v5 v5.2.2 // indirect + github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect + github.com/google/btree v1.1.3 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/hdevalence/ed25519consensus v0.2.0 // indirect + github.com/huin/goupnp v1.3.0 // indirect + github.com/jsimonetti/rtnetlink v1.4.1 // indirect + github.com/klauspost/compress v1.20.1 // indirect + github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 // indirect + github.com/mdlayher/socket v0.5.0 // indirect + github.com/mitchellh/go-ps v1.0.0 // indirect + github.com/pires/go-proxyproto v0.8.1 // indirect + github.com/safchain/ethtool v0.3.0 // indirect + github.com/tailscale/certstore v0.1.1-0.20260409135935-3638fb84b77d // indirect + github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 // indirect + github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd // indirect + github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc // indirect + github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 // indirect + github.com/tailscale/wireguard-go v0.0.0-20260715223240-2e01ba5b00f0 // indirect + github.com/x448/float16 v0.8.4 // indirect + go4.org/mem v0.0.0-20240501181205-ae6ca9944745 // indirect + go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect + golang.org/x/crypto v0.58.0 // indirect + golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect + golang.org/x/net v0.61.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/term v0.47.0 // indirect + golang.org/x/text v0.43.0 // indirect + golang.org/x/time v0.15.0 // indirect + golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect + golang.zx2c4.com/wireguard/windows v0.5.3 // indirect + gvisor.dev/gvisor v0.0.0-20260224225140-573d5e7127a8 // indirect +) diff --git a/tunnel/go.sum b/tunnel/go.sum new file mode 100644 index 000000000..3d6a66b12 --- /dev/null +++ b/tunnel/go.sum @@ -0,0 +1,236 @@ +9fans.net/go v0.0.8-0.20250307142834-96bdba94b63f h1:1C7nZuxUMNz7eiQALRfiqNOm04+m3edWlRff/BYHf0Q= +9fans.net/go v0.0.8-0.20250307142834-96bdba94b63f/go.mod h1:hHyrZRryGqVdqrknjq5OWDLGCTJ2NeEvtrpR96mjraM= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= +filippo.io/mkcert v1.4.4 h1:8eVbbwfVlaqUM7OwuftKc2nuYOoTDQWqsoXmzoXZdbc= +filippo.io/mkcert v1.4.4/go.mod h1:VyvOchVuAye3BoUsPUOOofKygVwLV2KQMVFJNRq+1dA= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/akutz/memconn v0.1.0 h1:NawI0TORU4hcOMsMr11g7vwlCdkYeLKXBcxWu2W/P8A= +github.com/akutz/memconn v0.1.0/go.mod h1:Jo8rI7m0NieZyLI5e2CDlRdRqRRB4S7Xp77ukDjH+Fw= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= +github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= +github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek= +github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/ssm v1.45.0 h1:IOdss+igJDFdic9w3WKwxGCmHqUxydvIhJOm9LJ32Dk= +github.com/aws/aws-sdk-go-v2/service/ssm v1.45.0/go.mod h1:Q7XIWsMo0JcMpI/6TGD6XXcXcV1DbTj6e9BKNntIMIM= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/axiomhq/hyperloglog v0.0.0-20240319100328-84253e514e02 h1:bXAPYSbdYbS5VTy92NIUbeDI1qyggi+JYh5op9IFlcQ= +github.com/axiomhq/hyperloglog v0.0.0-20240319100328-84253e514e02/go.mod h1:k08r+Yj1PRAmuayFiRK6MYuR5Ve4IuZtTfxErMIh0+c= +github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= +github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= +github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= +github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= +github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6 h1:8h5+bWd7R6AYUslN6c6iuZWTKsKxUFDlpnmilO6R2n0= +github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= +github.com/creachadair/mds v0.25.13 h1:PsSUHV6zsfPd29k4kvm1rMoee1YFia7JyNGeMPmDcPM= +github.com/creachadair/mds v0.25.13/go.mod h1:4hatI3hRM+qhzuAmqPRFvaBM8mONkS7nsLxkcuTYUIs= +github.com/creachadair/msync v0.8.1 h1:QRd8si3qZ2Q4TaDL7tS/MG/lFE3YND7U7J9fy42eAFM= +github.com/creachadair/msync v0.8.1/go.mod h1:dt0bscS09J8Ie3AdccK9JpCb7LfStaDGlAmDLukOlY4= +github.com/creachadair/taskgroup v0.13.2 h1:3KyqakBuFsm3KkXi/9XIb0QcA8tEzLHLgaoidf0MdVc= +github.com/creachadair/taskgroup v0.13.2/go.mod h1:i3V1Zx7H8RjwljUEeUWYT30Lmb9poewSb2XI1yTwD0g= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa h1:h8TfIT1xc8FWbwwpmHn1J5i43Y0uZP97GqasGCzSRJk= +github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa/go.mod h1:Nx87SkVqTKd8UtT+xu7sM/l+LgXs6c0aHrlKusR+2EQ= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc h1:8WFBn63wegobsYAX0YjD+8suexZDga5CctH4CCTx2+8= +github.com/dgryski/go-metro v0.0.0-20180109044635-280f6062b5bc/go.mod h1:c9O8+fpSOX1DM8cPNSkX/qsBWdkD4yd2dpciOWQjpBw= +github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e h1:vUmf0yezR0y7jJ5pceLHthLaYf4bA5T14B6q39S4q2Q= +github.com/digitalocean/go-smbios v0.0.0-20180907143718-390a4f403a8e/go.mod h1:YTIHhz/QFSYnu/EhlF2SpU2Uk+32abacUYA5ZPljz1A= +github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= +github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/gaissmai/bart v0.26.1 h1:+w4rnLGNlA2GDVn382Tfe3jOsK5vOr5n4KmigJ9lbTo= +github.com/gaissmai/bart v0.26.1/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c= +github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I= +github.com/github/fakeca v0.1.0/go.mod h1:+bormgoGMMuamOscx7N91aOuUST7wdaJ2rNjeohylyo= +github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433 h1:vymEbVwYFP/L05h5TKQxvkXoKxNvTpjxYKdF1Nlwuao= +github.com/go-json-experiment/json v0.0.0-20260214004413-d219187c3433/go.mod h1:tphK2c80bpPhMOI4v6bIc2xWywPfbqi1Z06+RcrMkDg= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go4org/hashtriemap v0.0.0-20251130024219-545ba229f689 h1:0psnKZ+N2IP43/SZC8SKx6OpFJwLmQb9m9QyV9BC2f8= +github.com/go4org/hashtriemap v0.0.0-20251130024219-545ba229f689/go.mod h1:OGmRfY/9QEK2P5zCRtmqfbCF283xPkU2dvVA4MvbvpI= +github.com/go4org/plan9netshell v0.0.0-20250324183649-788daa080737 h1:cf60tHxREO3g1nroKr2osU3JWZsJzkfi7rEg+oAB0Lo= +github.com/go4org/plan9netshell v0.0.0-20250324183649-788daa080737/go.mod h1:MIS0jDzbU/vuM9MC4YnBITCv+RYuTRq8dJzmCrFsK9g= +github.com/godbus/dbus/v5 v5.2.2 h1:TUR3TgtSVDmjiXOgAAyaZbYmIeP3DPkld3jgKGV8mXQ= +github.com/godbus/dbus/v5 v5.2.2/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 h1:f+oWsMOmNPc8JmEHVZIycC7hBoQxHH9pNKQORJNozsQ= +github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8/go.mod h1:wcDNUvekVysuuOpQKo3191zZyTpiI6se1N1ULghS0sw= +github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= +github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.4 h1:awZRf9FwOeTunQmHoDYSHJps3ie6f1UlhS1fOdPEt1I= +github.com/google/go-tpm v0.9.4/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806 h1:wG8RYIyctLhdFk6Vl1yPGtSRtwGpVkWyZww1OCil2MI= +github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hdevalence/ed25519consensus v0.2.0 h1:37ICyZqdyj0lAZ8P4D1d1id3HqbbG1N3iBb1Tb4rdcU= +github.com/hdevalence/ed25519consensus v0.2.0/go.mod h1:w3BHWjwJbFU29IRHL1Iqkw3sus+7FctEyM4RqDxYNzo= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/illarion/gonotify/v3 v3.0.2 h1:O7S6vcopHexutmpObkeWsnzMJt/r1hONIEogeVNmJMk= +github.com/illarion/gonotify/v3 v3.0.2/go.mod h1:HWGPdPe817GfvY3w7cx6zkbzNZfi3QjcBm/wgVvEL1U= +github.com/insomniacslk/dhcp v0.0.0-20240129002554-15c9b8791914 h1:kD8PseueGeYiid/Mmcv17Q0Qqicc4F46jcX22L/e/Hs= +github.com/insomniacslk/dhcp v0.0.0-20240129002554-15c9b8791914/go.mod h1:3A9PQ1cunSDF/1rbTq99Ts4pVnycWg+vlPkfeD2NLFI= +github.com/jellydator/ttlcache/v3 v3.1.0 h1:0gPFG0IHHP6xyUyXq+JaD8fwkDCqgqwohXNJBcYE71g= +github.com/jellydator/ttlcache/v3 v3.1.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jsimonetti/rtnetlink v1.4.1 h1:JfD4jthWBqZMEffc5RjgmlzpYttAVw1sdnmiNaPO3hE= +github.com/jsimonetti/rtnetlink v1.4.1/go.mod h1:xJjT7t59UIZ62GLZbv6PLLo8VFrostJMPBAheR6OM8w= +github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ= +github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a h1:+RR6SqnTkDLWyICxS1xpjCi/3dhyV+TgZwA6Ww3KncQ= +github.com/kortschak/wol v0.0.0-20200729010619-da482cc4850a/go.mod h1:YTtCCM3ryyfiu4F7t8HQ1mxvp1UBdWM2r6Xa+nGWvDk= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw= +github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42 h1:A1Cq6Ysb0GM0tpKMbdCXCIfBclan4oHk1Jb+Hrejirg= +github.com/mdlayher/netlink v1.7.3-0.20250113171957-fbb4dce95f42/go.mod h1:BB4YCPDOzfy7FniQ/lxuYQ3dgmM2cZumHbK8RpTjN2o= +github.com/mdlayher/sdnotify v1.0.0 h1:Ma9XeLVN/l0qpyx1tNeMSeTjCPH6NtuD6/N9XdTlQ3c= +github.com/mdlayher/sdnotify v1.0.0/go.mod h1:HQUmpM4XgYkhDLtd+Uad8ZFK1T9D5+pNxnXQjCeJlGE= +github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI= +github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI= +github.com/miekg/dns v1.1.58 h1:ca2Hdkz+cDg/7eNF6V56jjzuZ4aCAE+DbVkILdQWG/4= +github.com/miekg/dns v1.1.58/go.mod h1:Ypv+3b/KadlvW9vJfXOTf300O4UqaHFzFCuHz+rPkBY= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= +github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8= +github.com/pierrec/lz4/v4 v4.1.26 h1:GrpZw1gZttORinvzBdXPUXATeqlJjqUG/D87TKMnhjY= +github.com/pierrec/lz4/v4 v4.1.26/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4= +github.com/pires/go-proxyproto v0.8.1 h1:9KEixbdJfhrbtjpz/ZwCdWDD2Xem0NZ38qMYaASJgp0= +github.com/pires/go-proxyproto v0.8.1/go.mod h1:ZKAAyp3cgy5Y5Mo4n9AlScrkCZwUy0g3Jf+slqQVcuU= +github.com/pkg/sftp v1.13.6 h1:JFZT4XbOU7l77xGSpOdW+pwIMqP044IyjXX6FGyEKFo= +github.com/pkg/sftp v1.13.6/go.mod h1:tz1ryNURKu77RL+GuCzmoJYxQczL3wLNNpPWagdg4Qk= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.69.0 h1:OA85nJQS/T/MaYh/Q2CcgDKSGWqNIgrBDvDH85CuiNk= +github.com/prometheus/common v0.69.0/go.mod h1:ZzL3f6u94qUxh9p+tJTrF+FvBS1XXbbRAZCQkytAL0Y= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= +github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= +github.com/studio-b12/gowebdav v0.13.0 h1:OcwSg6IQHOFNdYHn3bPOHwSE8looG8N56Y5xTT1asqQ= +github.com/studio-b12/gowebdav v0.13.0/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE= +github.com/tailscale/certstore v0.1.1-0.20260409135935-3638fb84b77d h1:JcGKBZAL7ePLwOhUdN8qGQZlP5GueEiIZwY7R62pejE= +github.com/tailscale/certstore v0.1.1-0.20260409135935-3638fb84b77d/go.mod h1:XrBNfAFN+pwoWuksbFS9Ccxnopa15zJGgXRFN90l3K4= +github.com/tailscale/gliderssh v0.3.4-0.20260716005906-1a0f895faf28 h1:Azz5ILxxVsHN/KjIu3wkJPAmmtiijucZw4Ax5Ye8n+s= +github.com/tailscale/gliderssh v0.3.4-0.20260716005906-1a0f895faf28/go.mod h1:wn16Km1EZOX4UEAyaZa3dBwfFGOJ7neck40NcwosJUw= +github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 h1:Gzfnfk2TWrk8Jj4P4c1a3CtQyMaTVCznlkLZI++hok4= +github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55/go.mod h1:4k4QO+dQ3R5FofL+SanAUZe+/QfeK0+OIuwDIRu2vSg= +github.com/tailscale/golang-x-crypto v0.0.0-20260720153645-2ba0bf7866ed h1:uyvHhX1FQada0vVk8CSHa4tJT96EEAkTypaYz8Tq5Nc= +github.com/tailscale/golang-x-crypto v0.0.0-20260720153645-2ba0bf7866ed/go.mod h1:NC3xRCu4UR+m4n6ix8b6oLLbHa820Y0StbOQEdWTDo0= +github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd h1:Rf9uhF1+VJ7ZHqxrG8pJ6YacmHvVCmByDmGbAWCc/gA= +github.com/tailscale/hujson v0.0.0-20260302212456-ecc657c15afd/go.mod h1:EbW0wDK/qEUYI0A5bqq0C2kF8JTQwWONmGDBbzsxxHo= +github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7 h1:uFsXVBE9Qr4ZoF094vE6iYTLDl0qCiKzYXlL6UeWObU= +github.com/tailscale/netlink v1.1.1-0.20240822203006-4d49adab4de7/go.mod h1:NzVQi3Mleb+qzq8VmcWpSkcSYxXIg0DkI6XDzpVkhJ0= +github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc h1:24heQPtnFR+yfntqhI3oAu9i27nEojcQ4NuBQOo5ZFA= +github.com/tailscale/peercred v0.0.0-20250107143737-35a0c7bd7edc/go.mod h1:f93CXfllFsO9ZQVq+Zocb1Gp4G5Fz0b0rXHLOzt/Djc= +github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976 h1:UBPHPtv8+nEAy2PD8RyAhOYvau1ek0HDJqLS/Pysi14= +github.com/tailscale/web-client-prebuilt v0.0.0-20250124233751-d4cd19a26976/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ= +github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M= +github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y= +github.com/tailscale/wireguard-go v0.0.0-20260715223240-2e01ba5b00f0 h1:CnIEL2n7Xql6Ux1k+Vu5S5ubDHCT/kxFgkKCY8FjefU= +github.com/tailscale/wireguard-go v0.0.0-20260715223240-2e01ba5b00f0/go.mod h1:6SerzcvHWQchKO2BfNdmquA77CHSECZuFl+D9fp4RnI= +github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e h1:zOGKqN5D5hHhiYUp091JqK7DPCqSARyUfduhGUY8Bek= +github.com/tailscale/xnet v0.0.0-20240729143630-8497ac4dab2e/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg= +github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA= +github.com/tc-hib/winres v0.2.1/go.mod h1:C/JaNhH3KBvhNKVbvdlDWkbMDO9H4fKKDaN7/07SSuk= +github.com/u-root/u-root v0.14.0 h1:Ka4T10EEML7dQ5XDvO9c3MBN8z4nuSnGjcd1jmU2ivg= +github.com/u-root/u-root v0.14.0/go.mod h1:hAyZorapJe4qzbLWlAkmSVCJGbfoU9Pu4jpJ1WMluqE= +github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701 h1:pyC9PaHYZFgEKFdlp3G8RaCKgVpHZnecvArXvPXcFkM= +github.com/u-root/uio v0.0.0-20240224005618-d2acac8f3701/go.mod h1:P3a5rG4X7tI17Nn3aOIAYr5HbIMukwXG0urG0WuL8OA= +github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= +github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go4.org/mem v0.0.0-20240501181205-ae6ca9944745 h1:Tl++JLUCe4sxGu8cTpDzRLd3tN7US4hOxG5YpKCzkek= +go4.org/mem v0.0.0-20240501181205-ae6ca9944745/go.mod h1:reUoABIJ9ikfM5sgtSF3Wushcza7+WeD01VB9Lirh3g= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= +go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f h1:phY1HzDcf18Aq9A8KkmRtY9WvOFIxN8wgfvy6Zm1DV8= +golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/image v0.41.0 h1:8wS72eGJMJaBxK6okTzd4WaXumUlTVlb753MlsSvTCo= +golang.org/x/image v0.41.0/go.mod h1:uIc348UZMSvS5Z65CVZ7iDPaNobNFEPeJ4kbqTOszmA= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= +golang.org/x/net v0.61.0 h1:RR+6j/BTBCrggOLugxa+66R8FvoN6MHlzDlZIbN9YFw= +golang.org/x/net v0.61.0/go.mod h1:WgDCOTH8iwtB66T4/GfI94AHo/hhdb7Hah9NDseQfiI= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/term v0.47.0 h1:cpyOwQkr2Nzxe2p6KnrTG7Ei2wtzjsuSjhpXXepZ/BM= +golang.org/x/term v0.47.0/go.mod h1:efAfAuG6RKABvg1ArGebRkvX6LXul14SM67o2e4mUCY= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.51.0 h1:k4Xc/1Om9jwkBJBo4NVLMSARBoWtK10mx+W5BnXCeAI= +golang.org/x/tools v0.51.0/go.mod h1:9eEncMayCV6zRMGhR5eZEC2iBx98qWcF1HZ9Z7wJOoA= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= +golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= +golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446 h1:cqHQ3AycTHvM2R7ikgyX57D+XvtcSnGylsLkOVhta/w= +golang.zx2c4.com/wireguard v0.0.0-20260522210424-ecfc5a8d5446/go.mod h1:rpwXGsirqLqN2L0JDJQlwOboGHmptD5ZD6T2VmcqhTw= +golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE= +golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gvisor.dev/gvisor v0.0.0-20260224225140-573d5e7127a8 h1:Zy8IV/+FMLxy6j6p87vk/vQGKcdnbprwjTxc8UiUtsA= +gvisor.dev/gvisor v0.0.0-20260224225140-573d5e7127a8/go.mod h1:QkHjoMIBaYtpVufgwv3keYAbln78mBoCuShZrPrer1Q= +honnef.co/go/tools v0.7.0 h1:w6WUp1VbkqPEgLz4rkBzH/CSU6HkoqNLp6GstyTx3lU= +honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= +howett.net/plist v1.0.0 h1:7CrbWYbPPO/PyNy38b2EB/+gYbjCe2DXBxgtOOZbSQM= +howett.net/plist v1.0.0/go.mod h1:lqaXoTrLY4hg8tnEzNru53gicrbv7rrk+2xJA/7hw9g= +software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k= +software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI= +tailscale.com v1.102.5 h1:2jK9VxQU4Vq/tyR7f2U2NqINxU0pV7R14TDBFsfguHE= +tailscale.com v1.102.5/go.mod h1:47bv91Xbg4K1p5wti7F1dmKvUVWV5BXF78d9EWJ+d6c= diff --git a/tunnel/main.go b/tunnel/main.go index 64be7d11c..81879dedf 100644 --- a/tunnel/main.go +++ b/tunnel/main.go @@ -1,5 +1,5 @@ // Command tunnel is a relay client supervisor that runs as a pod and -// configures/starts/supervises a third-party relay process (frp, Tailscale, playit). +// configures/starts/supervises frp and playit, or embeds a Tailscale node. // The binary picks its behavior from the TUNNEL_TYPE env var and handles provider-specific // config rendering, credential management, and child process supervision. // @@ -21,7 +21,6 @@ package main import ( "context" - "encoding/json" "errors" "fmt" "log" @@ -50,45 +49,8 @@ const ( // be compile-time string constants -- not variables -- for buildCommand // below to pass gosec's G204 subprocess-argument check structurally // (see the doc comment on buildCommand). - frpConfigPath = "/tmp/gameplane-tunnel-frpc.toml" - tailscaleConfigPath = "/tmp/gameplane-tunnel-tailscaled.json" - playitAuthPath = "/tmp/gameplane-tunnel-playit-auth" - - // tailscaleSocketPath is the control socket tailscaled listens on, - // set explicitly via --socket (both when starting tailscaled and when - // later talking to it with the `tailscale` CLI) because the non-root - // distroless image can't create tailscaled's Linux default - // (/run/tailscale/tailscaled.sock). This mirrors playitSocketPath's - // --socket-path relocation in playit_reporter.go for the same reason. - tailscaleSocketPath = "/tmp/gameplane-tunnel-tailscaled.sock" - - // tailscaleAuthKeyPath holds the Tailscale auth key, mode 0600, when - // tags are requested. The key then stays out of tailscaled's config so - // that registration can happen through `tailscale up - // --auth-key=file: --advertise-tags=...` instead (see - // renderTailscaleConfig and registerTailscaleOnce). Passing a file keeps - // the key out of argv. It lives in /tmp next to the other rendered - // files and is removed when run returns. - tailscaleAuthKeyPath = "/tmp/gameplane-tunnel-tailscale-authkey" -) - -// Timing for registering a tagged Tailscale device (registerTailscaleOnce). -const ( - // tailscaleReadyTimeout bounds how long waitForTailscaled polls for - // tailscaled's socket to answer with a settled BackendState before - // giving up on registering the device for this supervisor run. It is - // generous because tailscaled may be restarting under the backoff loop - // in run, but bounded so a daemon that never comes up doesn't leave a - // goroutine polling forever. - tailscaleReadyTimeout = 2 * time.Minute - - // tailscaleReadyPollInterval is how often waitForTailscaled polls - // `tailscale status --json`. - tailscaleReadyPollInterval = 2 * time.Second - - // tailscaleUpTimeout bounds a single `tailscale up`, which includes a - // round trip to Tailscale's control plane. - tailscaleUpTimeout = 2 * time.Minute + frpConfigPath = "/tmp/gameplane-tunnel-frpc.toml" + playitAuthPath = "/tmp/gameplane-tunnel-playit-auth" ) // credentialsDir is the directory holding the mounted Secret's credential @@ -146,7 +108,7 @@ type Config struct { // Tailscale-specific TailscaleHostname string TailscaleTags string // comma-separated - BackingServicePorts string // "name:port,..." format + BackingServicePorts string // Tailscale: name:port:protocol; playit: name:port // Playit-specific PlayitTunnelName string @@ -237,11 +199,8 @@ func run(ctx context.Context, cfg Config) error { return fmt.Errorf("read credentials: %w", err) } - // tailscale with tags writes the auth key to its own file (see - // renderTailscaleConfig); remove it on the way out, including when - // rendering the config itself fails after the key was written. if cfg.TunnelType == "tailscale" { - defer func() { _ = os.Remove(tailscaleAuthKeyPath) }() + return runTailscale(ctx, cfg, creds) } // Render the provider's config file. @@ -267,25 +226,6 @@ func run(ctx context.Context, cfg Config) error { }() } - // tailscale with tags: renderTailscaleConfig left the auth key out of - // tailscaled's config, so register the device via `tailscale up` over - // tailscaled's socket (see registerTailscaleOnce). This keeps retrying - // until tailscaled settles and registration succeeds, or until ctx is - // cancelled: the login and the granted tags persist in tailscaled's - // local state file (/tmp/tailscale.state, --state in buildCommand) - // across a tailscaled restart that reuses it. Without tags, tailscaled - // logs in from its config by itself and nothing runs here. - if cfg.TunnelType == "tailscale" { - if tags, err := parseTailscaleTags(cfg.TailscaleTags); err == nil && len(tags) > 0 { - regCtx, stopRegistrar := context.WithCancel(ctx) - waitRegistrar := startTailscaleRegistrar(regCtx, cfg.TailscaleHostname, tags) - defer func() { - stopRegistrar() - waitRegistrar() - }() - } - } - // Supervise the relay process with exponential backoff on exit. backoff := &exponentialBackoff{} for { @@ -357,8 +297,6 @@ func renderConfig(cfg Config, credential string) (string, error) { switch cfg.TunnelType { case "frp": return renderFrpConfig(cfg, credential) - case "tailscale": - return renderTailscaleConfig(cfg.TailscaleHostname, credential, cfg.TailscaleTags) case "playit": return renderPlayitConfig(credential) default: @@ -423,110 +361,6 @@ remotePort = %s return frpConfigPath, nil } -// tailscaledConfig mirrors (the subset we use of) tailscaled's own -// ConfigVAlpha struct (ipn/conf.go in tailscale/tailscale), consumed via -// `tailscaled --config=`. This is tailscaled's documented declarative -// config file (https://tailscale.com/docs/reference/tailscaled/tailescaled-config-file); -// "alpha0" is the only version value it currently accepts, and the docs -// note the schema may still change. It is what lets a bare `tailscaled` -// process (no containerboot, no separate `tailscale up`) authenticate and -// set its hostname in a single exec: containerboot's TS_AUTHKEY env var and -// `tailscale up --hostname=` are containerboot/CLI conveniences that build -// on top of this same mechanism, but neither is read by tailscaled itself, -// which is why the previous TS_AUTHKEY env var and --hostname flag here -// were both no-ops (the latter would have made tailscaled reject the flag -// and exit immediately). -type tailscaledConfig struct { - Version string `json:"version"` - AuthKey string `json:"authKey,omitempty"` - Hostname string `json:"hostname,omitempty"` -} - -// renderTailscaleConfig generates tailscaled's declarative config file (see -// tailscaledConfig for the shape) at the fixed tailscaleConfigPath (see the -// const block near the top of the file for why the path is fixed rather -// than a random os.CreateTemp name). -// -// It never writes a "tags" key: tailscaled's alpha0 declarative config -// (ipn.ConfigVAlpha, confirmed against tailscale/tailscale v1.102.4's -// ipn/conf.go) has no field for ACL tags, and its loader (ipn/conffile) -// decodes with encoding/json's DisallowUnknownFields, so a "tags" key made -// tailscaled refuse the config and the tunnel never came up. -// -// ACL tags can only be requested when the device registers (`tailscale up -// --advertise-tags`; `tailscale set` has no such flag, and changing tags on -// a running node needs a fresh login), so the file's content depends on -// whether tagsStr yields any valid tag (see parseTailscaleTags): -// -// - No tags (tagsStr empty, blank, or invalid): the auth key and hostname -// go into the config and tailscaled logs in by itself, exactly as it -// did before tags were supported. Nothing else runs. -// - Tags: the config carries only the hostname and "locked": false, and -// the auth key goes to its own 0600 file at tailscaleAuthKeyPath. -// Without an auth key in the config, tailscaled starts in NeedsLogin -// and waits; startTailscaleRegistrar then registers it once over the -// control socket with `tailscale up --auth-key=file: -// --advertise-tags=...`. "locked": false is required for that: -// ipn.ConfigVAlpha's Locked defaults to true, and a locked config makes -// tailscaled reject every CLI prefs change ("config file is locked"). -// -// The JSON is built from a map rather than by marshaling the tailscaledConfig -// struct directly: gosec's G117 rule flags any exported struct field whose -// name or JSON tag matches a secret-like pattern (AuthKey / "authKey" both -// do) when it's passed straight to json.Marshal. That's a real signal in -// general, but here marshaling the key is the entire point -- tailscaled -// reads its auth key from exactly this file (see the type's doc comment) -- -// so there's nothing to fix behaviorally. G117 only inspects struct field -// tags/names, not map keys, so building the object as a map sidesteps the -// false positive structurally instead of suppressing it. tailscaledConfig -// itself is kept as the documented shape and is what tests decode the -// written file back into (see TestRenderTailscaleConfig). -func renderTailscaleConfig(hostname, authKey, tagsStr string) (string, error) { - tags, tagsErr := parseTailscaleTags(tagsStr) - renderTailscaleConfigLogTags(tagsStr, tags, tagsErr) - - fields := map[string]any{"version": "alpha0"} - if hostname != "" { - fields["hostname"] = hostname - } - if len(tags) > 0 { - fields["locked"] = false - if err := os.WriteFile(tailscaleAuthKeyPath, []byte(authKey), 0o600); err != nil { - return "", fmt.Errorf("write tailscale auth key file: %w", err) - } - } else if authKey != "" { - fields["authKey"] = authKey - } - - data, err := json.Marshal(fields) - if err != nil { - return "", fmt.Errorf("marshal tailscaled config: %w", err) - } - - if err := os.WriteFile(tailscaleConfigPath, data, 0o600); err != nil { - return "", fmt.Errorf("write tailscaled config: %w", err) - } - - return tailscaleConfigPath, nil -} - -// renderTailscaleConfigLogTags logs what renderTailscaleConfig does with a -// non-empty TAILSCALE_TAGS: either that the tags will be requested when the -// device registers (and what the tailnet ACL has to allow for that), or, -// when parseTailscaleTags rejected them, that they are ignored and the -// device registers untagged. Split out from renderTailscaleConfig so a test -// can assert on the message via log.SetOutput. -func renderTailscaleConfigLogTags(tagsStr string, tags []string, tagsErr error) { - switch { - case tagsStr == "": - return - case tagsErr != nil: - log.Printf("tailscale tunnel: ignoring TAILSCALE_TAGS=%q: %v; the device registers untagged", tagsStr, tagsErr) - case len(tags) > 0: - log.Printf("tailscale tunnel: TAILSCALE_TAGS=%q is set; tags %s will be requested when the device registers via `tailscale up --advertise-tags` (the tailnet ACL must grant tagOwners for them to the auth key's owner)", tagsStr, strings.Join(tags, ",")) - } -} - // tailscaleTagNameChar reports whether b may appear in a tag name after the // "tag:" prefix: ASCII letters, digits and '-', per tailcfg.CheckTag. func tailscaleTagNameChar(b byte) bool { @@ -542,8 +376,8 @@ func tailscaleTagNameChar(b byte) bool { // Every resulting tag must pass the same rule as tailscale's // tailcfg.CheckTag: "tag:" followed by an ASCII letter, then only ASCII // letters, digits or '-'. The first tag that fails is returned as an error -// and no tags are returned, so a bad value can never reach the -// `tailscale up` argv (see tailscaleUpArgs). It returns (nil, nil) when no +// and no tags are returned, so a bad value can never reach tsnet registration. +// It returns (nil, nil) when no // non-empty entry remains. func parseTailscaleTags(tagsStr string) ([]string, error) { var tags []string @@ -577,274 +411,6 @@ func parseTailscaleTags(tagsStr string) ([]string, error) { return tags, nil } -// tailscaleUpArgs builds the argv (excluding the binary itself) for the -// one-off `tailscale up` that registers the device when tags are requested: -// -// --socket= up --reset --auth-key=file: --hostname= [--advertise-tags=] -// -// Flag names and forms are confirmed against tailscale v1.102.4 (the -// version Dockerfile.tailscale ships): --socket is a root flag -// (cmd/tailscale/cli/cli.go); --auth-key reads the key from a file when the -// value starts with "file:" (up.go resolveValueFromFile), so the key never -// appears in argv or /proc//cmdline; --advertise-tags takes a -// comma-separated list. --reset resets every setting not named on the -// command line to its default. Without it, `tailscale up` on a node with -// saved prefs (a reused state file) refuses to run unless every non-default -// setting is repeated on the command line (checkForAccidentalSettingReverts). -// This supervisor sets nothing beyond the hostname, so the defaults are what -// it wants anyway. With no tags (the untagged fallback) --advertise-tags is -// left out, which with --reset clears any previously requested tags. -// -// Every value is a single "--flag=value" argv element, so no value can be -// read as a separate flag, and nothing goes through a shell. tags must come -// from parseTailscaleTags, which has already validated each one. -func tailscaleUpArgs(hostname string, tags []string) []string { - args := []string{ - "--socket=" + tailscaleSocketPath, - "up", - "--reset", - "--auth-key=file:" + tailscaleAuthKeyPath, - "--hostname=" + hostname, - } - if len(tags) > 0 { - args = append(args, "--advertise-tags="+strings.Join(tags, ",")) - } - return args -} - -// runTailscaleCLI runs the `tailscale` CLI (not tailscaled) against args -// and returns its stdout. It is a package-level variable, rather than a -// plain function, solely so tests can substitute a fake in place of -// exec'ing the real binary (see credentialsDir above for the same -// pattern). Production code never reassigns it; only tests do. -var runTailscaleCLI = execTailscaleCLI - -// execTailscaleCLI is runTailscaleCLI's real implementation. It execs the -// fixed /usr/local/bin/tailscale binary (shipped in the tailscale provider -// image alongside tailscaled; see Dockerfile.tailscale) with args, tied to -// ctx so supervisor shutdown also stops it. The binary path is a literal -// and args is a plain argv slice built only by this file -// (tailscaleStatusArgs and tailscaleUpArgs), never passed through a shell. -// The only non-constant values in it are the hostname, as a single -// "--hostname=" element, and tags already validated by parseTailscaleTags. -// The auth key is never in args; `tailscale up` reads it from a file. -// -// The argv is appended to cmd.Args after exec.CommandContext rather than -// passed to it, so gosec's G204 check sees a call with only literal -// arguments. The reasoning above is the actual justification. -func execTailscaleCLI(ctx context.Context, args ...string) ([]byte, error) { - cmd := exec.CommandContext(ctx, "/usr/local/bin/tailscale") - cmd.Args = append(cmd.Args, args...) - out, err := cmd.Output() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) && len(exitErr.Stderr) > 0 { - return out, fmt.Errorf("%w: %s", err, strings.TrimSpace(string(exitErr.Stderr))) - } - return out, err - } - return out, nil -} - -// tailscaleStatusArgs builds the argv for `tailscale status --json` over -// tailscaleSocketPath. --peers=false keeps the output small: only BackendState and Self -// are read. -func tailscaleStatusArgs() []string { - return []string{"--socket=" + tailscaleSocketPath, "status", "--json", "--peers=false"} -} - -// tailscaleStatus is the subset of `tailscale status --json`'s output this -// supervisor reads, confirmed against tailscale v1.102.4's -// ipn/ipnstate.Status: BackendState is an ipn.State string, and -// Self.Tags lists the ACL tags the control plane has granted to this node, -// fully qualified ("tag:x"). -type tailscaleStatus struct { - BackendState string `json:"BackendState"` - Self *struct { - Tags []string `json:"Tags"` - } `json:"Self"` -} - -// hasExactTags reports whether the node is Running and its granted tags -// are exactly want, in any order, as a set. Both granted and requested -// tags are deduplicated for comparison. -func (st tailscaleStatus) hasExactTags(want []string) bool { - if st.BackendState != "Running" || st.Self == nil { - return false - } - have := make(map[string]bool) - for _, tag := range st.Self.Tags { - have[tag] = true - } - wantSet := make(map[string]bool) - for _, tag := range want { - wantSet[tag] = true - } - if len(have) != len(wantSet) { - return false - } - for tag := range wantSet { - if !have[tag] { - return false - } - } - return true -} - -// tailscaleSettledStates are the BackendState values at which tailscaled -// has finished starting up and `tailscale up` can act on it. The others are -// transient: "NoState" before the config and state file are loaded, and -// "Starting" while an existing login is connecting. Running `up` during -// "Starting" would make a node that already has the right tags log in -// again for nothing. -var tailscaleSettledStates = map[string]bool{ - "NeedsLogin": true, - "NeedsMachineAuth": true, - "Stopped": true, - "Running": true, -} - -// queryTailscaleStatus runs `tailscale status --json` once over -// tailscaleSocketPath and decodes it. -func queryTailscaleStatus(ctx context.Context) (tailscaleStatus, error) { - var st tailscaleStatus - out, err := runTailscaleCLI(ctx, tailscaleStatusArgs()...) - if err != nil { - return st, err - } - if err := json.Unmarshal(out, &st); err != nil { - return st, fmt.Errorf("parse tailscale status: %w", err) - } - return st, nil -} - -// waitForTailscaled polls `tailscale status --json` over -// tailscaleSocketPath every tailscaleReadyPollInterval until tailscaled's -// socket answers with a settled BackendState (see tailscaleSettledStates), -// ctx is cancelled, or tailscaleReadyTimeout elapses. It returns the -// settled status, or the last error seen. -func waitForTailscaled(ctx context.Context) (tailscaleStatus, error) { - deadline := time.Now().Add(tailscaleReadyTimeout) - var lastErr error - for { - st, err := queryTailscaleStatus(ctx) - switch { - case err != nil: - lastErr = err - case tailscaleSettledStates[st.BackendState]: - return st, nil - default: - lastErr = fmt.Errorf("tailscaled not ready yet (BackendState=%q)", st.BackendState) - } - - if ctx.Err() != nil { - return tailscaleStatus{}, ctx.Err() - } - if time.Now().After(deadline) { - return tailscaleStatus{}, fmt.Errorf("timed out after %v waiting for tailscaled: %w", tailscaleReadyTimeout, lastErr) - } - select { - case <-ctx.Done(): - return tailscaleStatus{}, ctx.Err() - case <-time.After(tailscaleReadyPollInterval): - } - } -} - -// runTailscaleUp runs one bounded `tailscale up` with the argv from -// tailscaleUpArgs. -func runTailscaleUp(ctx context.Context, hostname string, tags []string) error { - upCtx, cancel := context.WithTimeout(ctx, tailscaleUpTimeout) - defer cancel() - _, err := runTailscaleCLI(upCtx, tailscaleUpArgs(hostname, tags)...) - return err -} - -// registerTailscaleOnce registers the device with the tailnet, requesting -// tags, for a tagged tunnel. In that case renderTailscaleConfig left the -// auth key out of tailscaled's config, so tailscaled waits in NeedsLogin -// until this runs. -// -// 1. Wait for tailscaled's socket to answer with a settled state -// (waitForTailscaled), retrying on timeout. -// 2. If the node is already Running with exactly these tags (a state file -// reused across a restart), do nothing. -// 3. Otherwise run `tailscale up ... --advertise-tags=` once. -// 4. If that fails, most often because the tailnet ACL doesn't grant -// tagOwners for the tags to the auth key's owner, log it. If the node -// is not Running afterwards, run `tailscale up` once more without -// --advertise-tags, so the tunnel still comes up, untagged. -// -// Every failure is logged and none of them stops the supervisor or the -// relay process, so an ACL problem only the tailnet admin can fix never -// turns into a crash loop. -func registerTailscaleOnce(ctx context.Context, hostname string, tags []string) { - var st tailscaleStatus - var err error - for { - st, err = waitForTailscaled(ctx) - if err == nil { - break - } - if ctx.Err() != nil { - return - } - log.Printf("tailscale tunnel: still waiting for tailscaled to settle: %v", err) - // Retry after a brief delay without spinning - select { - case <-time.After(5 * time.Second): - continue - case <-ctx.Done(): - return - } - } - - joined := strings.Join(tags, ",") - if st.hasExactTags(tags) { - log.Printf("tailscale tunnel: device is already running with tags %s; skipping `tailscale up`", joined) - return - } - - err = runTailscaleUp(ctx, hostname, tags) - if err == nil { - log.Printf("tailscale tunnel: registered the device with tags %s", joined) - return - } - if ctx.Err() != nil { - return - } - log.Printf("tailscale tunnel: `tailscale up --advertise-tags=%s` failed: %v; the tailnet ACL must grant tagOwners for these tags to the auth key's owner.", joined, err) - - if st, err := queryTailscaleStatus(ctx); err == nil && st.BackendState == "Running" { - if st.Self != nil && len(st.Self.Tags) > 0 { - oldTags := strings.Join(st.Self.Tags, ",") - log.Printf("tailscale tunnel: device is still running with previous tags %s; they remain granted until the tailnet ACL is fixed or they are removed in the tailnet admin console", oldTags) - return - } - log.Printf("tailscale tunnel: device is still running without tags") - return - } - if err := runTailscaleUp(ctx, hostname, nil); err != nil { - if ctx.Err() == nil { - log.Printf("tailscale tunnel: untagged `tailscale up` also failed: %v", err) - } - return - } - log.Printf("tailscale tunnel: registered the device without tags") -} - -// startTailscaleRegistrar launches registerTailscaleOnce in a goroutine -// and returns a function that waits for it to finish, mirroring -// startPlayitReporter's shape in playit_reporter.go. -func startTailscaleRegistrar(ctx context.Context, hostname string, tags []string) (wait func()) { - done := make(chan struct{}) - go func() { - defer close(done) - registerTailscaleOnce(ctx, hostname, tags) - }() - return func() { <-done } -} - // renderPlayitConfig writes the secret key to the fixed playitAuthPath // (see the const block near the top of the file for why the path is fixed // rather than a random os.CreateTemp name) for playitd's --secret-path flag. @@ -886,12 +452,9 @@ func escapeTomlString(s string) string { // is ever reached in the normal main() contract, so the nil case only // happens if a caller builds a Config by hand (see TestBuildCommandUnknownType). // -// All three providers take their secret via the rendered config file -// (see renderConfig and the frpConfigPath/tailscaleConfigPath/ -// playitAuthPath consts) rather than a raw credential argument here -- -// frpc's TOML embeds the token, tailscaled's declarative config JSON embeds -// the auth key and hostname, and playitd's --secret-path points straight at -// the secret file -- so buildCommand itself never needs the raw credential. +// frpc's TOML embeds its token, and playitd's --secret-path points to a +// private credential file. buildCommand never needs the raw credential. +// Tailscale uses the embedded node path and does not call this function. // // The relay binary path and config-file path are both named constants // (rather than parameters or computed variables) so every argument to @@ -917,24 +480,6 @@ func buildCommand(ctx context.Context, cfg Config) *exec.Cmd { // Confirmed against fatedier/frp's cmd/frpc/sub/root.go: "-c"/"--config" // is frpc's real config-file flag. return exec.CommandContext(ctx, "/usr/local/bin/frpc", "-c", frpConfigPath) - case "tailscale": - // Tailscale uses tailscaled (daemon) with flags rather than a config file. - // The pod's hardened securityContext (no NET_ADMIN, no /dev/net/tun device) - // requires userspace networking mode via --tun=userspace-networking flag. - // --config points at the declarative config file (see tailscaledConfig) - // carrying the hostname and, unless tags are requested, the auth key. - // --socket relocates the control socket to a path this non-root image - // can create (see tailscaleSocketPath's doc comment); with tags, - // registerTailscaleOnce registers the device over this same socket - // via the `tailscale` CLI. All flags - // confirmed against tailscale/tailscale's cmd/tailscaled/tailscaled.go - // flag definitions. - return exec.CommandContext(ctx, "/usr/local/bin/tailscaled", - "--tun=userspace-networking", - "--state=/tmp/tailscale.state", - "--config="+tailscaleConfigPath, - "--socket="+tailscaleSocketPath, - ) case "playit": // Confirmed against playit-cloud/playit-agent's packages/playitd/src/bin/playitd.rs // and its own Dockerfile/docker/entrypoint.sh, which exec exactly this diff --git a/tunnel/main_test.go b/tunnel/main_test.go index ce0881de1..05f4784df 100644 --- a/tunnel/main_test.go +++ b/tunnel/main_test.go @@ -1,17 +1,12 @@ package main import ( - "bytes" "context" - "encoding/json" "errors" - "log" "os" "os/exec" "path/filepath" "strings" - "sync" - "sync/atomic" "testing" "time" ) @@ -76,7 +71,7 @@ func TestLoadConfigTailscale(t *testing.T) { "TAILSCALE_HOSTNAME": "my-game", "TAILSCALE_TAGS": "tag:gameplane,tag:game", "BACKING_SERVICE_DNS": "my-server.games.svc", - "BACKING_SERVICE_PORTS": "game:25565", + "BACKING_SERVICE_PORTS": "game:25565:tcp", })) if err != nil { t.Fatalf("loadConfig() error = %v", err) @@ -419,248 +414,6 @@ func TestRenderFrpConfigInvalidProtocol(t *testing.T) { } } -func TestRenderTailscaleConfig(t *testing.T) { - path, err := renderTailscaleConfig("my-game", "test-auth-key", "") - if err != nil { - t.Fatalf("renderTailscaleConfig() error = %v", err) - } - defer os.Remove(path) - - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read config file: %v", err) - } - - var got tailscaledConfig - if err := json.Unmarshal(data, &got); err != nil { - t.Fatalf("tailscaled config file is not valid JSON: %v (content: %s)", err, data) - } - if got.Version != "alpha0" { - t.Errorf("Version = %q, want %q", got.Version, "alpha0") - } - if got.AuthKey != "test-auth-key" { - t.Errorf("AuthKey = %q, want %q", got.AuthKey, "test-auth-key") - } - if got.Hostname != "my-game" { - t.Errorf("Hostname = %q, want %q", got.Hostname, "my-game") - } -} - -func TestRenderTailscaleConfigNoHostname(t *testing.T) { - path, err := renderTailscaleConfig("", "test-auth-key", "") - if err != nil { - t.Fatalf("renderTailscaleConfig() error = %v", err) - } - defer os.Remove(path) - - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read config file: %v", err) - } - - var got tailscaledConfig - if err := json.Unmarshal(data, &got); err != nil { - t.Fatalf("tailscaled config file is not valid JSON: %v", err) - } - if got.Hostname != "" { - t.Errorf("Hostname = %q, want empty", got.Hostname) - } - if got.AuthKey != "test-auth-key" { - t.Errorf("AuthKey = %q, want %q", got.AuthKey, "test-auth-key") - } -} - -// removeTailscaleRenderedFiles deletes the files renderTailscaleConfig can -// write, before and after a test, so one test's auth key file can't make -// another test's "not written" assertion pass or fail by accident. -func removeTailscaleRenderedFiles(t *testing.T) { - t.Helper() - _ = os.Remove(tailscaleConfigPath) - _ = os.Remove(tailscaleAuthKeyPath) - t.Cleanup(func() { - _ = os.Remove(tailscaleConfigPath) - _ = os.Remove(tailscaleAuthKeyPath) - }) -} - -// readTailscaleConfigMap decodes the rendered tailscaled config into a -// generic map, so a test can check for keys tailscaledConfig doesn't -// declare ("tags", "locked"). -func readTailscaleConfigMap(t *testing.T, path string) map[string]any { - t.Helper() - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read config file: %v", err) - } - var got map[string]any - if err := json.Unmarshal(data, &got); err != nil { - t.Fatalf("tailscaled config is not valid JSON: %v (content: %s)", err, data) - } - return got -} - -// TestRenderTailscaleConfigWithTagsOmitsTags asserts what renderTailscaleConfig -// writes when TAILSCALE_TAGS holds valid tags: -// - no "tags" key: tailscaled's alpha0 declarative config (ipn.ConfigVAlpha) -// has no field for ACL tags, and its loader (ipn/conffile) decodes with -// DisallowUnknownFields, so the key would make tailscaled refuse to start; -// - no "authKey" key, so tailscaled waits in NeedsLogin for -// registerTailscaleOnce's `tailscale up --advertise-tags` instead of -// logging in untagged by itself; -// - "locked": false, without which tailscaled rejects that `tailscale up` -// ("config file is locked"); -// - the auth key in tailscaleAuthKeyPath, mode 0600, for -// `tailscale up --auth-key=file:`. -// -// Decoding into a generic map (rather than tailscaledConfig) is what lets -// this test catch a "tags" key reappearing in the JSON. -func TestRenderTailscaleConfigWithTagsOmitsTags(t *testing.T) { - removeTailscaleRenderedFiles(t) - - path, err := renderTailscaleConfig("my-game", "test-auth-key", "tag:gameplane,tag:game") - if err != nil { - t.Fatalf("renderTailscaleConfig() error = %v", err) - } - - got := readTailscaleConfigMap(t, path) - if _, ok := got["tags"]; ok { - t.Errorf("rendered tailscaled config has a %q key = %v; tailscaled's alpha0 config has no such field and DisallowUnknownFields would make tailscaled refuse to start", "tags", got["tags"]) - } - if _, ok := got["authKey"]; ok { - t.Errorf("rendered tailscaled config has an authKey with tags set; tailscaled would log in untagged before `tailscale up --advertise-tags` runs") - } - if locked, ok := got["locked"]; !ok || locked != false { - t.Errorf("locked = %v (present %v), want false; a locked config makes tailscaled reject `tailscale up`", locked, ok) - } - if got["hostname"] != "my-game" { - t.Errorf("hostname = %v, want %q", got["hostname"], "my-game") - } - if got["version"] != "alpha0" { - t.Errorf("version = %v, want %q", got["version"], "alpha0") - } - - key, err := os.ReadFile(tailscaleAuthKeyPath) - if err != nil { - t.Fatalf("read auth key file: %v", err) - } - if string(key) != "test-auth-key" { - t.Errorf("auth key file = %q, want %q", key, "test-auth-key") - } - info, err := os.Stat(tailscaleAuthKeyPath) - if err != nil { - t.Fatalf("stat auth key file: %v", err) - } - if perm := info.Mode().Perm(); perm != 0o600 { - t.Errorf("auth key file mode = %o, want 600", perm) - } -} - -// TestRenderTailscaleConfigNoTags asserts that without tags the config is -// what it was before tags were supported: auth key inside the config, no -// "tags" or "locked" key, and no separate auth key file. -func TestRenderTailscaleConfigNoTags(t *testing.T) { - removeTailscaleRenderedFiles(t) - - path, err := renderTailscaleConfig("my-game", "test-auth-key", "") - if err != nil { - t.Fatalf("renderTailscaleConfig() error = %v", err) - } - - got := readTailscaleConfigMap(t, path) - if _, ok := got["tags"]; ok { - t.Errorf("rendered tailscaled config has a %q key = %v, want none", "tags", got["tags"]) - } - if _, ok := got["locked"]; ok { - t.Errorf("rendered tailscaled config has a %q key = %v, want none without tags", "locked", got["locked"]) - } - if got["authKey"] != "test-auth-key" { - t.Errorf("authKey = %v, want %q", got["authKey"], "test-auth-key") - } - if _, err := os.Stat(tailscaleAuthKeyPath); !os.IsNotExist(err) { - t.Errorf("auth key file %s exists (stat err = %v), want it not written without tags", tailscaleAuthKeyPath, err) - } -} - -// TestRenderTailscaleConfigInvalidTagsRegistersUntagged asserts that tags -// parseTailscaleTags rejects are ignored with a log line, and the config -// falls back to the untagged shape (auth key in the config, no key file), -// so a bad tag can't keep the tunnel from coming up. -func TestRenderTailscaleConfigInvalidTagsRegistersUntagged(t *testing.T) { - removeTailscaleRenderedFiles(t) - - var buf bytes.Buffer - orig := log.Writer() - log.SetOutput(&buf) - defer log.SetOutput(orig) - - path, err := renderTailscaleConfig("my-game", "test-auth-key", "tag:ok,tag:not ok") - if err != nil { - t.Fatalf("renderTailscaleConfig() error = %v", err) - } - - got := readTailscaleConfigMap(t, path) - if got["authKey"] != "test-auth-key" { - t.Errorf("authKey = %v, want %q", got["authKey"], "test-auth-key") - } - if _, ok := got["locked"]; ok { - t.Errorf("rendered tailscaled config has a %q key = %v, want none for rejected tags", "locked", got["locked"]) - } - if _, err := os.Stat(tailscaleAuthKeyPath); !os.IsNotExist(err) { - t.Errorf("auth key file %s exists (stat err = %v), want it not written for rejected tags", tailscaleAuthKeyPath, err) - } - if out := buf.String(); !strings.Contains(out, `ignoring TAILSCALE_TAGS="tag:ok,tag:not ok"`) { - t.Errorf("log output = %q, want it to say the tags are ignored", out) - } -} - -// TestRenderTailscaleConfigLogTagsNotesRegistration asserts that -// renderTailscaleConfig logs an informational line when TAILSCALE_TAGS is -// set: the raw value, and that the tags are requested at registration via -// `tailscale up --advertise-tags`, which needs a tagOwners grant. -func TestRenderTailscaleConfigLogTagsNotesRegistration(t *testing.T) { - removeTailscaleRenderedFiles(t) - - var buf bytes.Buffer - orig := log.Writer() - log.SetOutput(&buf) - defer log.SetOutput(orig) - - if _, err := renderTailscaleConfig("my-game", "test-auth-key", "tag:gameplane,tag:game"); err != nil { - t.Fatalf("renderTailscaleConfig() error = %v", err) - } - - got := buf.String() - if !strings.Contains(got, `TAILSCALE_TAGS="tag:gameplane,tag:game"`) { - t.Errorf("log output = %q, want it to mention the requested tags", got) - } - if !strings.Contains(got, "tailscale up --advertise-tags") || !strings.Contains(got, "tagOwners") { - t.Errorf("log output = %q, want it to name `tailscale up --advertise-tags` and the tagOwners requirement", got) - } -} - -// TestRenderTailscaleConfigLogTagsSilentWhenUnset asserts nothing is -// logged when TAILSCALE_TAGS is empty. -func TestRenderTailscaleConfigLogTagsSilentWhenUnset(t *testing.T) { - removeTailscaleRenderedFiles(t) - - var buf bytes.Buffer - orig := log.Writer() - log.SetOutput(&buf) - defer log.SetOutput(orig) - - if _, err := renderTailscaleConfig("my-game", "test-auth-key", ""); err != nil { - t.Fatalf("renderTailscaleConfig() error = %v", err) - } - - if got := buf.String(); got != "" { - t.Errorf("log output = %q, want empty", got) - } -} - -// ----------------------------------------------------------------------- -// parseTailscaleTags / tailscaleUpArgs / registerTailscaleOnce tests -// ----------------------------------------------------------------------- - func TestParseTailscaleTags(t *testing.T) { tests := []struct { name string @@ -705,329 +458,6 @@ func TestParseTailscaleTagsDeduplicate(t *testing.T) { } } -// TestHasExactTagsRejectsDuplicateWant asserts that duplicate requested tags -// do not fool the exact-set check when the granted set differs. -func TestHasExactTagsRejectsDuplicateWant(t *testing.T) { - st := tailscaleStatus{ - BackendState: "Running", - Self: &struct { - Tags []string `json:"Tags"` - }{ - Tags: []string{"tag:a", "tag:b"}, - }, - } - if st.hasExactTags([]string{"tag:a", "tag:a"}) { - t.Error("hasExactTags([tag:a tag:a]) = true, want false when granted [tag:a tag:b]") - } -} - -func TestTailscaleUpArgs(t *testing.T) { - tests := []struct { - name string - hostname string - tags []string - want []string - }{ - { - name: "with tags", - hostname: "my-game", - tags: []string{"tag:a", "tag:b"}, - want: []string{ - "--socket=/tmp/gameplane-tunnel-tailscaled.sock", - "up", - "--reset", - "--auth-key=file:/tmp/gameplane-tunnel-tailscale-authkey", - "--hostname=my-game", - "--advertise-tags=tag:a,tag:b", - }, - }, - { - name: "untagged fallback omits --advertise-tags", - hostname: "my-game", - tags: nil, - want: []string{ - "--socket=/tmp/gameplane-tunnel-tailscaled.sock", - "up", - "--reset", - "--auth-key=file:/tmp/gameplane-tunnel-tailscale-authkey", - "--hostname=my-game", - }, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := tailscaleUpArgs(tt.hostname, tt.tags) - if !equalArgs(got, tt.want) { - t.Errorf("tailscaleUpArgs() = %v, want %v", got, tt.want) - } - for _, a := range got { - if strings.Contains(a, "test-auth-key") { - t.Errorf("argv %v contains the auth key", got) - } - } - }) - } -} - -func TestTailscaleStatusArgs(t *testing.T) { - want := []string{"--socket=/tmp/gameplane-tunnel-tailscaled.sock", "status", "--json", "--peers=false"} - if got := tailscaleStatusArgs(); !equalArgs(got, want) { - t.Errorf("tailscaleStatusArgs() = %v, want %v", got, want) - } -} - -// stubTailscaleCLI replaces runTailscaleCLI with fn for the duration of the -// test, restoring the real implementation afterward. -func stubTailscaleCLI(t *testing.T, fn func(ctx context.Context, args ...string) ([]byte, error)) { - t.Helper() - orig := runTailscaleCLI - runTailscaleCLI = fn - t.Cleanup(func() { runTailscaleCLI = orig }) -} - -// fakeTailscale is a scripted `tailscale` CLI for registerTailscaleOnce -// tests. Each `status` call returns the next entry of statuses (the last -// one repeats); each `up` call is recorded and returns the next entry of -// upErrs (nil once they run out). -type fakeTailscale struct { - mu sync.Mutex - statuses []string - statusCalls int - upErrs []error - upCalls [][]string -} - -func (f *fakeTailscale) run(_ context.Context, args ...string) ([]byte, error) { - f.mu.Lock() - defer f.mu.Unlock() - if len(args) < 2 || args[0] != "--socket="+tailscaleSocketPath { - return nil, errors.New("unexpected tailscale argv") - } - switch args[1] { - case "status": - f.statusCalls++ - st := f.statuses[0] - if len(f.statuses) > 1 { - f.statuses = f.statuses[1:] - } - return []byte(st), nil - case "up": - f.upCalls = append(f.upCalls, args) - if len(f.upErrs) == 0 { - return nil, nil - } - err := f.upErrs[0] - f.upErrs = f.upErrs[1:] - return nil, err - } - return nil, errors.New("unexpected tailscale subcommand") -} - -func captureLog(t *testing.T) *bytes.Buffer { - t.Helper() - var buf bytes.Buffer - orig := log.Writer() - log.SetOutput(&buf) - t.Cleanup(func() { log.SetOutput(orig) }) - return &buf -} - -// TestRegisterTailscaleOnceRequestsTags asserts that when tailscaled is -// waiting in NeedsLogin, registerTailscaleOnce runs exactly one -// `tailscale up`, with exactly tailscaleUpArgs' tagged argv. -func TestRegisterTailscaleOnceRequestsTags(t *testing.T) { - fake := &fakeTailscale{statuses: []string{`{"BackendState":"NeedsLogin"}`}} - stubTailscaleCLI(t, fake.run) - - registerTailscaleOnce(context.Background(), "my-game", []string{"tag:gameplane", "tag:game"}) - - want := [][]string{{ - "--socket=/tmp/gameplane-tunnel-tailscaled.sock", - "up", - "--reset", - "--auth-key=file:/tmp/gameplane-tunnel-tailscale-authkey", - "--hostname=my-game", - "--advertise-tags=tag:gameplane,tag:game", - }} - if len(fake.upCalls) != len(want) || !equalArgs(fake.upCalls[0], want[0]) { - t.Errorf("tailscale up calls = %v, want %v", fake.upCalls, want) - } -} - -// TestRegisterTailscaleOnceWaitsThroughStarting asserts that a transient -// BackendState ("Starting") is waited out rather than acted on, and that a -// node that then reports Running with exactly the requested tags (in any -// order), as with a reused state file, gets no `tailscale up` at all. -func TestRegisterTailscaleOnceWaitsThroughStarting(t *testing.T) { - fake := &fakeTailscale{statuses: []string{ - `{"BackendState":"Starting"}`, - `{"BackendState":"Running","Self":{"Tags":["tag:game","tag:gameplane"]}}`, - }} - stubTailscaleCLI(t, fake.run) - - registerTailscaleOnce(context.Background(), "my-game", []string{"tag:gameplane", "tag:game"}) - - if len(fake.upCalls) != 0 { - t.Errorf("tailscale up calls = %v, want none for a node already running with the requested tags", fake.upCalls) - } - if fake.statusCalls != 2 { - t.Errorf("status calls = %d, want 2 (Starting, then Running)", fake.statusCalls) - } -} - -// TestRegisterTailscaleOnceRunningWithOtherTags asserts that a node -// running with different tags (or none) is re-registered with the -// requested ones. -func TestRegisterTailscaleOnceRunningWithOtherTags(t *testing.T) { - fake := &fakeTailscale{statuses: []string{`{"BackendState":"Running","Self":{"Tags":["tag:old"]}}`}} - stubTailscaleCLI(t, fake.run) - - registerTailscaleOnce(context.Background(), "my-game", []string{"tag:new"}) - - if len(fake.upCalls) != 1 || !containsArg(fake.upCalls[0], "--advertise-tags=tag:new") { - t.Errorf("tailscale up calls = %v, want one requesting tag:new", fake.upCalls) - } -} - -// TestRegisterTailscaleOnceFallsBackUntagged asserts that when the tagged -// `tailscale up` fails (e.g. no tagOwners grant) and the node is not -// Running afterwards, the failure is logged and one untagged -// `tailscale up` follows, so the tunnel still comes up. -func TestRegisterTailscaleOnceFallsBackUntagged(t *testing.T) { - fake := &fakeTailscale{ - statuses: []string{`{"BackendState":"NeedsLogin"}`}, - upErrs: []error{errors.New(`requested tags [tag:gameplane] are invalid or not permitted`)}, - } - stubTailscaleCLI(t, fake.run) - buf := captureLog(t) - - registerTailscaleOnce(context.Background(), "my-game", []string{"tag:gameplane"}) - - if len(fake.upCalls) != 2 { - t.Fatalf("tailscale up calls = %v, want 2 (tagged, then untagged)", fake.upCalls) - } - if !containsArg(fake.upCalls[0], "--advertise-tags=tag:gameplane") { - t.Errorf("first tailscale up = %v, want it to request tag:gameplane", fake.upCalls[0]) - } - wantFallback := []string{ - "--socket=/tmp/gameplane-tunnel-tailscaled.sock", - "up", - "--reset", - "--auth-key=file:/tmp/gameplane-tunnel-tailscale-authkey", - "--hostname=my-game", - } - if !equalArgs(fake.upCalls[1], wantFallback) { - t.Errorf("fallback tailscale up = %v, want %v", fake.upCalls[1], wantFallback) - } - got := buf.String() - if !strings.Contains(got, "not permitted") || !strings.Contains(got, "tagOwners") { - t.Errorf("log output = %q, want the tailscale error and the tagOwners hint", got) - } - if !strings.Contains(got, "registered the device without tags") { - t.Errorf("log output = %q, want it to report the untagged registration", got) - } -} - -// TestRegisterTailscaleOnceNoFallbackWhenStillRunning asserts that no -// untagged `tailscale up` runs when the node is still Running after the -// tagged one failed: it is already up, so a second login would be pointless. -func TestRegisterTailscaleOnceNoFallbackWhenStillRunning(t *testing.T) { - fake := &fakeTailscale{ - statuses: []string{`{"BackendState":"Running","Self":{}}`}, - upErrs: []error{errors.New("not permitted")}, - } - stubTailscaleCLI(t, fake.run) - captureLog(t) - - registerTailscaleOnce(context.Background(), "my-game", []string{"tag:gameplane"}) - - if len(fake.upCalls) != 1 { - t.Errorf("tailscale up calls = %v, want only the tagged attempt", fake.upCalls) - } -} - -// TestRegisterTailscaleOnceRunningWithOldTagsNoFallback asserts that when a -// running node retains old tags after a failed tagged attempt, no untagged -// up runs and the log explicitly surfaces the retained tags. -func TestRegisterTailscaleOnceRunningWithOldTagsNoFallback(t *testing.T) { - fake := &fakeTailscale{ - statuses: []string{`{"BackendState":"Running","Self":{"Tags":["tag:old"]}}`, `{"BackendState":"Running","Self":{"Tags":["tag:old"]}}`}, - upErrs: []error{errors.New("not permitted")}, - } - stubTailscaleCLI(t, fake.run) - buf := captureLog(t) - - registerTailscaleOnce(context.Background(), "my-game", []string{"tag:new"}) - - if len(fake.upCalls) != 1 { - t.Errorf("tailscale up calls = %v, want only the tagged attempt", fake.upCalls) - } - if !strings.Contains(buf.String(), "still running with previous tags") { - t.Errorf("log does not mention retained tags; output:\n%s", buf.String()) - } -} - -// TestStartTailscaleRegistrarReturnsAfterFailure asserts that when every -// `tailscale up` fails, registerTailscaleOnce logs and returns normally, so -// startTailscaleRegistrar's wait() (which run()'s shutdown defers on) -// completes and the relay supervision loop in run() is unaffected. -func TestStartTailscaleRegistrarReturnsAfterFailure(t *testing.T) { - fake := &fakeTailscale{ - statuses: []string{`{"BackendState":"NeedsLogin"}`}, - upErrs: []error{errors.New("not permitted"), errors.New("still failing")}, - } - stubTailscaleCLI(t, fake.run) - buf := captureLog(t) - - done := make(chan struct{}) - wait := startTailscaleRegistrar(context.Background(), "my-game", []string{"tag:gameplane"}) - go func() { - wait() - close(done) - }() - - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("startTailscaleRegistrar's wait() did not return after `tailscale up` failures; it would block run()'s shutdown") - } - - if got := buf.String(); !strings.Contains(got, "untagged `tailscale up` also failed: still failing") { - t.Errorf("log output = %q, want it to report the failed fallback", got) - } -} - -// TestRegisterTailscaleOnceStopsOnCancel asserts that a cancelled context -// ends the wait for tailscaled's socket promptly, without running -// `tailscale up`. -func TestRegisterTailscaleOnceStopsOnCancel(t *testing.T) { - var upCalled atomic.Bool - stubTailscaleCLI(t, func(_ context.Context, args ...string) ([]byte, error) { - if len(args) > 1 && args[1] == "up" { - upCalled.Store(true) - } - return nil, errors.New("dial unix /tmp/gameplane-tunnel-tailscaled.sock: connect: no such file or directory") - }) - - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan struct{}) - go func() { - registerTailscaleOnce(ctx, "my-game", []string{"tag:gameplane"}) - close(done) - }() - cancel() - - select { - case <-done: - case <-time.After(5 * time.Second): - t.Fatal("registerTailscaleOnce did not return after ctx was cancelled") - } - if upCalled.Load() { - t.Error("tailscale up ran after ctx was cancelled") - } -} - func TestRenderFrpConfigServerAddrEscaped(t *testing.T) { cfg := Config{ FrpServerAddr: `server.example.com"` + "\n" + `extra = "value`, @@ -1142,14 +572,6 @@ func TestRenderConfigDispatchesByType(t *testing.T) { BackingServicePort: "game:25565:30000:tcp", }, }, - { - "tailscale", - Config{ - TunnelType: "tailscale", - TailscaleHostname: "my-game", - BackingServicePorts: "game:25565", - }, - }, { "playit", Config{ @@ -1339,38 +761,6 @@ func TestBuildCommandPlayit(t *testing.T) { } } -func TestBuildCommandTailscale(t *testing.T) { - cfg := Config{TunnelType: "tailscale", TailscaleHostname: "my-game"} - cmd := buildCommand(context.Background(), cfg) - if cmd == nil { - t.Fatal("buildCommand() returned nil") - } - - // The pod has no NET_ADMIN and no /dev/net/tun, so tailscaled must run in - // userspace-networking mode -- this is load-bearing, not a style choice. - // Confirmed against tailscale/tailscale's cmd/tailscaled/tailscaled.go flags. - if !containsArg(cmd.Args, "--tun=userspace-networking") { - t.Errorf("Args = %v, missing --tun=userspace-networking", cmd.Args) - } - // Hostname and auth key travel via the declarative --config file (see - // tailscaledConfig / renderTailscaleConfig), not a flag or env var: - // tailscaled has no --hostname flag and does not read TS_AUTHKEY itself. - if !containsArg(cmd.Args, "--config="+tailscaleConfigPath) { - t.Errorf("Args = %v, missing --config=%s", cmd.Args, tailscaleConfigPath) - } - // registerTailscaleOnce reaches tailscaled over this socket; the non-root - // image can't create tailscaled's default /run/tailscale path. - if !containsArg(cmd.Args, "--socket="+tailscaleSocketPath) { - t.Errorf("Args = %v, missing --socket=%s", cmd.Args, tailscaleSocketPath) - } - if containsArg(cmd.Args, "--hostname=my-game") { - t.Errorf("Args = %v, should not pass --hostname (not a real tailscaled flag)", cmd.Args) - } - if containsEnv(cmd.Env, "TS_AUTHKEY=test-auth-key") { - t.Errorf("Env = %v, should not set TS_AUTHKEY (bare tailscaled does not read it)", cmd.Env) - } -} - func TestBuildCommandUnknownType(t *testing.T) { cfg := Config{TunnelType: "bogus"} cmd := buildCommand(context.Background(), cfg) @@ -1391,24 +781,6 @@ func equalArgs(got, want []string) bool { return true } -func containsArg(args []string, want string) bool { - for _, a := range args { - if a == want { - return true - } - } - return false -} - -func containsEnv(env []string, want string) bool { - for _, e := range env { - if e == want { - return true - } - } - return false -} - // ----------------------------------------------------------------------- // exponentialBackoff tests // ----------------------------------------------------------------------- diff --git a/tunnel/specs.md b/tunnel/specs.md index 810da868a..26b70f0d9 100644 --- a/tunnel/specs.md +++ b/tunnel/specs.md @@ -2,48 +2,52 @@ **Status:** pre-v1 (v0.3.0) **Module / command:** `github.com/GameplanePanel/gameplane/tunnel` -**Dependencies:** stdlib only +**Dependencies:** Go stdlib and `tailscale.com/tsnet` (F-173, #722) ## Purpose -Relay client supervisor that configures and supervises a third-party tunnel process (frp, Tailscale, or playit). The supervisor runs as a single-replica Deployment per GameServer, reads provider-specific configuration and credentials via environment variables and a mounted Secret, renders provider-specific config files, spawns the relay binary, and supervises it with exponential backoff until context cancellation or an unrecoverable error. The tunnel pod never sleeps — it holds the public relay address across the full GameServer lifecycle so connection attempts can trigger wake-on-connect. +Relay client supervisor that configures and supervises frp/playit child processes or an embedded Tailscale node. The supervisor runs as a single-replica Deployment per GameServer, reads provider-specific configuration and credentials via environment variables and a mounted Secret, renders provider-specific config files, spawns the relay binary, and supervises it with exponential backoff until context cancellation or an unrecoverable error. The tunnel pod never sleeps — it holds the public relay address across the full GameServer lifecycle so connection attempts can trigger wake-on-connect. ## Responsibilities 1. Parse environment variables to determine the tunnel provider (frp, tailscale, or playit), backing service address, and provider-specific config. 2. Read provider-specific credentials from the mounted Secret at `/etc/gameplane/tunnel-auth` (read-only). -3. Render provider-specific config files at fixed paths (`/tmp/gameplane-tunnel-frpc.toml` for frp, `/tmp/gameplane-tunnel-tailscaled.json` for tailscale, `/tmp/gameplane-tunnel-playit-auth` for playit). -4. Build provider-specific relay command-line arguments, referencing only fixed config file paths (never passing secrets via argv). -5. Spawn the relay binary (frpc, tailscaled, or playitd) and supervise it until context cancellation or unrecoverable failure. +3. Render frp TOML or playit credentials at fixed paths, with secrets excluded from argv. +4. Start and supervise frpc or playitd as child processes. +5. For Tailscale, authenticate an embedded tsnet node, open advertised TCP/UDP listeners, and forward only to the GameServer's backing Service. 6. On transient failure (exit code other than 126/127, not a permission error), apply exponential backoff (2^n seconds, capped at 300 seconds) and restart. 7. On unrecoverable failure (exit code 126/127 or "permission denied" error), exit immediately. 8. Forward SIGTERM for graceful shutdown, waiting up to 10 seconds before SIGKILL. 9. (Playit only) Poll playitd's IPC control socket for the assigned relay addresses and patch them into the GameServer's `status.tunnelEndpoints`, which the operator validates and merges into `status.endpoints` (see "Playit Address Reporting"). -10. (Tailscale only, when `TAILSCALE_TAGS` holds valid tags) Register the device once with `tailscale up --advertise-tags` over tailscaled's socket, requesting the tags at registration. If the tailnet refuses them, log the error; only if the backend is not `Running` afterwards, attempt one untagged registration, which can also fail and is then logged (see "Tailscale Tag Application"). +10. Tailscale readiness requires an online tailnet identity with the requested hostname/tags and all forwarding listeners active. Invalid or denied tags remain not ready; there is no untagged fallback. ## Non-goals / boundaries - Does **not** run the game server itself — that is the operator and game container's job. - Does **not** create or manage the relay infrastructure — frp/Tailscale/playit accounts and credentials are pre-configured. - Does **not** authenticate or authorize clients connecting to the relay — relay access control is the provider's responsibility. -- Does **not** validate or filter traffic — the relay process handles all inbound connections and port forwarding. +- Does **not** validate or filter traffic — the provider handles access control; the Tailscale bridge copies bytes/datagrams to fixed destinations. - Does **not** modify the game pod or game container — the tunnel runs as a separate Deployment. - Does **not** report metrics or logs to external systems — logs flow to the pod's stdout/stderr for the cluster to capture. - Does **not** support runtime configuration changes — config is read once at startup and cannot be updated without redeploying. -- Does **not** implement Tailscale, frp, or playit protocols — delegates to their official binaries. +- Does **not** implement Tailscale, frp, or playit protocols — uses official frp/playit binaries and the official Tailscale Go library. ## Directory & package layout ``` tunnel/ -├── main.go # Entry point; config loading; credential reading; config rendering; relay spawning and supervision; tailscale tagged registration +├── main.go # Entry point; config loading; credential reading; config rendering; relay spawning and supervision +├── tailscale.go # Embedded node lifecycle, health and identity checks +├── tailscale_forward.go # Bounded TCP/UDP bridge to the fixed Service +├── tailscale_test.go # Registration configuration and readiness lifecycle +├── tailscale_forward_test.go # Real TCP/UDP sockets, isolation, limits and shutdown ├── playit_reporter.go # playit only: polls playitd's IPC socket for assigned addresses; patches status.tunnelEndpoints ├── playit_reporter_test.go # Fake playitd IPC server + httptest API server tests for the reporter ├── main_test.go # Config parsing; credential reading; config rendering per provider; command building; backoff; error classification; supervision lifecycle ├── Dockerfile.frp # Image build for frp provider (sets Version via -ldflags) ├── Dockerfile.playit # Image build for playit provider (one image per provider) ├── Dockerfile.tailscale # Image build for tailscale provider -├── go.mod # Dependencies: stdlib only (no external deps, so no go.sum) +├── go.mod # Pinned tsnet dependency; go.sum records the transitive graph └── .testcoverage.yml # 70% coverage gate ``` @@ -63,8 +67,8 @@ Single executable module; no subdirectories or packages. | `FRP_SERVER_PORT` | int | no (default 7000) | frp server port; validated 1–65535 | | `BACKING_SERVICE_PORT` | string | if frp | Port mappings for frp, format `name:localPort:remotePort:protocol,...` (e.g., `game:34197:30000:udp`); localPort and protocol come from the GameTemplate port, remotePort from `spec.networking.tunnel.frp.remotePorts` (F-052) | | `TAILSCALE_HOSTNAME` | string | if tailscale | MagicDNS hostname in the tailnet; fatal if unset for tailscale provider | -| `TAILSCALE_TAGS` | string | no | Comma-separated ACL tags requested for the Tailscale device when it registers. Each entry is trimmed, empty entries are dropped, and a bare name gets the `tag:` prefix. Each tag must then be `tag:` followed by a letter and only letters, digits or `-` (tailscale's `tailcfg.CheckTag` rule); otherwise the whole value is logged and ignored, and the device registers untagged. Never written into tailscaled's declarative config: its alpha0 config has no field for ACL tags (`ipn.ConfigVAlpha`), and its loader rejects unknown JSON fields, so a `tags` key would make tailscaled refuse to start. Tags are requested through a one-off `tailscale up --advertise-tags` instead (see "Tailscale Tag Application"). The tailnet's ACL must grant `tagOwners` for the tags to the auth key's owner. If it doesn't, the error is logged and the tunnel runs untagged rather than crash-looping. | -| `BACKING_SERVICE_PORTS` | string | if tailscale or playit | Container ports for non-frp providers, format `name:port,name:port` | +| `TAILSCALE_TAGS` | string | no | Comma-separated requested tags. Whitespace/empty entries are removed, bare names receive `tag:`, duplicates are removed. Invalid tags fail startup. Requested tags must be granted by tailnet `tagOwners`; refusal does not fall back to an untagged identity. | +| `BACKING_SERVICE_PORTS` | string | if tailscale or playit | Tailscale: `name:port:protocol,...`, only `tcp` and `udp`, at most 32 unique protocol/port pairs. Playit retains `name:port,...`. Upgrade operator and tunnel images together; ambiguous legacy Tailscale mappings fail closed. | | `PLAYIT_TUNNEL_NAME` | string | if playit | Label/name for the playit tunnel; fatal if unset for playit provider | **Validation**: At startup, `loadConfig` enforces mutual requirement rules: frp mandates `FRP_SERVER_ADDR` and `BACKING_SERVICE_PORT`; tailscale mandates `TAILSCALE_HOSTNAME` and `BACKING_SERVICE_PORTS`; playit mandates `PLAYIT_TUNNEL_NAME` and `BACKING_SERVICE_PORTS`. Invalid `FRP_SERVER_PORT` values (non-numeric, out of range 1–65535) are fatal. Unknown `TUNNEL_TYPE` values are fatal. @@ -76,7 +80,7 @@ The operator mounts a read-only Secret volume at `/etc/gameplane/tunnel-auth` co | Provider | Key Name | Example Usage | Notes | |---|---|---|---| | frp | `token` | Auth token for the frpc client | Read by `readCredentials` and embedded in rendered `/tmp/gameplane-tunnel-frpc.toml` | -| tailscale | `authKey` | Auth key (one-time or reusable) for device registration | Read and embedded in rendered `/tmp/gameplane-tunnel-tailscaled.json` | +| tailscale | `authKey` | Reusable auth key for registration | Read into tsnet.Server.AuthKey in memory; identity state is stored in the Pod volume | | playit | `secretKey` | API secret key for playitd authentication | Read and written to `/tmp/gameplane-tunnel-playit-auth` for playitd's `--secret-path` flag | Whitespace (leading/trailing newlines, spaces) is trimmed from credential values before use. Missing credentials are fatal errors and prevent relay startup. @@ -86,18 +90,15 @@ Whitespace (leading/trailing newlines, spaces) is trimmed from credential values | Provider | Path | Format | Ownership | |---|---|---|---| | frp | `/tmp/gameplane-tunnel-frpc.toml` | TOML | Rendered by `renderFrpConfig`; read by frpc process | -| tailscale | `/tmp/gameplane-tunnel-tailscaled.json` | JSON | Rendered by `renderTailscaleConfig`; read by tailscaled via `--config` flag. Holds `version` and `hostname`, plus `authKey` without tags, or `"locked": false` with tags | -| tailscale (tags only) | `/tmp/gameplane-tunnel-tailscale-authkey` | Raw text (auth key) | Written by `renderTailscaleConfig` when tags are requested; read by `tailscale up --auth-key=file:` | | playit | `/tmp/gameplane-tunnel-playit-auth` | Raw text (secret key) | Rendered by `renderPlayitConfig`; read by playitd via `--secret-path` flag | -All files are written with mode `0o600` (read/write by owner only) via `os.WriteFile` — see `renderFrpConfig`, `renderTailscaleConfig`, `renderPlayitConfig` in `main.go`. The rendered config file (and, for tailscale, the auth key file) is removed by a deferred `os.Remove` when the supervisor's `run` returns (context cancellation or an unrecoverable error); it is not removed on SIGKILL or on each relay restart inside the backoff loop. tailscaled's `--state` file (`/tmp/tailscale.state`) is never removed by the process. All of these files live under `/tmp` in the pod's ephemeral, per-pod filesystem, so any that remain are discarded when the pod terminates and its filesystem is reclaimed. +frp/playit rendered files use mode 0600 and are removed when `run` returns. Tailscale uses the state volume described below; no daemon config or CLI auth file is produced. ### Relay Binaries and Command-Line Arguments | Provider | Binary | Arguments | |---|---|---| | frp | `/usr/local/bin/frpc` | `-c /tmp/gameplane-tunnel-frpc.toml` (config file flag) | -| tailscale | `/usr/local/bin/tailscaled` | `--tun=userspace-networking` (hardened pod context), `--state=/tmp/tailscale.state`, `--config=/tmp/gameplane-tunnel-tailscaled.json`, `--socket=/tmp/gameplane-tunnel-tailscaled.sock` | | playit | `/usr/local/bin/playitd` | `--secret-path /tmp/gameplane-tunnel-playit-auth`, `--socket-path /tmp/gameplane-tunnel-playitd.sock`, `--platform-docker` | All binary paths are fixed constants, not configurable. Secrets are never passed via command-line arguments (gosec G204 compliance). @@ -112,27 +113,26 @@ playit.gg assigns a tunnel's public address server-side, so the supervisor learn 4. When the mapped set differs from the last one reported, it's merge-patched into `status.tunnelEndpoints` on `PATCH https://kubernetes.default.svc/apis/gameplane.local/v1alpha1/namespaces//gameservers//status`. The request uses the pod's projected ServiceAccount token and CA from `/var/run/secrets/kubernetes.io/serviceaccount`. The first report after startup is always sent, and an empty set is sent as `null` so stale addresses from a previous pod are cleared. The operator validates those entries (`validatePlayitEndpoints`) and merges them into `status.endpoints`, the same place frp/tailscale endpoints land. 5. Timing: while the socket is down, playitd isn't `running` yet, or a patch fails, the reporter backs off from 1s, doubling up to 30s. Once a set has been reported, it re-polls every 30s so address changes are picked up. It stops when `run`'s context ends. If the ServiceAccount CA can't be read (not in a cluster) or `BACKING_SERVICE_PORTS` doesn't parse, the reporter logs and is skipped. The relay still runs. -### Tailscale Tag Application +### Embedded Tailscale forwarding (F-173 / #722) -Tailscale ACL tags can only be requested when a device registers. tailscaled's declarative `--config` file has no field for them (see the `TAILSCALE_TAGS` row above). `tailscale set` has no `--advertise-tags` flag, and changing the tags of a registered node needs a fresh login. So when `TAILSCALE_TAGS` holds valid tags, the supervisor lets tailscaled start unauthenticated and registers it itself. Flags below are confirmed against tailscale v1.102.4, the version `Dockerfile.tailscale` ships. +The maintainer chose embedded `tailscale.com/tsnet` for TCP and UDP; the decision is recorded in [OPEN-DECISIONS.md](../specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md#od-031-tailscale-game-forwarding-f-173--resolved-2026-10-21). No tailscaled executable, CLI, subnet router, exit node, Serve or Funnel configuration is used. -1. `renderTailscaleConfig` writes the config with `version`, `hostname` and `"locked": false`, and no `authKey`. The auth key goes to `/tmp/gameplane-tunnel-tailscale-authkey` (mode 0600). `locked` must be false: `ipn.ConfigVAlpha.Locked` defaults to true, and a locked config makes tailscaled reject every CLI prefs change. -2. `tailscaled` is started as usual, with `--socket=/tmp/gameplane-tunnel-tailscaled.sock`. Its Linux default, `/run/tailscale/tailscaled.sock`, can't be created by the non-root distroless image; this is the same reasoning as `playitSocketPath`. The `tailscale` CLI, bundled in `Dockerfile.tailscale` alongside `tailscaled`, reaches it over the same socket via `--socket=`. With no auth key, tailscaled waits in `NeedsLogin`. -3. `run` starts one goroutine, `startTailscaleRegistrar`, which calls `registerTailscaleOnce` once per supervisor lifetime, in parallel with the relay supervision loop. It is not rerun on a relay restart: the login and granted tags persist in tailscaled's state file (`/tmp/tailscale.state`, `--state` in `buildCommand`). -4. `waitForTailscaled` polls `tailscale --socket= status --json --peers=false` every `tailscaleReadyPollInterval` (2 seconds), bounded by `tailscaleReadyTimeout` (2 minutes). It waits until the socket answers with a settled `BackendState`: `NeedsLogin`, `NeedsMachineAuth`, `Stopped` or `Running`. Transient states (`NoState`, `Starting`) keep it polling. -5. If the node is already `Running` and `Self.Tags` is exactly the requested set, as with a reused state file, nothing more happens. -6. Otherwise it runs `tailscale --socket= up --reset --auth-key=file:/tmp/gameplane-tunnel-tailscale-authkey --hostname= --advertise-tags=` once, bounded by `tailscaleUpTimeout` (2 minutes). The pure `tailscaleUpArgs` builds this argv. `--auth-key=file:` makes the CLI read the key from the file, so the key never appears in argv. `--reset` resets every setting not named on the command line to its default. Without it, `tailscale up` on a node with saved prefs refuses to run unless every non-default setting is repeated. -7. If that fails, most commonly because the tailnet's ACL doesn't grant `tagOwners` for the tags to the auth key's owner, the error is logged. If the node is not `Running` afterwards, `tailscale up` runs once more without `--advertise-tags`, so the tunnel registers untagged. Every failure, including the readiness timeout, is logged and none stops the supervisor or the relay process. An ACL problem only the tailnet admin can fix never becomes a crash loop. +- `tailscaleForwardPlan` accepts only the exact `..svc` host and explicitly advertised protocol/port pairs. Inbound bytes never select a backend address. TCP listens on both tailnet families; UDP binds each assigned tailnet IP because tsnet requires an explicit address. +- The ordinary Go network dialer resolves the Service for each new TCP connection or UDP session. Existing streams/sessions can fail during Service/backend replacement; reconnecting clients resolve the current Service. UDP sessions expire after 30 seconds without client packets. +- A shared 128-session budget covers all ports/protocols/families. Each UDP session queues at most four datagrams (65,507 bytes each) and uses one response buffer. Full budgets or queues drop incoming UDP; excess TCP connections close. Dials and UDP writes time out after five seconds. TCP sockets have five-minute activity timeouts. Shutdown closes listeners and sockets, cancels pending dials, then joins workers. +- `tsnet.Server` receives the existing hostname, normalized requested tags and mounted auth key. There is no untagged fallback. The fixed state directory `/var/lib/gameplane-tailscale` uses a per-Pod EmptyDir owned through fsGroup 65532. Container restarts reuse identity. Pod replacement loses state and needs a reusable valid auth key; the Deployment uses Recreate. This is not durable identity across Pod replacement. Remove stale devices in the tailnet admin console to free the hostname. A suffixed/conflicting MagicDNS name cannot report the originally advertised name as ready. +- `Up` has a two-minute attempt deadline; failed attempts use the existing capped retry delay. After binding every listener, the binary polls local tailnet status every two seconds with a two-second deadline. Loss of online status, tags or hostname clears readiness. Address changes recreate listeners. Failed listeners close all forwarding sockets and retry. Credentials and interactive login URLs are excluded from logs. +- HTTP `:8091/readyz` returns 200 only while that state is ready, otherwise 503. `/healthz` remains independent of tailnet availability. The operator probes every five seconds and requires a current observed Deployment generation with an updated ready replica. Legacy deployments without the forwarding probe cannot report `TunnelReady=True`. Tailscale endpoints are published only while ready. +- Readiness establishes local forwarding capacity and tailnet authentication. It does not prove a particular player's grant or game application health, and it deliberately remains ready while the game sleeps so traffic can reach the sentinel. Tailnet grants/ACLs still determine which peers may connect. Missing tags, denied registration, device approval and hostname conflicts require administrator repair. +- Upgrading from the former daemon implementation re-registers the Pod. Replace the auth key first if it was single-use. Coordinate operator and tunnel image upgrades; the new protocol-bearing port contract rejects old ambiguous entries. -The CLI is run by `execTailscaleCLI` with `exec.CommandContext` tied to `run`'s context. The binary path is the literal `/usr/local/bin/tailscale`, and nothing goes through a shell. The argv is appended to `cmd.Args`, so gosec's G204 check sees only literal arguments. The actual guarantee is that argv is built only by `tailscaleStatusArgs` and `tailscaleUpArgs`: every value is a single `--flag=value` element, the tags are validated by `parseTailscaleTags`, and the auth key is never in argv. - -Without tags, none of this runs. The auth key goes into the config and tailscaled logs in by itself, as before. +The disposable integration procedure in [tailscale-forwarding-integration.md](../docs/tailscale-forwarding-integration.md) covers actual tailnet TCP/UDP, denied grants, backend replacement and registration/restart outcomes. Unit transports use real local sockets; they do not establish live tailnet coverage. ## Key Invariants 1. **Tunnel pod never sleeps.** The tunnel Deployment persists for the full GameServer lifecycle, even when the server idles or suspends. The pod must remain running to hold the relay address so a connection attempt can trigger wake-on-connect. -2. **Credentials are never passed via command-line.** All three providers use file-based credential delivery: frp and tailscale embed credentials in rendered config files, playitd reads a secret file via `--secret-path`. This guards against argv scraping and satisfies gosec's G204 (subprocess argument constant verification). +2. **Credentials are never passed via command-line.** frp embeds credentials in a private config file, playitd reads a secret file via `--secret-path`, and tsnet receives its key in memory. This guards against argv scraping and satisfies gosec's G204 (subprocess argument constant verification). 3. **Exponential backoff is capped and unshed.** On transient failure, retry delays follow 2^n seconds (1s, 2s, 4s, ..., 256s) capped at 300 seconds (5 minutes). No jitter is added because each pod runs a single replica against its own relay connection; jitter spreads out a thundering herd, not a singleton. @@ -144,10 +144,9 @@ Without tags, none of this runs. The auth key goes into the config and tailscale 7. **Operator image selection per provider.** The operator injects provider-specific tunnel images via flags (`--tunnel-frp-image`, `--tunnel-tailscale-image`, `--tunnel-playit-image`), each with a `dev` default. Each pod runs exactly one provider's binary, selected by `TUNNEL_TYPE`. -## Known Gaps +## Known Limits -- **Tailscale doesn't forward tailnet traffic to the game (F-173):** `BACKING_SERVICE_DNS` and `BACKING_SERVICE_PORTS` are read and validated for presence when `TUNNEL_TYPE=tailscale`, but neither reaches `renderTailscaleConfig` or the `tailscaled` process. The rendered config carries only `version`, `hostname`, and `authKey` or `locked`. ACL tags are requested at registration through `tailscale up`; see "Tailscale Tag Application". The tunnel pod registers as a tailnet device, but nothing bridges an inbound tailnet connection to the backing Service — tailscaled's declarative `--config` file has no serve/forward field, and `tailscale serve` (the real mechanism) only proxies TCP/HTTP(S) to a local `127.0.0.1` target, never a remote host, and has no UDP support. Fixing this is an architecture decision (a local proxy + headless `tailscale serve`, TCP-only; a subnet route via `advertiseRoutes`, which needs tailnet ACL auto-approval and widens reachability into cluster-internal networking; or replacing subprocess `tailscaled` with an embedded `tsnet` listener, which would end this module's stdlib-only dependency policy), tracked pending a decision in `specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md`. -- **Single-use Tailscale auth keys and tag retries:** when `tags` is set, the supervisor logs in with `tailscale up`. If that tagged login is refused (missing `tagOwners`), or a restarted pod reuses a state file with different tags, the untagged fallback `up` reuses the same auth key; a single-use (ephemeral, non-reusable) key has already been consumed, so the fallback fails too. The failure is logged and the supervisor does not crash-loop, but the device stays logged out until a fresh key is stored. Use a reusable auth key when setting `tags`. +Tailscale identity survives container restarts but not Pod replacement; a reusable auth key and free MagicDNS hostname are required for re-registration. Live tailnet integration needs an explicitly provisioned disposable tailnet and is separate from ordinary unit CI. The prior registration-only F-173 gap is replaced by the embedded TCP/UDP bridge above. ## CRD Integration @@ -227,16 +226,16 @@ When the pod receives SIGTERM (e.g., during cluster shutdown or pod deletion): ## Dependencies **Internal:** None -**External:** Go stdlib only (bufio, bytes, context, crypto/tls, crypto/x509, encoding/json, errors, fmt, io, log, net, net/http, net/url, os, os/exec, os/signal, path/filepath, reflect, sort, strconv, strings, sync, syscall, time). The playit status patch uses plain `net/http` against the API server rather than client-go, keeping the module free of third-party dependencies. -**Go version:** 1.26+ +**External:** `tailscale.com/tsnet` v1.102.5 and Go stdlib. The playit status patch uses plain `net/http` against the API server without adding client-go. +**Go version:** 1.26.10+ (or 1.27.3+ on the 1.27 line; both include the October 2026 crypto/x509 security fix) -No third-party dependencies. The operator provides provider-specific binaries (frpc, tailscaled, playitd) in each container image. +The maintainer explicitly approved ending the stdlib-only policy for tsnet in #722. Version 1.102.5 supports Go 1.26 and the required TCP/UDP APIs. CI resolves the latest Go 1.26 patch; tunnel image builders use Go 1.27.3. Track future tsnet/security updates through go.mod/go.sum. frpc and playitd remain external binaries. ## Security Considerations 1. **Secret mounting (read-only):** Credentials are mounted from Kubernetes Secrets into the pod at `/etc/gameplane/tunnel-auth` as a read-only VolumeMount. The credential files are accessed only after a path-containment check via `filepath.Rel` to ensure the resolved path does not escape the mount directory, providing defense-in-depth against directory traversal. -2. **Credentials never on argv:** All three providers accept secrets via file-based config (embedded in TOML/JSON for frp/tailscale, separate `--secret-path` file for playitd). This ensures secrets don't appear in `/proc//cmdline`. +2. **Credentials never on argv:** frp/playit accept secrets via private files; Tailscale receives the mounted key in memory. This ensures secrets don't appear in `/proc//cmdline`. 3. **Hardened pod security context:** UID 65532 (nonroot), runAsNonRoot=true, allowPrivilegeEscalation=false, all capabilities dropped. No privileged escalation or host access. @@ -244,7 +243,7 @@ No third-party dependencies. The operator provides provider-specific binaries (f 5. **Network egress policy:** The tunnel pod's outbound traffic is controlled by a per-server NetworkPolicy. Without it, the games namespace's default-deny-egress would block relay connections. -6. **Image provider isolation:** Each provider's tunnel binary (frpc, tailscaled, playitd) runs in its own container image. A vulnerability in one provider's binary doesn't affect others; operators can patch images independently. +6. **Image provider isolation:** Provider images remain separate. The shared Go executable now links tsnet; all three tunnel images must be rebuilt for a dependency security update. 7. **No credential validation at tunnel pod level:** The tunnel supervisor does not validate frp tokens, Tailscale auth keys, or playit secret keys. Validation happens at the provider's server. Invalid credentials manifest as authentication failures on the provider side. @@ -255,9 +254,9 @@ No third-party dependencies. The operator provides provider-specific binaries (f - **Config loading:** `TestLoadConfigFrp`, `TestLoadConfigTailscale`, `TestLoadConfigPlayit` verify environment variable parsing and provider-specific validation (required fields per provider, default values). - **Config validation errors:** `TestLoadConfigMissingRequired`, `TestLoadConfigFrpMissingAddress`, `TestLoadConfigInvalidFrpPort`, `TestLoadConfigTailscaleMissingHostname`, `TestLoadConfigPlayitMissingTunnelName` verify that missing or invalid required fields are fatal with clear error messages. - **Credential reading:** `TestReadCredentialsSuccess`, `TestReadCredentialsMissingFile`, `TestReadCredentialsKeyNames`, `TestReadCredentialsUnknownTunnelType` verify credential path isolation (defense-in-depth `filepath.Rel` check) and per-provider key names. -- **Config rendering:** `TestRenderFrpConfig`, `TestRenderFrpConfigInvalidPortMapping`, `TestRenderFrpConfigMultiplePorts`, `TestRenderTailscaleConfig`, `TestRenderTailscaleConfigNoHostname`, `TestRenderTailscaleConfigWithTagsOmitsTags`, `TestRenderTailscaleConfigNoTags`, `TestRenderTailscaleConfigInvalidTagsRegistersUntagged`, `TestRenderTailscaleConfigLogTagsNotesRegistration`, `TestRenderTailscaleConfigLogTagsSilentWhenUnset` and `TestRenderPlayitConfig` verify the TOML/JSON/text output for each provider. The tailscale tests also check that no `tags` key is ever written into tailscaled's config. With valid tags, `authKey` is left out, `"locked": false` is written, and the key goes to a 0600 file. Without tags or with invalid tags, the config keeps its original shape and no key file is written. The log line names `tailscale up --advertise-tags` and the `tagOwners` requirement. -- **Command building:** `TestBuildCommandFrp`, `TestBuildCommandPlayit`, `TestBuildCommandTailscale`, `TestBuildCommandUnknownType` verify that command-line arguments are correctly assembled and that secrets never appear in argv. -- **Tailscale tagged registration:** `TestParseTailscaleTags` (table-driven) checks that tags are trimmed, empty entries dropped and bare names given the `tag:` prefix, and that invalid tags are rejected: empty names, a leading digit, spaces, other prefixes, flag-like values and shell metacharacters. `TestTailscaleUpArgs` and `TestTailscaleStatusArgs` check the exact argv. The auth key never appears in it. `TestRegisterTailscaleOnceRequestsTags` checks that exactly one tagged `tailscale up` runs from `NeedsLogin`. `TestRegisterTailscaleOnceWaitsThroughStarting` checks that a transient state is polled past and that a node already `Running` with the same tags is skipped. `TestRegisterTailscaleOnceRunningWithOtherTags` checks that a node with different tags is re-registered. `TestRegisterTailscaleOnceFallsBackUntagged` checks that the tagged failure is logged with the `tagOwners` hint and followed by the exact untagged argv. `TestRegisterTailscaleOnceNoFallbackWhenStillRunning` checks that no second login runs on a node that is still up. `TestStartTailscaleRegistrarReturnsAfterFailure` checks that `wait()` returns after failures, so `run`'s shutdown is never blocked. `TestRegisterTailscaleOnceStopsOnCancel` checks that cancellation ends the wait without running `up`. +- **Config rendering:** frp and playit tests verify private file formats, escaping, credential delivery and invalid mappings. +- **Command building:** frp/playit tests check argv and fixed paths. +- **Tailscale:** local socket regressions cover bidirectional TCP/UDP, peer isolation, shared session limits, idle eviction and shutdown. Lifecycle tests cover hostname/tag configuration, denied registration/tags, dual-stack listener creation, partial bind cleanup, online/offline readiness and hostname collisions. Operator tests verify protocol projection, readiness generation/probe requirements, unsupported protocols and EmptyDir/Recreate deployment wiring. - **Exponential backoff:** `TestExponentialBackoff`, `TestExponentialBackoffCap` verify delay calculations and the 300-second cap. - **Error classification:** `TestIsUnrecoverable` verifies that exit codes 126/127 and "permission denied" errors are unrecoverable, and that other errors trigger retry. - **Supervision lifecycle:** `TestRunContextCancellation`, `TestRunTransientFailureBacksOffThenCancels`, `TestRunRenderConfigFailure`, `TestRunReadCredentialsFailure`, `TestRunPlayitConfigDispatch` verify clean context cancellation, backoff retry, and error path handling. @@ -267,12 +266,12 @@ No third-party dependencies. The operator provides provider-specific binaries (f **Test doubles:** - **`withCredentialsDir`:** Temporary directory helper that repoints the package-level `credentialsDir` var at `t.TempDir()` for the duration of a test, allowing credential reads to be exercised without touching the real mount point. -- **`stubTailscaleCLI` / `fakeTailscale`:** `stubTailscaleCLI` points the package-level `runTailscaleCLI` var at a fake `func(ctx, args...) ([]byte, error)` for the duration of a test. This is the same seam pattern as `credentialsDir`. `fakeTailscale` scripts the `status --json` replies and `up` results and records each `up` argv, so `waitForTailscaled` and `registerTailscaleOnce` can be tested without running a real `tailscale` binary. +- **`fakeTailnetNode`:** controls registration/status outcomes while opening real local listeners; no provider credentials are needed. - **Real system binaries in some tests:** `TestRunCommandContextCancellation` uses the real `sleep` command (no relay binary needed) to verify that context cancellation triggers SIGTERM forwarding, allowing the child to exit cleanly within the 10-second grace period. **Coverage gate:** 70% per `.testcoverage.yml`. Uncovered paths include error cases in relay binary spawning (operator-level validation and pod constraints already catch most configuration issues before the tunnel binary runs) and platform-specific signal handling edge cases. -**Coverage rationale:** The tunnel supervisor's primary responsibility is configuration management and child process supervision. Tests cover all config parsing, credential reading, config rendering, command building, and supervision loop paths. The relay binaries themselves (frpc, tailscaled, playitd) are third-party and tested by their own projects. The tunnel pod's job is to correctly invoke them and restart on failure; actual relay functionality is end-to-end tested at the e2e suite level. +**Coverage rationale:** The tunnel supervisor's primary responsibility is configuration management and child process supervision. Tests cover all config parsing, credential reading, config rendering, command building, and supervision loop paths. frpc/playitd and the embedded tsnet library are third-party; local socket tests cover Gameplane's bridge. The tunnel pod's job is to correctly invoke them and restart on failure; actual relay functionality is end-to-end tested at the e2e suite level. ## References diff --git a/tunnel/tailscale.go b/tunnel/tailscale.go new file mode 100644 index 000000000..1a8eaff17 --- /dev/null +++ b/tunnel/tailscale.go @@ -0,0 +1,197 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "net/http" + "net/netip" + "slices" + "strings" + "sync/atomic" + "time" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/tsnet" +) + +const tailscaleStateDir = "/var/lib/gameplane-tailscale" + +type tailnetNode interface { + Up(context.Context) (*ipnstate.Status, error) + Status(context.Context) (*ipnstate.Status, error) + Listen(string, string) (net.Listener, error) + ListenPacket(string, string) (net.PacketConn, error) +} + +type embeddedTailnet struct{ *tsnet.Server } + +func (s embeddedTailnet) Up(ctx context.Context) (*ipnstate.Status, error) { + status, err := s.Server.Up(ctx) + if err != nil { + return nil, err + } + return checkTailnetHostname(status, s.Hostname) +} + +func (s embeddedTailnet) Status(ctx context.Context) (*ipnstate.Status, error) { + client, err := s.LocalClient() + if err != nil { + return nil, err + } + status, err := client.StatusWithoutPeers(ctx) + if err != nil { + return nil, err + } + return checkTailnetHostname(status, s.Hostname) +} + +func checkTailnetHostname(status *ipnstate.Status, hostname string) (*ipnstate.Status, error) { + if status == nil || status.Self == nil { + return nil, errors.New("tailnet identity unavailable") + } + name, _, _ := strings.Cut(status.Self.DNSName, ".") + if name != hostname { + return nil, errors.New("tailnet hostname differs from advertised endpoint; enable MagicDNS and remove conflicting device names") + } + return status, nil +} + +func tailscaleServer(cfg Config, authKey string) (*tsnet.Server, []string, error) { + tags, err := parseTailscaleTags(cfg.TailscaleTags) + if err != nil { + return nil, nil, fmt.Errorf("Tailscale tags: %w", err) + } + if cfg.TailscaleHostname == "" || authKey == "" { + return nil, nil, errors.New("Tailscale hostname and authKey are required") + } + return &tsnet.Server{Hostname: cfg.TailscaleHostname, AuthKey: authKey, AdvertiseTags: tags, Dir: tailscaleStateDir, + // Neither verbose daemon logs nor interactive login URLs belong in pod + // logs. Readiness and sanitized retry messages expose failure instead. + Logf: func(string, ...any) {}, UserLogf: func(string, ...any) {}, + }, tags, nil +} + +func runTailscale(ctx context.Context, cfg Config, authKey string) error { + ports, err := tailscaleForwardPlan(cfg) + if err != nil { + return err + } + var ready atomic.Bool + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", ":8091") + if err != nil { + return fmt.Errorf("Tailscale health listener: %w", err) + } + health := &http.Server{Handler: tailscaleHealth(&ready), ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second} + healthDone := make(chan error, 1) + go func() { healthDone <- health.Serve(listener) }() + defer func() { _ = health.Close() }() + backoff := &exponentialBackoff{} + for ctx.Err() == nil { + server, tags, err := tailscaleServer(cfg, authKey) + if err != nil { + return err + } + err = serveTailnet(ctx, embeddedTailnet{server}, tags, ports, &ready, 2*time.Second) + ready.Store(false) + _ = server.Close() + if ctx.Err() != nil { + return nil + } + // Up errors may embed registration responses. Keep credentials and + // login URLs out of logs; the operator exposes a pending condition. + if err != nil { + log.Print("Tailscale forwarding unavailable; check credentials, requested tagOwners, device approval, unique hostname, MagicDNS and tailnet connectivity") + } + select { + case <-ctx.Done(): + return nil + case err := <-healthDone: + return fmt.Errorf("Tailscale health server: %w", err) + case <-time.After(backoff.next()): + } + } + return nil +} + +func tailscaleHealth(ready *atomic.Bool) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) + mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { + if !ready.Load() { + http.Error(w, "tailnet or forwarding listeners unavailable", http.StatusServiceUnavailable) + return + } + w.WriteHeader(http.StatusOK) + }) + return mux +} + +func tailnetReady(status *ipnstate.Status, tags []string) bool { + if status == nil || status.BackendState != "Running" || status.Self == nil || !status.Self.Online || len(status.TailscaleIPs) == 0 { + return false + } + for _, tag := range tags { + if status.Self.Tags == nil || !slices.Contains(status.Self.Tags.AsSlice(), tag) { + return false + } + } + return true +} + +func serveTailnet(ctx context.Context, node tailnetNode, tags []string, ports []forwardPort, ready *atomic.Bool, poll time.Duration) error { + ready.Store(false) + defer ready.Store(false) + upCtx, cancel := context.WithTimeout(ctx, 2*time.Minute) + status, err := node.Up(upCtx) + cancel() + if err != nil { + return fmt.Errorf("join tailnet: %w", err) + } + if !tailnetReady(status, tags) { + return errors.New("tailnet is not online with the requested tags") + } + ips := append([]netip.Addr(nil), status.TailscaleIPs...) + f := newForwarder(ctx, defaultForwardLimits) + defer f.Close() + for _, port := range ports { + if port.network == "tcp" { + listener, err := node.Listen("tcp", ":"+port.port) + if err != nil { + return fmt.Errorf("TCP listener %s: %w", port.port, err) + } + f.serveTCP(listener, port.target) + } else { + // tsnet requires an explicit IP for UDP. Bind both families so + // MagicDNS choosing IPv6 cannot bypass the advertised datapath. + for _, ip := range ips { + listener, err := node.ListenPacket("udp", net.JoinHostPort(ip.String(), port.port)) + if err != nil { + return fmt.Errorf("UDP listener %s: %w", port.port, err) + } + f.serveUDP(listener, port.target) + } + } + } + ready.Store(true) + ticker := time.NewTicker(poll) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return nil + case err := <-f.failed: + return err + case <-ticker.C: + statusCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + status, err := node.Status(statusCtx) + cancel() + ready.Store(err == nil && tailnetReady(status, tags)) + if err == nil && status != nil && !slices.Equal(status.TailscaleIPs, ips) { + return errors.New("tailnet addresses changed; recreating forwarding listeners") + } + } + } +} diff --git a/tunnel/tailscale_forward.go b/tunnel/tailscale_forward.go new file mode 100644 index 000000000..d55b6bc06 --- /dev/null +++ b/tunnel/tailscale_forward.go @@ -0,0 +1,253 @@ +package main + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "strconv" + "strings" + "sync" + "time" +) + +type forwardPort struct{ network, port, target string } + +// The operator supplies only advertised ports and this exact Service. No client +// packet can choose a destination, and this node never advertises subnet routes. +func tailscaleForwardPlan(cfg Config) ([]forwardPort, error) { + if cfg.GameServerName == "" || cfg.GameServerNamespace == "" || cfg.BackingServiceDNS != cfg.GameServerName+"."+cfg.GameServerNamespace+".svc" { + return nil, errors.New("Tailscale target must be the GameServer's backing Service DNS") + } + entries := strings.Split(cfg.BackingServicePorts, ",") + if len(entries) > 32 { + return nil, errors.New("Tailscale supports at most 32 advertised ports") + } + ports := make([]forwardPort, 0, len(entries)) + seen := map[string]bool{} + for _, entry := range entries { + parts := strings.Split(entry, ":") + if len(parts) != 3 || parts[0] == "" { + return nil, fmt.Errorf("Tailscale port %q must be name:port:tcp or name:port:udp; update the operator and tunnel images together", entry) + } + port, err := strconv.Atoi(parts[1]) + if err != nil || port < 1 || port > 65535 { + return nil, fmt.Errorf("invalid Tailscale port %q", entry) + } + if parts[2] != "tcp" && parts[2] != "udp" { + return nil, fmt.Errorf("unsupported Tailscale protocol %q; only TCP and UDP can be forwarded", parts[2]) + } + key := parts[2] + ":" + strconv.Itoa(port) + if seen[key] { + return nil, fmt.Errorf("duplicate Tailscale listener %s", key) + } + seen[key] = true + ports = append(ports, forwardPort{network: parts[2], port: strconv.Itoa(port), target: net.JoinHostPort(cfg.BackingServiceDNS, strconv.Itoa(port))}) + } + return ports, nil +} + +type forwardLimits struct { + sessions, udpQueue int + idle, dial time.Duration +} + +// The budget is shared by all listeners, protocols and address families. +// At most 128 sessions, four queued UDP packets each and one 64KiB response +// buffer per UDP session bound memory independent of client fan-out. +var defaultForwardLimits = forwardLimits{sessions: 128, udpQueue: 4, idle: 30 * time.Second, dial: 5 * time.Second} + +type forwarder struct { + ctx context.Context + cancel context.CancelFunc + limits forwardLimits + slots chan struct{} + wg sync.WaitGroup + listeners []io.Closer // registered before Close; no concurrent additions + closeOnce sync.Once + failed chan error +} + +func newForwarder(ctx context.Context, limits forwardLimits) *forwarder { + ctx, cancel := context.WithCancel(ctx) + return &forwarder{ctx: ctx, cancel: cancel, limits: limits, slots: make(chan struct{}, limits.sessions), failed: make(chan error, 1)} +} + +func (f *forwarder) Close() { + f.closeOnce.Do(func() { + f.cancel() + for _, listener := range f.listeners { + _ = listener.Close() + } + f.wg.Wait() + }) +} + +func (f *forwarder) fail(err error) { + if f.ctx.Err() == nil { + select { + case f.failed <- err: + default: + } + } +} + +func (f *forwarder) acquire() bool { + select { + case f.slots <- struct{}{}: + return true + default: + return false + } +} + +func (f *forwarder) serveTCP(listener net.Listener, target string) { + f.listeners = append(f.listeners, listener) + f.wg.Add(1) + go func() { + defer f.wg.Done() + for { + client, err := listener.Accept() + if err != nil { + f.fail(fmt.Errorf("TCP listener: %w", err)) + return + } + if !f.acquire() { + _ = client.Close() + continue + } + f.wg.Add(1) + go func() { defer f.wg.Done(); defer func() { <-f.slots }(); f.proxyTCP(client, target) }() + } + }() +} + +func (f *forwarder) proxyTCP(client net.Conn, target string) { + defer func() { _ = client.Close() }() + stopClient := context.AfterFunc(f.ctx, func() { _ = client.Close() }) + defer stopClient() + upstream, err := (&net.Dialer{Timeout: f.limits.dial}).DialContext(f.ctx, "tcp", target) + if err != nil { + return + } + defer func() { _ = upstream.Close() }() + stopUpstream := context.AfterFunc(f.ctx, func() { _ = upstream.Close() }) + defer stopUpstream() + // Activity in either direction refreshes each socket's idle timeout. + a := &idleConn{Conn: client, idle: 5 * time.Minute} + b := &idleConn{Conn: upstream, idle: 5 * time.Minute} + done := make(chan struct{}) + go func() { _, _ = io.CopyBuffer(b, a, make([]byte, 32*1024)); closeWrite(upstream); close(done) }() + _, _ = io.CopyBuffer(a, b, make([]byte, 32*1024)) + closeWrite(client) + _ = upstream.Close() + _ = client.Close() + <-done +} + +func closeWrite(conn net.Conn) { + if half, ok := conn.(interface{ CloseWrite() error }); ok { + _ = half.CloseWrite() + } +} + +type idleConn struct { + net.Conn + idle time.Duration +} + +func (c *idleConn) Read(p []byte) (int, error) { + _ = c.SetDeadline(time.Now().Add(c.idle)) + return c.Conn.Read(p) +} +func (c *idleConn) Write(p []byte) (int, error) { + _ = c.SetDeadline(time.Now().Add(c.idle)) + return c.Conn.Write(p) +} + +type udpForwardSession struct{ packets chan []byte } + +func (f *forwarder) serveUDP(listener net.PacketConn, target string) { + f.listeners = append(f.listeners, listener) + f.wg.Add(1) + go func() { + defer f.wg.Done() + var mu sync.Mutex + sessions := map[string]*udpForwardSession{} + buf := make([]byte, 65536) + for { + n, peer, err := listener.ReadFrom(buf) + if err != nil { + f.fail(fmt.Errorf("UDP listener: %w", err)) + return + } + if n > 65507 || f.ctx.Err() != nil { + continue + } + key := peer.String() + mu.Lock() + session := sessions[key] + if session == nil && f.acquire() { + session = &udpForwardSession{packets: make(chan []byte, f.limits.udpQueue)} + sessions[key] = session + f.wg.Add(1) + go func(session *udpForwardSession, peer net.Addr, key string) { + defer f.wg.Done() + defer func() { mu.Lock(); delete(sessions, key); mu.Unlock(); <-f.slots }() + f.proxyUDP(listener, peer, session.packets, target) + }(session, peer, key) + } + if session != nil { + // A slow backend or full queue drops the datagram; neither makes + // the receive loop wait, allocate workers, or grow memory. + if len(session.packets) < cap(session.packets) { + session.packets <- append([]byte(nil), buf[:n]...) + } + } + mu.Unlock() + } + }() +} + +func (f *forwarder) proxyUDP(listener net.PacketConn, peer net.Addr, packets <-chan []byte, target string) { + upstream, err := (&net.Dialer{Timeout: f.limits.dial}).DialContext(f.ctx, "udp", target) + if err != nil { + return + } + stop := context.AfterFunc(f.ctx, func() { _ = upstream.Close() }) + defer stop() + done := make(chan struct{}) + _ = upstream.SetReadDeadline(time.Now().Add(f.limits.idle)) + go func() { + defer close(done) + buf := make([]byte, 65536) + for { + n, err := upstream.Read(buf) + if err != nil { + return + } + if n <= 65507 { + _ = listener.SetWriteDeadline(time.Now().Add(f.limits.dial)) + if _, err := listener.WriteTo(buf[:n], peer); err != nil { + return + } + } + } + }() + defer func() { _ = upstream.Close(); <-done }() + for { + select { + case <-f.ctx.Done(): + return + case <-done: + return + case packet := <-packets: + _ = upstream.SetWriteDeadline(time.Now().Add(f.limits.dial)) + _ = upstream.SetReadDeadline(time.Now().Add(f.limits.idle)) + if _, err := upstream.Write(packet); err != nil { + return + } + } + } +} diff --git a/tunnel/tailscale_forward_test.go b/tunnel/tailscale_forward_test.go new file mode 100644 index 000000000..20376bbb2 --- /dev/null +++ b/tunnel/tailscale_forward_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "io" + "net" + "strconv" + "strings" + "testing" + "time" +) + +func TestTailscaleForwardPlanRestrictsDestinations(t *testing.T) { + cfg := Config{GameServerName: "game", GameServerNamespace: "games", BackingServiceDNS: "game.games.svc", BackingServicePorts: "java:25565:tcp,bedrock:19132:udp"} + ports, err := tailscaleForwardPlan(cfg) + if err != nil || len(ports) != 2 || ports[0].target != "game.games.svc:25565" || ports[1].network != "udp" { + t.Fatalf("plan: %+v, %v", ports, err) + } + for _, bad := range []string{"other.games.svc", "127.0.0.1", "game.games.svc:80", "http://game.games.svc"} { + cfg.BackingServiceDNS = bad + if _, err := tailscaleForwardPlan(cfg); err == nil { + t.Fatalf("accepted unrelated target %q", bad) + } + } + cfg.BackingServiceDNS = "game.games.svc" + for _, bad := range []string{"game:25565", "game:0:tcp", "game:65536:udp", "game:22:sctp", "game:22:tcp,other:22:tcp", strings.Repeat("p:1:tcp,", 33)} { + cfg.BackingServicePorts = bad + if _, err := tailscaleForwardPlan(cfg); err == nil { + t.Fatalf("accepted invalid ports %q", bad) + } + } +} + +func TestTailscaleTCPForwardAndShutdown(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + backend, err := (&net.ListenConfig{}).Listen(ctx, "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer backend.Close() + go func() { + c, e := backend.Accept() + if e == nil { + defer c.Close() + _, _ = io.Copy(c, c) + } + }() + listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f := newForwarder(ctx, forwardLimits{sessions: 2, udpQueue: 2, idle: time.Second, dial: time.Second}) + f.serveTCP(listener, backend.Addr().String()) + defer f.Close() + conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + _ = conn.SetDeadline(time.Now().Add(3 * time.Second)) + _, _ = conn.Write([]byte("gameplay")) + buf := make([]byte, 8) + if _, err := io.ReadFull(conn, buf); err != nil || string(buf) != "gameplay" { + t.Fatalf("TCP reply %q: %v", buf, err) + } + f.Close() + if _, err := conn.Read(buf); err == nil { + t.Fatal("connection survived forwarding shutdown") + } +} + +func TestTailscaleUDPForwardIsolationLimitAndExpiry(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + backend, err := (&net.ListenConfig{}).ListenPacket(ctx, "udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer backend.Close() + go func() { + buf := make([]byte, 65536) + for { + n, peer, e := backend.ReadFrom(buf) + if e != nil { + return + } + _, _ = backend.WriteTo(buf[:n], peer) + } + }() + listener, err := (&net.ListenConfig{}).ListenPacket(ctx, "udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f := newForwarder(ctx, forwardLimits{sessions: 2, udpQueue: 2, idle: time.Second, dial: time.Second}) + f.serveUDP(listener, backend.LocalAddr().String()) + defer f.Close() + var clients []net.Conn + for i := 0; i < 3; i++ { + c, e := (&net.Dialer{}).DialContext(ctx, "udp", listener.LocalAddr().String()) + if e != nil { + t.Fatal(e) + } + defer c.Close() + clients = append(clients, c) + } + for i := 0; i < 2; i++ { + _ = clients[i].SetDeadline(time.Now().Add(time.Second)) + payload := "peer-" + strconv.Itoa(i) + _, _ = clients[i].Write([]byte(payload)) + buf := make([]byte, 32) + n, e := clients[i].Read(buf) + if e != nil || string(buf[:n]) != payload { + t.Fatalf("peer %d reply %q: %v", i, buf[:n], e) + } + } + _ = clients[2].SetDeadline(time.Now().Add(50 * time.Millisecond)) + _, _ = clients[2].Write([]byte("over limit")) + if _, err := clients[2].Read(make([]byte, 32)); err == nil { + t.Fatal("UDP session limit not enforced") + } + deadline := time.Now().Add(3 * time.Second) + for len(f.slots) != 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if len(f.slots) != 0 { + t.Fatal("idle UDP sessions were not reclaimed") + } + _ = clients[2].SetDeadline(time.Now().Add(time.Second)) + _, _ = clients[2].Write([]byte("new session")) + buf := make([]byte, 32) + n, err := clients[2].Read(buf) + if err != nil || string(buf[:n]) != "new session" { + t.Fatalf("session after expiry: %q %v", buf[:n], err) + } + f.Close() + if len(f.slots) != 0 { + t.Fatal("shutdown leaked sessions") + } +} diff --git a/tunnel/tailscale_test.go b/tunnel/tailscale_test.go new file mode 100644 index 000000000..d57d4ce6d --- /dev/null +++ b/tunnel/tailscale_test.go @@ -0,0 +1,214 @@ +package main + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "net/netip" + "sync" + "sync/atomic" + "testing" + "time" + + "tailscale.com/ipn/ipnstate" + "tailscale.com/types/views" +) + +func TestTailscaleServerPreservesIdentityAndTags(t *testing.T) { + cfg := Config{TailscaleHostname: "my-game", TailscaleTags: "game,tag:game,tag:friends"} + s, tags, err := tailscaleServer(cfg, "test-auth-key") + if err != nil { + t.Fatal(err) + } + if s.Hostname != "my-game" || s.AuthKey != "test-auth-key" || s.Dir != tailscaleStateDir || s.Ephemeral || len(tags) != 2 || !equalArgs(tags, s.AdvertiseTags) { + t.Fatalf("server lost identity or tags: %+v", tags) + } + cfg.TailscaleTags = "tag:bad tag" + if _, _, err := tailscaleServer(cfg, "key"); err == nil { + t.Fatal("invalid tags silently dropped") + } + cfg.TailscaleTags = "" + if _, _, err := tailscaleServer(cfg, ""); err == nil { + t.Fatal("empty auth key accepted") + } +} + +func onlineTailnet() *ipnstate.Status { + tags := views.SliceOf([]string{"tag:game"}) + return &ipnstate.Status{BackendState: "Running", TailscaleIPs: []netip.Addr{netip.MustParseAddr("100.64.0.1"), netip.MustParseAddr("fd7a:115c:a1e0::1")}, Self: &ipnstate.PeerStatus{Online: true, Tags: &tags}} +} + +func TestTailscaleHostnameMustMatchAdvertisedEndpoint(t *testing.T) { + status := onlineTailnet() + status.Self.DNSName = "game.tailnet.ts.net." + if _, err := checkTailnetHostname(status, "game"); err != nil { + t.Fatal(err) + } + status.Self.DNSName = "game-1.tailnet.ts.net." + if _, err := checkTailnetHostname(status, "game"); err == nil { + t.Fatal("colliding hostname advertised as ready") + } +} + +type fakeTailnetNode struct { + mu sync.Mutex + status *ipnstate.Status + upErr error + listenErr bool + listened []string + listeners []net.Listener + packets []net.PacketConn +} + +func (f *fakeTailnetNode) Up(ctx context.Context) (*ipnstate.Status, error) { + if f.upErr != nil { + return nil, f.upErr + } + return f.Status(ctx) +} +func (f *fakeTailnetNode) Status(_ context.Context) (*ipnstate.Status, error) { + f.mu.Lock() + defer f.mu.Unlock() + return f.status, nil +} +func (f *fakeTailnetNode) Listen(network, addr string) (net.Listener, error) { + f.listened = append(f.listened, network+addr) + l, err := (&net.ListenConfig{}).Listen(context.Background(), network, "127.0.0.1:0") + if err == nil { + f.listeners = append(f.listeners, l) + } + return l, err +} +func (f *fakeTailnetNode) ListenPacket(network, addr string) (net.PacketConn, error) { + f.listened = append(f.listened, network+addr) + if f.listenErr { + return nil, errors.New("bind failed") + } + l, err := (&net.ListenConfig{}).ListenPacket(context.Background(), network, "127.0.0.1:0") + if err == nil { + f.packets = append(f.packets, l) + } + return l, err +} + +func awaitTailnetReady(t *testing.T, ready *atomic.Bool, want bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for ready.Load() != want && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if ready.Load() != want { + t.Fatalf("readiness=%v want %v", ready.Load(), want) + } +} + +func TestTailnetReadinessTracksAuthenticationListenersAndDisconnect(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + node := &fakeTailnetNode{status: onlineTailnet()} + var ready atomic.Bool + ports := []forwardPort{{network: "tcp", port: "25565", target: "127.0.0.1:1"}, {network: "udp", port: "19132", target: "127.0.0.1:2"}} + done := make(chan error, 1) + go func() { done <- serveTailnet(ctx, node, []string{"tag:game"}, ports, &ready, time.Millisecond) }() + awaitTailnetReady(t, &ready, true) + if len(node.listened) != 3 || node.listened[0] != "tcp:25565" || node.listened[1] != "udp100.64.0.1:19132" || node.listened[2] != "udp[fd7a:115c:a1e0::1]:19132" { + t.Fatalf("unexpected port/family exposure: %v", node.listened) + } + node.mu.Lock() + offline := onlineTailnet() + offline.Self.Online = false + node.status = offline + node.mu.Unlock() + awaitTailnetReady(t, &ready, false) + node.mu.Lock() + node.status = onlineTailnet() + node.mu.Unlock() + awaitTailnetReady(t, &ready, true) + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("shutdown stalled") + } + if ready.Load() { + t.Fatal("readiness survives shutdown") + } + for _, listener := range node.listeners { + if _, err := listener.Accept(); err == nil { + t.Fatal("listener survives shutdown") + } + } +} + +func TestTailnetFailuresNeverBecomeReady(t *testing.T) { + for _, scenario := range []string{"authentication", "denied tag", "listener"} { + t.Run(scenario, func(t *testing.T) { + node := &fakeTailnetNode{status: onlineTailnet()} + tags := []string{"tag:game"} + switch scenario { + case "authentication": + node.upErr = errors.New("unauthorized") + case "denied tag": + tags = []string{"tag:ungranted"} + case "listener": + node.listenErr = true + } + var ready atomic.Bool + err := serveTailnet(context.Background(), node, tags, []forwardPort{{network: "tcp", port: "1", target: "127.0.0.1:1"}, {network: "udp", port: "2", target: "127.0.0.1:2"}}, &ready, time.Millisecond) + if err == nil || ready.Load() { + t.Fatalf("failure became ready: %v", err) + } + for _, listener := range node.listeners { + if _, err := listener.Accept(); err == nil { + t.Fatal("partially opened listener leaked") + } + } + }) + } +} + +func TestTailnetListenerFailureClearsReady(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + node := &fakeTailnetNode{status: onlineTailnet()} + var ready atomic.Bool + done := make(chan error, 1) + go func() { + done <- serveTailnet(ctx, node, nil, []forwardPort{{network: "tcp", port: "25565", target: "127.0.0.1:1"}}, &ready, time.Millisecond) + }() + awaitTailnetReady(t, &ready, true) + _ = node.listeners[0].Close() + select { + case err := <-done: + if err == nil { + t.Fatal("failed listener was not reported") + } + case <-time.After(time.Second): + t.Fatal("failed listener did not stop forwarding") + } + if ready.Load() { + t.Fatal("failed forwarding listener still ready") + } +} + +func TestTailnetHealthNeedsForwardingReadiness(t *testing.T) { + var ready atomic.Bool + handler := tailscaleHealth(&ready) + for _, check := range []struct { + path string + ready bool + want int + }{{"/readyz", false, 503}, {"/healthz", false, 200}, {"/readyz", true, 200}, {"/readyz", false, 503}} { + ready.Store(check.ready) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequestWithContext(context.Background(), http.MethodGet, check.path, nil)) + if rec.Code != check.want { + t.Fatalf("%s=%d want%d", check.path, rec.Code, check.want) + } + } +} From 94b7fd76b75a245172cb9437c60672619fd5f889 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 11:50:34 -0400 Subject: [PATCH 04/17] fix: handle wrapped decoder EOF in placement drill Signed-off-by: Baylem --- test/e2e/management_placement_e2e_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/management_placement_e2e_test.go b/test/e2e/management_placement_e2e_test.go index c212de9ea..ec4525dfa 100644 --- a/test/e2e/management_placement_e2e_test.go +++ b/test/e2e/management_placement_e2e_test.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -106,7 +107,7 @@ func TestManagementPlacementAndDisruption(t *testing.T) { decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 4096) for { var object unstructured.Unstructured - if err := decoder.Decode(&object); err == io.EOF { + if err := decoder.Decode(&object); errors.Is(err, io.EOF) { break } else if err != nil { t.Fatal(err) From 65d1beb41b7b2a00a9f26d69c9a6b1cd49b957cd Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:09:34 -0400 Subject: [PATCH 05/17] fix(tunnel): isolate frp identities and report relay health Signed-off-by: Baylem --- .github/workflows/ci.yaml | 8 +- .../crds/gameplane.local_backups.yaml | 2 +- .../crds/gameplane.local_backupschedules.yaml | 2 +- .../crds/gameplane.local_clusters.yaml | 2 +- .../crds/gameplane.local_gameservers.yaml | 17 +- .../crds/gameplane.local_gametemplates.yaml | 2 +- .../crds/gameplane.local_modules.yaml | 2 +- .../crds/gameplane.local_modulesources.yaml | 2 +- .../crds/gameplane.local_networkcaptures.yaml | 2 +- .../crds/gameplane.local_restores.yaml | 2 +- .../gameplane.local_backups.yaml | 2 +- .../gameplane.local_backupschedules.yaml | 2 +- .../gameplane.local_clusters.yaml | 2 +- .../gameplane.local_gameservers.yaml | 17 +- .../gameplane.local_gametemplates.yaml | 2 +- .../gameplane.local_modules.yaml | 2 +- .../gameplane.local_modulesources.yaml | 2 +- .../gameplane.local_networkcaptures.yaml | 2 +- .../gameplane.local_restores.yaml | 2 +- docs/relay-health.md | 110 +++ docs/tailscale-forwarding-integration.md | 150 ++++ docs/tunnels.md | 41 +- go.work | 2 +- go.work.sum | 9 +- i18n/locales/en/app.json | 12 + operator/api/v1alpha1/gameserver_types.go | 15 +- .../crd/gameplane.local_gameservers.yaml | 15 +- operator/hack/condition-ids/main.go | 3 + .../gameserver_relay_health_test.go | 106 +++ .../internal/controller/gameserver_status.go | 99 ++- .../controller/gameserver_tailscale_test.go | 48 ++ .../internal/controller/gameserver_tunnel.go | 53 +- .../gameserver_tunnel_envtest_test.go | 26 + operator/specs.md | 3 + .../OPEN-DECISIONS.md | 6 + test/e2e/optional_components_e2e_test.go | 12 +- test/e2e/relay_health_e2e_test.go | 280 ++++++++ test/e2e/specs.md | 2 + tunnel/Dockerfile.frp | 7 +- tunnel/Dockerfile.playit | 3 +- tunnel/Dockerfile.tailscale | 10 +- tunnel/go.mod | 50 +- tunnel/go.sum | 236 +++++++ tunnel/main.go | 542 ++------------- tunnel/main_test.go | 650 +----------------- tunnel/playit_reporter.go | 4 +- tunnel/relay_health.go | 272 ++++++++ tunnel/relay_health_test.go | 223 ++++++ tunnel/specs.md | 119 ++-- tunnel/tailscale.go | 218 ++++++ tunnel/tailscale_forward.go | 253 +++++++ tunnel/tailscale_forward_test.go | 140 ++++ tunnel/tailscale_test.go | 249 +++++++ 53 files changed, 2734 insertions(+), 1308 deletions(-) create mode 100644 docs/relay-health.md create mode 100644 docs/tailscale-forwarding-integration.md create mode 100644 operator/internal/controller/gameserver_relay_health_test.go create mode 100644 operator/internal/controller/gameserver_tailscale_test.go create mode 100644 test/e2e/relay_health_e2e_test.go create mode 100644 tunnel/go.sum create mode 100644 tunnel/relay_health.go create mode 100644 tunnel/relay_health_test.go create mode 100644 tunnel/tailscale.go create mode 100644 tunnel/tailscale_forward.go create mode 100644 tunnel/tailscale_forward_test.go create mode 100644 tunnel/tailscale_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a902708f0..b1bec0211 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1704,8 +1704,8 @@ jobs: job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite. - bucket: optional-components parallel: 1 - test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order - job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + test_timeout: 35m # lifecycle plus two-GameServer TCP/UDP relay identity, outage and recovery checks + job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite. env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e @@ -1805,8 +1805,8 @@ jobs: job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite. - bucket: optional-components parallel: 1 - test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order - job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + test_timeout: 35m # lifecycle plus two-GameServer TCP/UDP relay identity, outage and recovery checks + job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite. env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index d902ea995..8b0b52abe 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index d85406090..513c2dbca 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 4d6aadcbb..b19459f33 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index 95178e56e..fba969dab 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1546,15 +1546,12 @@ spec: tags: description: |- Tags are Tailscale ACL tags requested for the device when it - registers, via `tailscale up --advertise-tags` (tailscaled's - declarative config has no field for tags, and changing the tags of a - registered node needs a fresh login). Each entry is "tag:" or a bare - name, which gets the "tag:" prefix. The tailnet ACL must grant - tagOwners for these tags to the auth key's owner; if the request is - refused, the tunnel pod logs the error and the tunnel runs untagged. - If any entry is invalid, the whole list is logged and ignored and the - device registers untagged. See - tunnel/specs.md. + registers through embedded tsnet. Each entry is "tag:" or a + bare name, which gets the "tag:" prefix. The tailnet policy must grant + tagOwners to the auth key's owner. Invalid or refused tags keep the + tunnel not ready; it never silently registers without requested tags. + Changing tags replaces the Pod and requires a reusable auth key. + See tunnel/specs.md for identity persistence and registration details. items: type: string maxItems: 8 diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index e5e3e1e3b..9d2de5bda 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 5ade28cee..b0e1c69b5 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 6c5dcbf36..953e12936 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index cc113e7e8..ef8e19316 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index b77cd5c25..14d0478f0 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index d902ea995..8b0b52abe 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index d85406090..513c2dbca 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 4d6aadcbb..b19459f33 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index 95178e56e..fba969dab 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1546,15 +1546,12 @@ spec: tags: description: |- Tags are Tailscale ACL tags requested for the device when it - registers, via `tailscale up --advertise-tags` (tailscaled's - declarative config has no field for tags, and changing the tags of a - registered node needs a fresh login). Each entry is "tag:" or a bare - name, which gets the "tag:" prefix. The tailnet ACL must grant - tagOwners for these tags to the auth key's owner; if the request is - refused, the tunnel pod logs the error and the tunnel runs untagged. - If any entry is invalid, the whole list is logged and ignored and the - device registers untagged. See - tunnel/specs.md. + registers through embedded tsnet. Each entry is "tag:" or a + bare name, which gets the "tag:" prefix. The tailnet policy must grant + tagOwners to the auth key's owner. Invalid or refused tags keep the + tunnel not ready; it never silently registers without requested tags. + Changing tags replaces the Pod and requires a reusable auth key. + See tunnel/specs.md for identity persistence and registration details. items: type: string maxItems: 8 diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index e5e3e1e3b..9d2de5bda 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 5ade28cee..b0e1c69b5 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 6c5dcbf36..953e12936 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index cc113e7e8..ef8e19316 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index b77cd5c25..14d0478f0 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 83d2087948ea79ebc932231b61b4cb802e0d778cdf4736dfdbf03858daafdcd8 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/docs/relay-health.md b/docs/relay-health.md new file mode 100644 index 000000000..6be7a1658 --- /dev/null +++ b/docs/relay-health.md @@ -0,0 +1,110 @@ +# Relay readiness and registration identity + +The frp/Playit wrapper's process health and its provider connection are separate. +On Pod TCP port 8091, `/healthz` reports that the wrapper's HTTP server is alive, +`/readyz` returns 200 only when the provider connection and configured forwards +are confirmed, and `/status` returns a small JSON state without credentials or +provider response text. The operator installs a startup probe on `/healthz` and +a readiness probe on `/readyz`. An external relay outage does not trigger a +liveness restart of the wrapper. + +Provider checks run every two seconds with a two-second query deadline. Local +observations expire after ten seconds, and child exit immediately clears them. +The operator also requires the current Deployment generation, updated replicas +and the expected readiness probe; an old Pod without the probe cannot satisfy +the new readiness contract. Configured addresses remain visible while the +connection is unconfirmed, but `TunnelReady` is not true. + +## frp + +The generated frpc TOML sets `user` to the immutable GameServer UID. The wire +registration is `.`; the GameTemplate's port names and +`remotePorts[].name` do not change. Pod and container restarts reuse the UID. +Deleting and recreating a GameServer creates a new identity, including when its +namespace and name are reused. Distinct clusters' UIDs also separate their +registrations on a shared frps. + +frpc's authenticated status API listens only on `127.0.0.1:7400`. It uses the +mounted token as its local admin password; it is never exposed by a Service or +forwarded from the public relay. The wrapper does not use an environment HTTP +proxy or follow redirects for this request. Every configured TCP/UDP proxy must +appear with the matching name and protocol, a `running` status and no error. +Authentication failure, an occupied registration, absent proxy or unavailable +local API cannot produce readiness. This confirms the relay registrations, not +the health of a sleeping game backend. + +Application heartbeat settings are five seconds between requests and fifteen +seconds before a silent control connection is closed; TCP multiplex keepalive +also uses five seconds. Allow the provider detection interval, local check and +Kubernetes probe/reconcile intervals for an outage to reach the dashboard. + +### Upgrade and replacement + +Upgrade the operator and tunnel image together. The new image requires +`GAMESERVER_UID`, so a mixed old-operator/new-image installation fails closed. +Existing unprefixed proxy names change to UID-prefixed names. Stop the old +client, wait for frps to unregister its old proxies and release their public +ports, or remove those registrations using the frps administrator's normal +procedure. Readiness stays false while a port is still occupied. Do not remove +an unrelated GameServer's registration to make an upgrade succeed. + +The operator uses one replica and the Recreate strategy for relay Deployments. +It never deliberately rolls out two clients with the same provider identity. +Replacement has a short forwarding interruption while the old connection +closes and the new one registers. When a deleted old Pod's node is unreachable, +wait for frps to expire its old session before expecting the replacement ready. +Do not manually scale a relay Deployment above one replica. + +## Playit connection evidence + +The pinned Playit v1.0.10 IPC v2 contract exposes lifecycle and address +assignment, but no authenticated relay heartbeat or connection timestamp. +Its daemon caches `Running` after a successful API refresh and leaves that +cache unchanged when later refreshes fail. Receiving another `get_state` +response therefore cannot prove that a relay is connected or refresh a health +lease. Process age and the presence of an assigned address do not prove it +either. + +Gameplane keeps reporting assigned addresses, but Playit's `/readyz` returns +503 and `TunnelReady` reports `RelayConnectionUnknown`; before assignment it +reports `AwaitingAddress`. `/healthz` remains successful, the child keeps running +and forwarding when it can, and Kubernetes does not restart it solely for this +unknown connection state. The JSON status distinguishes known assignment from +unknown connection. This is a deliberate limitation of the pinned provider, +not a claim that every Playit tunnel is disconnected. + +Positive Playit readiness requires a future provider interface that exposes +fresh, authenticated relay connection evidence, or a separately designed +end-to-end probe. The relevant upstream definitions are +[the IPC model](https://github.com/playit-cloud/playit-agent/blob/v1.0.10/packages/playit-ipc/src/model.rs) +and [the daemon's cached lifecycle publisher](https://github.com/playit-cloud/playit-agent/blob/v1.0.10/packages/playitd/src/daemon.rs). + +## Retry and resource policy + +Unexpected child exits, including clean exits, wait 1, 2, 4, 8 seconds and so on, +up to 300 seconds, before another attempt. Cancellation interrupts the wait. +There is no jitter, as requested by the maintainer. Retry escalation resets only +after one uninterrupted minute of confirmed provider readiness. A failed or +stale observation breaks that interval. A long-lived disconnected process does +not reset it. Playit's unsupported connection evidence cannot reset it either. +The embedded Tailscale loop uses the same interval, fed by confirmed online +tailnet status and active forwarding listeners rather than process lifetime. + +Resources use the existing `operator.tunnelResources` Helm value and +`--tunnel-resources` operator flag from the workload-resource policy. Omitted or +null configuration defaults to requests of 25m CPU and 64Mi memory, with no +limits. A non-null empty object explicitly clears defaults; supplied requests +and limits replace them. These are scheduling reservations, not a fleet +capacity guarantee. Readiness and replacement changes preserve this contract. + +## Regression coverage + +The CI optional-components lifecycle test runs two GameServers from the same +TCP/UDP echo template against one real frps, with different remote ports. An +independent client Pod checks replies identify the intended backend through +both protocols. It also covers registration rejection, replacement isolation, +invalid credentials, a shared relay outage and recovery, and credential-free +logs. Unit tests cover UID validation/stability, namespace/recreation identity, +status decoding, stale expiry, bounded retries/cancellation, healthy-interval +reset, old-deployment readiness, Playit's cached-running limitation, and +resource override removal. These regression tests are executed in CI. diff --git a/docs/tailscale-forwarding-integration.md b/docs/tailscale-forwarding-integration.md new file mode 100644 index 000000000..2874177b5 --- /dev/null +++ b/docs/tailscale-forwarding-integration.md @@ -0,0 +1,150 @@ +# Tailscale forwarding integration + +This procedure needs a disposable Kubernetes installation and a disposable +tailnet with two client devices. It creates real Tailscale devices and sends real +TCP and UDP traffic. It is an integration environment for #722, not evidence +that a live tailnet run has passed. Ordinary CI runs the local socket and +controller regressions without provider credentials. Follow repository policy: +execute this in an authorized integration environment, not a local test run. + +Use the operator and tunnel image built from the same revision. Record their +digests, the revision, client versions, tailnet device IDs, conditions and traffic +results. Do not record auth keys, login URLs or private node state. + +## Provision the disposable environment + +Enable MagicDNS. Create a reusable auth key permitted to request +`tag:gameplane-test`. Configure a tailnet policy with no broad pre-existing allow +rules: permit the approved client to reach that tag on TCP 25565 and UDP 19132. +The second client must have no grant to the tag. For example, replacing the +example email with the approved test user's identity: + +```json +{ + "tagOwners": {"tag:gameplane-test": ["autogroup:admin"]}, + "grants": [{ + "src": ["approved-test-user@example.com"], + "dst": ["tag:gameplane-test"], + "ip": ["tcp:25565", "udp:19132"] + }] +} +``` + +Install this fixture in the disposable Gameplane cluster. Its third TCP port is +deliberately unadvertised. The backend prefixes replies so replacing it can be +distinguished from continuing to reach an old process. + +```yaml +apiVersion: gameplane.local/v1alpha1 +kind: GameTemplate +metadata: + name: tailnet-echo +spec: + displayName: Tailnet echo integration + game: tailnet-echo + version: "1" + image: python:3.14-alpine + consoleMode: none + command: [python, -u, -c] + args: + - | + import socketserver, threading + class TCP(socketserver.BaseRequestHandler): + def handle(self): + while True: + data = self.request.recv(65536) + if not data: return + self.request.sendall(b'backend-v1:' + data) + class UDP(socketserver.BaseRequestHandler): + def handle(self): + data, sock = self.request + sock.sendto(b'backend-v1:' + data, self.client_address) + for port in (25565, 25575): + server = socketserver.ThreadingTCPServer(('0.0.0.0', port), TCP) + threading.Thread(target=server.serve_forever, daemon=True).start() + socketserver.ThreadingUDPServer(('0.0.0.0', 19132), UDP).serve_forever() + ports: + - {name: game, containerPort: 25565, protocol: TCP, advertise: true} + - {name: query, containerPort: 19132, protocol: UDP, advertise: true} + - {name: private, containerPort: 25575, protocol: TCP, advertise: false} +--- +apiVersion: gameplane.local/v1alpha1 +kind: GameServer +metadata: + name: tailnet-echo + namespace: gameplane-games +spec: + templateRef: {name: tailnet-echo} + networking: + expose: ClusterIP + tunnel: + enabled: true + provider: tailscale + credentialsSecretRef: {name: tailnet-echo-tunnel-auth} + tailscale: + hostname: gameplane-forward-test + tags: ["tag:gameplane-test"] +``` + +Provide the reusable auth key through the dashboard's tunnel credential form or +the credential API; these create the required GameServer ownership reference. +Wait for the tunnel Deployment's ready replica and `TunnelReady=True`. Confirm +the admin console shows the requested tag and exact hostname, with no advertised +subnet routes. Record the node ID and tailnet IPs. Remove conflicting stale test +devices if the hostname has acquired a suffix; that state must remain not ready. + +## Exercise the datapath + +On the approved tailnet client, run this with the fixture's MagicDNS name: + +```python +import socket +host = 'gameplane-forward-test' +for kind, port in [(socket.SOCK_STREAM, 25565), (socket.SOCK_DGRAM, 19132)]: + with socket.socket(socket.AF_INET, kind) as client: + client.settimeout(5) + client.connect((socket.gethostbyname(host), port)) + client.sendall(b'game-payload') + assert client.recv(65536) == b'backend-v1:game-payload' +``` + +Repeat with the node's IPv6 tailnet address and `AF_INET6` when the environment +supports IPv6. Run two UDP clients concurrently with distinct payloads; each +must receive only its own reply. TCP 25575 and any other unadvertised port must +fail even from the approved client. Repeat the advertised-port probes from the +denied client; both must time out/fail while the tunnel remains ready. Removing +the approved grant must block new flows too; restore it before continuing. + +Create an unrelated echo Service with a distinguishable prefix in the cluster. +It must not be reachable through this tailnet device on any port, and no route +to its ClusterIP or the cluster's Service/Pod CIDRs may appear on either client. +The local regression also rejects unrelated backing DNS names before listeners +are opened; no incoming packet contains a selectable destination. + +Change the fixture backend prefix to `backend-v2:` and wait for its game Pod +rollout. New TCP connections must receive v2. Stop UDP traffic for over 30 seconds +to expire the old session, then send again and require v2. In the disposable +cluster, delete only the fixture's game Service and wait for the operator to +recreate it; repeat both probes. Existing connections may fail during replacement; +new sessions must resolve the Service again and recover without replacing the +tunnel Pod. + +## Identity, authentication and shutdown outcomes + +| Change | Required outcome | +|---|---| +| Restart the tunnel **container** while retaining the Pod and EmptyDir (use the integration runner's container runtime controls) | Same node ID, hostname and tags reconnect; TCP/UDP recover. | +| Delete the tunnel **Pod**, remove its stale device from the admin console, allow replacement | New identity registers using the reusable key; exact hostname/tags recover and both protocols work. | +| Replace Pod with an expired/revoked key in its credential Secret | Registration never makes the Pod ready and no Tailscale endpoint is advertised. Store a valid reusable key, replace the Pod and require recovery. | +| Request a tag not permitted by tagOwners | Not ready; there must be no untagged fallback device serving traffic. Restore permitted tags and repair credentials/device state, then verify recovery. | +| Rename an existing device to collide with the configured hostname | A suffix assigned to the replacement device cannot make Gameplane advertise the wrong hostname as ready. Resolve the collision and re-register. | +| Disconnect the node from the tailnet/control plane | Readiness turns false after the status/probe intervals; restore connectivity and require recovery. Do not use client ACL denial as a simulation of a disconnected server. | +| Delete/terminate tunnel Pod during active TCP and UDP traffic | Listeners and sessions close, worker shutdown does not hang, and clients reconnect only after the replacement is ready. | + +Remove the GameServer and fixture template, revoke the disposable auth key, and +delete all test devices from the tailnet. Retain only the sanitized result record. + +The implementation uses the official [tsnet APIs](https://tailscale.com/docs/reference/tsnet-server-api). +Its Go 1.26.10 minimum and Go 1.27.3 image builders include the `crypto/x509` +security fix described in the [Go release history](https://go.dev/doc/devel/release) +and [Tailscale security bulletin TS-2026-013](https://tailscale.com/security-bulletins#ts-2026-013). diff --git a/docs/tunnels.md b/docs/tunnels.md index b8ce3fd2a..504d98d96 100644 --- a/docs/tunnels.md +++ b/docs/tunnels.md @@ -197,21 +197,27 @@ specify in `spec.networking.tunnel.tailscale.hostname` (or defaults to the GameServer name if left empty). Port 25565 is available to devices on your tailnet. -> **`tags` requires an ACL grant.** `spec.networking.tunnel.tailscale.tags` -> is requested when the device registers with your tailnet: the tunnel pod -> runs `tailscale up --advertise-tags=` once, over the local daemon's -> socket. Tailscale's headless daemon config has no field for tags, and -> changing the tags of a registered device needs a fresh login (see -> `tunnel/specs.md`). The tailnet's ACL must grant `tagOwners` for the -> requested tags to the auth key's owner. If it doesn't, Tailscale refuses -> the request and the tunnel pod logs the error. If the device is not -> running afterwards, the pod tries once more without tags. That untagged -> attempt can fail too (for example with a single-use auth key), in which -> case the error is logged and the device stays logged out. Configure `tagOwners` in your tailnet's ACL before setting -> `tags` here. Each tag is `tag:` or a bare ``, where the name -> starts with a letter and holds only letters, digits and `-`. If any tag is -> invalid, the whole list is logged and ignored and the device registers -> untagged. +The embedded Tailscale node forwards every advertised TCP and UDP port to this +GameServer's backing Service. It does not provide a subnet route or expose +unadvertised ports. `TunnelReady` requires an online device and active forwarding +listeners. Tailnet grants still govern each player's access; readiness is not a +test of an individual grant or of the sleeping game's application health. + +Requested tags require a matching `tagOwners` policy. A bare tag name receives +the `tag:` prefix. Invalid or denied tags keep the tunnel not ready; it never +silently falls back to an untagged identity. + +Container restarts retain identity in a Pod volume. Replacing the Pod, changing +its tags, or upgrading from the older tailscaled implementation re-registers the +device and requires a valid reusable auth key. Remove stale devices from the +tailnet admin console so the configured MagicDNS hostname remains available. +If Tailscale assigns a suffixed name because of a conflict, Gameplane keeps the +advertised endpoint not ready until the conflict is resolved. Enable MagicDNS. + +Upgrade the operator and tunnel images together: the forwarding contract now +carries each port's TCP/UDP protocol. See the +[disposable tailnet integration procedure](tailscale-forwarding-integration.md) +for traffic, access-policy and restart checks. The `credentialsSecretRef.name` is the name that the dashboard or API write after you provide the credential. @@ -503,3 +509,8 @@ changes. the same security practices as you would for a public-facing LoadBalancer (e.g., disable admin consoles, run security updates, restrict in-game permissions). + +See [relay readiness and registration identity](relay-health.md) for the frp +UID migration, replacement policy, health probes, retry reset and the pinned +Playit version's connection-status limitation. An assigned address alone does +not make a relay ready. diff --git a/go.work b/go.work index fa6b1bb44..7ccf88062 100644 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.26.0 +go 1.26.10 use ( ./agent diff --git a/go.work.sum b/go.work.sum index 2da1910fe..1a92b2fe8 100644 --- a/go.work.sum +++ b/go.work.sum @@ -148,6 +148,7 @@ github.com/cavaliercoder/badio v0.0.0-20160213150051-ce5280129e9e/go.mod h1:V284 github.com/cavaliercoder/go-rpm v0.0.0-20200122174316-8cb9fd9c31a8/go.mod h1:AZIh1CCnMrcVm6afFf96PBvE2MRpWFco91z8ObJtgDY= github.com/cavaliergopher/cpio v1.0.1/go.mod h1:pBdaqQjnvXxdS/6CvNDwIANIFSP0xRKI16PX4xejRQc= github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= +github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/chainguard-dev/clog v1.7.0/go.mod h1:4+WFhRMsGH79etYXY3plYdp+tCz/KCkU8fAr0HoaPvs= github.com/chainguard-dev/clog v1.8.0/go.mod h1:5MQOZi+Iu7fV7GcJG8ag8rCB5elEOpqRMKEASgnGVdo= @@ -218,6 +219,8 @@ github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJP github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= +github.com/evanphx/json-patch v5.9.11+incompatible h1:ixHHqfcGvxhWkniF1tWxBHA0yb4Z+d1UQi45df52xW8= +github.com/evanphx/json-patch v5.9.11+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= github.com/firefart/nonamedreturns v1.0.6/go.mod h1:R8NisJnSIpvPWheCq0mNRXJok6D8h7fagJTF8EMEwCo= github.com/flynn/go-docopt v0.0.0-20140912013429-f6dd2ebbb31e/go.mod h1:HyVoz1Mz5Co8TFO8EupIdlcpwShBmY98dkT2xeHkvEI= @@ -306,7 +309,6 @@ github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba/go.mod h1:sC github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e/go.mod h1:Vrn4B5oR9qRwM+f54koyeH3yzphlecwERs0el27Fr/s= github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= github.com/google/addlicense v1.1.1/go.mod h1:Sm/DHu7Jk+T5miFHHehdIjbi4M5+dJDRS3Cq0rncIxA= -github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/cel-go v0.26.0/go.mod h1:A9O8OU9rdvrK5MQyrqfIxo1a0u4g3sF8KB6PUIaryMM= github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= @@ -405,6 +407,8 @@ github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGq github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= @@ -721,6 +725,7 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= @@ -756,7 +761,6 @@ golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= -golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7 h1:EBZoQjiKKPaLbPrbpssUfuHtwM6KV/vb4U85g/cigFY= @@ -809,7 +813,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/go-jose/go-jose.v2 v2.6.3/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -honnef.co/go/tools v0.7.0/go.mod h1:pm29oPxeP3P82ISxZDgIYeOaf9ta6Pi0EWvCFoLG2vc= k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index 059470fd2..a39c85d1a 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -567,6 +567,10 @@ "defaultMessage": "These settings apply to the backing Service, not to tunnel traffic: {settings}.", "description": "Informational condition when hostname, source ranges or node ports are set on a server with a tunnel. settings is a comma-separated list of field names." }, + "condition.tunnelready.awaitingaddress": { + "defaultMessage": "Waiting for the provider to assign a tunnel endpoint; the relay connection is unconfirmed.", + "description": "TunnelReady condition before Playit assigns an endpoint." + }, "condition.tunnelready.deploymentnotready": { "defaultMessage": "Waiting for the tunnel deployment to become ready.", "description": "TunnelReady condition while the tunnel pod is not ready." @@ -587,6 +591,10 @@ "defaultMessage": "{provider, select, playit {The tunnel deployment is ready. Waiting for the playit endpoint assignment.} tailscale {The tunnel deployment is ready and authenticated.} other {The tunnel deployment is ready and all ports are mapped.}}", "description": "TunnelReady condition when the tunnel works." }, + "condition.tunnelready.relayconnectionunknown": { + "defaultMessage": "The relay connection cannot be confirmed. Check the provider's connection status and the tunnel deployment's readiness probe.", + "description": "TunnelReady condition when provider connection evidence is missing or unsupported." + }, "condition.tunnelready.tunnelcredentialrefused": { "defaultMessage": "The tunnel credentials secret ''{secret}'' is not owned by this GameServer. {remedy, select, defaultName {Add an ownerReference to this GameServer to the Secret, or delete the Secret and save the credential again from the dashboard.} other {Save the credential from the dashboard, or add an ownerReference to this GameServer to the Secret.}}", "description": "TunnelReady condition when the operator refuses to mount a credentials secret. secret is the secret name, remedy is defaultName or other." @@ -595,6 +603,10 @@ "defaultMessage": "Unknown tunnel provider ''{provider}''.", "description": "TunnelReady condition when the provider name is not supported." }, + "condition.tunnelready.unsupportedprotocol": { + "defaultMessage": "Tailscale forwards only advertised TCP and UDP ports. Remove unsupported protocols from the advertised ports.", + "description": "TunnelReady condition when a Tailscale template advertises a protocol other than TCP or UDP." + }, "condition.unquiesced.failed": { "defaultMessage": "The server could not be resumed after the backup. Its world may still have auto-save turned off.", "description": "Unquiesced condition when the unfreeze failed." diff --git a/operator/api/v1alpha1/gameserver_types.go b/operator/api/v1alpha1/gameserver_types.go index 654cba6ed..df2a65660 100644 --- a/operator/api/v1alpha1/gameserver_types.go +++ b/operator/api/v1alpha1/gameserver_types.go @@ -460,15 +460,12 @@ type TailscaleTunnelSpec struct { Hostname string `json:"hostname,omitempty"` // Tags are Tailscale ACL tags requested for the device when it - // registers, via `tailscale up --advertise-tags` (tailscaled's - // declarative config has no field for tags, and changing the tags of a - // registered node needs a fresh login). Each entry is "tag:" or a bare - // name, which gets the "tag:" prefix. The tailnet ACL must grant - // tagOwners for these tags to the auth key's owner; if the request is - // refused, the tunnel pod logs the error and the tunnel runs untagged. - // If any entry is invalid, the whole list is logged and ignored and the - // device registers untagged. See - // tunnel/specs.md. + // registers through embedded tsnet. Each entry is "tag:" or a + // bare name, which gets the "tag:" prefix. The tailnet policy must grant + // tagOwners to the auth key's owner. Invalid or refused tags keep the + // tunnel not ready; it never silently registers without requested tags. + // Changing tags replaces the Pod and requires a reusable auth key. + // See tunnel/specs.md for identity persistence and registration details. // +kubebuilder:validation:MaxItems=8 // +optional Tags []string `json:"tags,omitempty"` diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index 4be670ebc..b323bcd1d 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -1545,15 +1545,12 @@ spec: tags: description: |- Tags are Tailscale ACL tags requested for the device when it - registers, via `tailscale up --advertise-tags` (tailscaled's - declarative config has no field for tags, and changing the tags of a - registered node needs a fresh login). Each entry is "tag:" or a bare - name, which gets the "tag:" prefix. The tailnet ACL must grant - tagOwners for these tags to the auth key's owner; if the request is - refused, the tunnel pod logs the error and the tunnel runs untagged. - If any entry is invalid, the whole list is logged and ignored and the - device registers untagged. See - tunnel/specs.md. + registers through embedded tsnet. Each entry is "tag:" or a + bare name, which gets the "tag:" prefix. The tailnet policy must grant + tagOwners to the auth key's owner. Invalid or refused tags keep the + tunnel not ready; it never silently registers without requested tags. + Changing tags replaces the Pod and requires a reusable auth key. + See tunnel/specs.md for identity persistence and registration details. items: type: string maxItems: 8 diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index e644cf041..cf7502c17 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -121,8 +121,11 @@ var rows = []row{ {types: []string{"AddressAssignment"}, reason: "Assigned", english: "{assignedFrom, select, pool {Address {address} assigned from pool ''{pool}''.} other {Address {address} assigned.}}", description: "AddressAssignment condition once an address is assigned. assignedFrom is pool or default."}, // GameServer: tunnel and other + {types: []string{"TunnelReady"}, reason: "UnsupportedProtocol", english: "Tailscale forwards only advertised TCP and UDP ports. Remove unsupported protocols from the advertised ports.", description: "TunnelReady condition when a Tailscale template advertises a protocol other than TCP or UDP."}, {types: []string{"TunnelReady"}, reason: "InvalidConfig", english: "{provider, select, none {The tunnel is enabled but the server has no tunnel configuration.} other {The ''{provider}'' tunnel provider is selected but its configuration is missing.}}", description: "TunnelReady condition for an incomplete tunnel setup. provider is the selected provider, or none."}, {types: []string{"TunnelReady"}, reason: "DeploymentNotReady", english: "Waiting for the tunnel deployment to become ready.", description: "TunnelReady condition while the tunnel pod is not ready."}, + {types: []string{"TunnelReady"}, reason: "RelayConnectionUnknown", english: "The relay connection cannot be confirmed. Check the provider's connection status and the tunnel deployment's readiness probe.", description: "TunnelReady condition when provider connection evidence is missing or unsupported."}, + {types: []string{"TunnelReady"}, reason: "AwaitingAddress", english: "Waiting for the provider to assign a tunnel endpoint; the relay connection is unconfirmed.", description: "TunnelReady condition before Playit assigns an endpoint."}, {types: []string{"TunnelReady"}, reason: "PortNotMapped", english: "No frp remote port is configured for: {ports}.", description: "TunnelReady condition for frp when advertised ports have no remote port. ports is a comma-separated list of port names."}, {types: []string{"TunnelReady"}, reason: "NoCredentials", english: "No credentials secret was provided, so the ''{provider}'' tunnel cannot authenticate.", description: "TunnelReady condition when a tunnel provider needs credentials that are missing."}, {types: []string{"TunnelReady"}, reason: "Ready", english: "{provider, select, playit {The tunnel deployment is ready. Waiting for the playit endpoint assignment.} tailscale {The tunnel deployment is ready and authenticated.} other {The tunnel deployment is ready and all ports are mapped.}}", description: "TunnelReady condition when the tunnel works."}, diff --git a/operator/internal/controller/gameserver_relay_health_test.go b/operator/internal/controller/gameserver_relay_health_test.go new file mode 100644 index 000000000..2f3597791 --- /dev/null +++ b/operator/internal/controller/gameserver_relay_health_test.go @@ -0,0 +1,106 @@ +package controller + +import ( + "reflect" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestRelayDeploymentIdentityProbesAndResourceRemoval(t *testing.T) { + t.Parallel() + s := wipeScheme(t) + for _, identity := range []struct{ namespace, uid string }{{"one", "uid-one"}, {"two", "uid-two"}, {"one", "replacement-uid"}} { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "same", Namespace: identity.namespace, UID: types.UID(identity.uid)}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "frp", Frp: &gameplanev1alpha1.FrpTunnelSpec{ServerAddr: "relay.example"}}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + for _, resources := range []*corev1.ResourceRequirements{nil, {Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("90m")}, Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("128Mi")}}, {}, nil} { + r.TunnelResources = resources + if err := r.reconcileTunnel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + var dep appsv1.Deployment + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-tunnel"}, &dep); err != nil { + t.Fatal(err) + } + if dep.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType || dep.Spec.Strategy.RollingUpdate != nil { + t.Fatal("relay permits overlapping identities") + } + container := dep.Spec.Template.Spec.Containers[0] + if container.StartupProbe == nil || container.StartupProbe.HTTPGet == nil || container.StartupProbe.HTTPGet.Path != "/healthz" || !hasRelayReadinessProbe(&dep) || container.LivenessProbe != nil { + t.Fatal("provider outage must affect readiness without restarting the supervisor") + } + if !reflect.DeepEqual(container.Resources, auxiliaryResources(resources, "25m", "64Mi")) { + t.Fatal("resource defaults or override removal changed") + } + found := false + for _, env := range container.Env { + if env.Name == "GAMESERVER_UID" { + found = env.Value == identity.uid + } + } + if !found { + t.Fatal("UID identity missing or unstable") + } + } + } +} + +func TestRelayConditionsRequireCurrentProviderProbe(t *testing.T) { + t.Parallel() + s := wipeScheme(t) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "relay", Namespace: "games", UID: "uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Provider: "frp", Frp: &gameplanev1alpha1.FrpTunnelSpec{ServerAddr: "relay"}}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + if err := r.reconcileTunnel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + var dep appsv1.Deployment + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: "relay-tunnel"}, &dep); err != nil { + t.Fatal(err) + } + dep.Generation = 2 + dep.Status = appsv1.DeploymentStatus{ReadyReplicas: 1, UpdatedReplicas: 1, ObservedGeneration: 2} + assert := func(want metav1.ConditionStatus) { + t.Helper() + cond := meta.FindStatusCondition(computeTunnelConditions(gs, tunnelPlan{wantTunnel: true}, &dep), "TunnelReady") + if cond == nil || cond.Status != want { + t.Fatalf("condition=%+v want=%s", cond, want) + } + } + assert(metav1.ConditionTrue) + dep.Status.ObservedGeneration = 1 + assert(metav1.ConditionUnknown) + dep.Status.ObservedGeneration = 2 + dep.Status.ReadyReplicas = 0 + assert(metav1.ConditionFalse) + dep.Status.ReadyReplicas = 1 + dep.Spec.Template.Spec.Containers[0].ReadinessProbe = nil + assert(metav1.ConditionUnknown) +} + +func TestPlayitAssignmentDoesNotImplyRelayReady(t *testing.T) { + t.Parallel() + gs := &gameplanev1alpha1.GameServer{Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Provider: "playit", Playit: &gameplanev1alpha1.PlayitTunnelSpec{}, CredentialsSecretRef: &gameplanev1alpha1.SecretNameRef{Name: "relay-credentials"}}}}} + dep := tunnelReadyDeployment() + for _, endpoints := range [][]gameplanev1alpha1.GameServerEndpoint{nil, {{Name: "game", Host: "example.playit.gg", Port: 30000}}} { + gs.Status.TunnelEndpoints = endpoints + cond := meta.FindStatusCondition(computeTunnelConditions(gs, tunnelPlan{wantTunnel: true}, dep), "TunnelReady") + wantStatus, wantReason := metav1.ConditionUnknown, "RelayConnectionUnknown" + if len(endpoints) == 0 { + wantStatus, wantReason = metav1.ConditionFalse, "AwaitingAddress" + } + if cond == nil || cond.Status != wantStatus || cond.Reason != wantReason { + t.Fatalf("playit readiness=%+v, want %s/%s", cond, wantStatus, wantReason) + } + } +} diff --git a/operator/internal/controller/gameserver_status.go b/operator/internal/controller/gameserver_status.go index 54393de0b..19ecbe8d4 100644 --- a/operator/internal/controller/gameserver_status.go +++ b/operator/internal/controller/gameserver_status.go @@ -260,8 +260,19 @@ func (r *GameServerReconciler) reconcileStatus( // Connection card reads the address players actually use. if svcExists { gs.Status.Endpoints = endpointsFromService(&svc, addrPlan) + } else { + // A deleted/replacing Service must not preserve a stale tailnet endpoint + // when the forwarding deployment has since lost readiness. + var retained []gameplanev1alpha1.GameServerEndpoint + for _, endpoint := range gs.Status.Endpoints { + if endpoint.TunnelProvider != "tailscale" { + retained = append(retained, endpoint) + } + } + gs.Status.Endpoints = retained } - if tunnelPlan.wantTunnel && len(tunnelPlan.endpoints) > 0 { + if tunnelPlan.wantTunnel && len(tunnelPlan.endpoints) > 0 && + (gs.Spec.Networking.Tunnel.Provider != "tailscale" || meta.IsStatusConditionTrue(gs.Status.Conditions, "TunnelReady")) { gs.Status.Endpoints = append(tunnelPlan.endpoints, gs.Status.Endpoints...) } if phase == gameplanev1alpha1.GameServerPhaseRunning { @@ -1105,6 +1116,21 @@ func computeTunnelConditions( } // Tunnel Deployment not yet created or not ready. + if tunnel.Provider == "tailscale" && len(plan.unsupportedPorts) > 0 { + tunnelReady.Status = metav1.ConditionFalse + tunnelReady.Reason = "UnsupportedProtocol" + tunnelReady.Message = "Tailscale supports only TCP and UDP; unsupported advertised ports: " + strings.Join(plan.unsupportedPorts, ", ") + return upsertCondition(conds, tunnelReady) + } + if tunnel.Provider == "playit" && (tunnel.Playit == nil || tunnel.CredentialsSecretRef == nil || tunnel.CredentialsSecretRef.Name == "") { + tunnelReady.Status = metav1.ConditionFalse + tunnelReady.Reason = "NoCredentials" + tunnelReady.Message = "playit configuration and credentials are required before relay connection can be confirmed" + if tunnel.Playit == nil { + tunnelReady.Reason = "InvalidConfig" + } + return upsertCondition(conds, tunnelReady) + } if tunnelDep == nil { tunnelReady.Status = metav1.ConditionFalse tunnelReady.Reason = "DeploymentNotReady" @@ -1112,11 +1138,30 @@ func computeTunnelConditions( conds = upsertCondition(conds, tunnelReady) return conds } + if tunnel.Provider == "frp" || tunnel.Provider == "playit" { + if !hasRelayReadinessProbe(tunnelDep) || tunnelDep.Status.ObservedGeneration < tunnelDep.Generation || tunnelDep.Status.UpdatedReplicas == 0 { + tunnelReady.Status = metav1.ConditionUnknown + tunnelReady.Reason = "RelayConnectionUnknown" + tunnelReady.Message = "waiting for the current relay deployment and its connection readiness probe" + return upsertCondition(conds, tunnelReady) + } + if tunnel.Provider == "playit" { + tunnelReady.Status = metav1.ConditionUnknown + tunnelReady.Reason = "RelayConnectionUnknown" + tunnelReady.Message = "playit v1.0.10 IPC reports cached assignment but cannot confirm relay connection; check provider connectivity separately" + if len(gs.Status.TunnelEndpoints) == 0 { + tunnelReady.Status = metav1.ConditionFalse + tunnelReady.Reason = "AwaitingAddress" + tunnelReady.Message = "waiting for playit endpoint assignment; relay connection is unconfirmed" + } + return upsertCondition(conds, tunnelReady) + } + } if tunnelDep.Status.ReadyReplicas == 0 { tunnelReady.Status = metav1.ConditionFalse tunnelReady.Reason = "DeploymentNotReady" - tunnelReady.Message = "tunnel deployment has no ready replicas" + tunnelReady.Message = "tunnel deployment has no ready replicas; relay connection or forward registration is unconfirmed" conds = upsertCondition(conds, tunnelReady) return conds } @@ -1147,7 +1192,7 @@ func computeTunnelConditions( // All endpoints known: deployment ready and mappings complete. tunnelReady.Status = metav1.ConditionTrue tunnelReady.Reason = "Ready" - tunnelReady.Message = "tunnel deployment ready and all ports mapped" + tunnelReady.Message = "relay connection and all configured frp forward registrations confirmed" case "tailscale": if tunnel.Tailscale == nil { @@ -1167,37 +1212,23 @@ func computeTunnelConditions( return conds } - // Deployment is ready and credentials are configured. - tunnelReady.Status = metav1.ConditionTrue - tunnelReady.Reason = "Ready" - tunnelReady.Message = "tunnel deployment ready and authenticated" - - case "playit": - if tunnel.Playit == nil { - tunnelReady.Status = metav1.ConditionFalse - tunnelReady.Reason = "InvalidConfig" - tunnelReady.Message = "playit provider selected but no playit config" - conds = upsertCondition(conds, tunnelReady) - return conds + // A legacy image merely registering a device cannot satisfy readiness. + forwardingProbe := false + for _, container := range tunnelDep.Spec.Template.Spec.Containers { + if container.Name == "tunnel" && container.ReadinessProbe != nil && container.ReadinessProbe.HTTPGet != nil { + probe := container.ReadinessProbe.HTTPGet + forwardingProbe = probe.Path == "/readyz" && probe.Port.IntValue() == 8091 + } } - - // Check if credentials are provided (playit requires authentication). - if tunnel.CredentialsSecretRef == nil { + if !forwardingProbe || tunnelDep.Status.ObservedGeneration < tunnelDep.Generation || tunnelDep.Status.UpdatedReplicas == 0 { tunnelReady.Status = metav1.ConditionFalse - tunnelReady.Reason = "NoCredentials" - tunnelReady.Message = "no credentials secret provided; playit requires authentication" - conds = upsertCondition(conds, tunnelReady) - return conds + tunnelReady.Reason = "DeploymentNotReady" + tunnelReady.Message = "waiting for the Tailscale forwarding deployment and its readiness probe" + return upsertCondition(conds, tunnelReady) } - - // Deployment is ready; playit endpoint will arrive asynchronously. - // Report AwaitingAddress while the tunnel pod hasn't yet reported an address. - // This is optimistic: the condition flips based on whether the tunnel pod - // has written to status.tunnelEndpoint. The agent or operator reconcile must - // populate it separately. tunnelReady.Status = metav1.ConditionTrue tunnelReady.Reason = "Ready" - tunnelReady.Message = "tunnel deployment ready; waiting for playit endpoint assignment" + tunnelReady.Message = "tailnet online and all advertised TCP/UDP forwarding listeners active; client access remains subject to tailnet policy" default: tunnelReady.Status = metav1.ConditionFalse @@ -1209,6 +1240,16 @@ func computeTunnelConditions( return conds } +func hasRelayReadinessProbe(dep *appsv1.Deployment) bool { + for _, container := range dep.Spec.Template.Spec.Containers { + if container.Name == "tunnel" && container.ReadinessProbe != nil && container.ReadinessProbe.HTTPGet != nil { + probe := container.ReadinessProbe.HTTPGet + return probe.Path == "/readyz" && probe.Port.IntValue() == 8091 + } + } + return false +} + // removeCondition removes a condition by type from the list, or returns the list unchanged. func removeCondition(conds []metav1.Condition, condType string) []metav1.Condition { out := make([]metav1.Condition, 0, len(conds)) diff --git a/operator/internal/controller/gameserver_tailscale_test.go b/operator/internal/controller/gameserver_tailscale_test.go new file mode 100644 index 000000000..1653fccf2 --- /dev/null +++ b/operator/internal/controller/gameserver_tailscale_test.go @@ -0,0 +1,48 @@ +package controller + +import ( + "context" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestTailscaleAdvertisedProtocolsAndReadiness(t *testing.T) { + tmpl := &gameplanev1alpha1.GameTemplate{Spec: gameplanev1alpha1.GameTemplateSpec{Ports: []gameplanev1alpha1.GamePort{ + {Name: "game", ContainerPort: 25565, Advertise: true}, + {Name: "query", ContainerPort: 19132, Protocol: corev1.ProtocolUDP, Advertise: true}, + {Name: "rcon", ContainerPort: 25575, Protocol: corev1.ProtocolTCP}, + }}} + if got := buildTailscalePortsConfig(tmpl); got != "game:25565:tcp,query:19132:udp" { + t.Fatalf("advertised mapping: %s", got) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "tailscale", Tailscale: &gameplanev1alpha1.TailscaleTunnelSpec{}, CredentialsSecretRef: &gameplanev1alpha1.SecretNameRef{Name: "auth"}}}}} + r := &GameServerReconciler{} + plan := r.planTunnel(context.Background(), gs, tmpl) + dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Generation: 2}, Status: appsv1.DeploymentStatus{ReadyReplicas: 1, UpdatedReplicas: 1, ObservedGeneration: 2}} + assert := func(want metav1.ConditionStatus) { + t.Helper() + c := meta.FindStatusCondition(computeTunnelConditions(gs, plan, dep), "TunnelReady") + if c == nil || c.Status != want { + t.Fatalf("TunnelReady=%+v, want %s", c, want) + } + } + assert(metav1.ConditionFalse) // A registered legacy deployment has no forwarding probe. + dep.Spec.Template.Spec.Containers = []corev1.Container{{Name: "tunnel", ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt(8091)}}}}} + assert(metav1.ConditionTrue) + dep.Status.ObservedGeneration = 1 + assert(metav1.ConditionFalse) + dep.Status.ObservedGeneration = 2 + dep.Status.ReadyReplicas = 0 + assert(metav1.ConditionFalse) + dep.Status.ReadyReplicas = 1 + tmpl.Spec.Ports[0].Protocol = corev1.ProtocolSCTP + plan = r.planTunnel(context.Background(), gs, tmpl) + assert(metav1.ConditionFalse) +} diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index e5712d96b..03785c236 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -52,7 +52,8 @@ type tunnelPlan struct { endpoints []gameplanev1alpha1.GameServerEndpoint // noMapping is the set of advertised port names that have no frp remote-port mapping. // Used only by frp to report a helpful condition reason. - noMapping []string + noMapping []string + unsupportedPorts []string } // planTunnel is the single source of truth for the tunnel pod's existence @@ -74,6 +75,7 @@ func (r *GameServerReconciler) planTunnel( tunnel := gs.Spec.Networking.Tunnel endpoints := make([]gameplanev1alpha1.GameServerEndpoint, 0) var noMapping []string + var unsupportedPorts []string switch tunnel.Provider { case "frp": @@ -128,6 +130,10 @@ func (r *GameServerReconciler) planTunnel( if !p.Advertise { continue } + if p.Protocol != "" && p.Protocol != corev1.ProtocolTCP && p.Protocol != corev1.ProtocolUDP { + unsupportedPorts = append(unsupportedPorts, p.Name) + continue + } ep := gameplanev1alpha1.GameServerEndpoint{ Name: p.Name, @@ -152,9 +158,10 @@ func (r *GameServerReconciler) planTunnel( } return tunnelPlan{ - wantTunnel: true, - endpoints: endpoints, - noMapping: noMapping, + wantTunnel: true, + endpoints: endpoints, + noMapping: noMapping, + unsupportedPorts: unsupportedPorts, } } @@ -199,6 +206,11 @@ func (r *GameServerReconciler) reconcileTunnel( refusedSecret = "" replicas := int32(1) dep.Spec.Replicas = &replicas + // A provider identity owns one relay session. Recreate unregisters the + // old child before its replacement can claim the same names/ports. + if tunnel.Provider == "frp" || tunnel.Provider == "playit" || tunnel.Provider == "tailscale" { + dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType} + } dep.Spec.Selector = &metav1.LabelSelector{ MatchLabels: map[string]string{ tunnelLabel: tunnelValue, @@ -272,6 +284,7 @@ func (r *GameServerReconciler) reconcileTunnel( envVars = []corev1.EnvVar{ {Name: "GAMESERVER_NAME", Value: gs.Name}, {Name: "GAMESERVER_NAMESPACE", Value: gs.Namespace}, + {Name: "GAMESERVER_UID", Value: string(gs.UID)}, {Name: "TUNNEL_TYPE", Value: "frp"}, {Name: "FRP_SERVER_ADDR", Value: tunnel.Frp.ServerAddr}, {Name: "FRP_SERVER_PORT", Value: fmt.Sprintf("%d", serverPort)}, @@ -388,14 +401,21 @@ func (r *GameServerReconciler) reconcileTunnel( } } + var readiness *corev1.Probe + if tunnel.Provider == "tailscale" { + volumes = append(volumes, corev1.Volume{Name: "tailscale-state", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}) + volumeMounts = append(volumeMounts, corev1.VolumeMount{Name: "tailscale-state", MountPath: "/var/lib/gameplane-tailscale"}) + readiness = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt(8091)}}, PeriodSeconds: 5, TimeoutSeconds: 2, FailureThreshold: 1} + } dep.Spec.Template.Spec.Volumes = volumes dep.Spec.Template.Spec.Containers = []corev1.Container{{ - Name: "tunnel", - Image: image, - Resources: auxiliaryResources(r.TunnelResources, "25m", "64Mi"), - Env: envVars, - Ports: ports, - VolumeMounts: volumeMounts, + Name: "tunnel", + Image: image, + Resources: auxiliaryResources(r.TunnelResources, "25m", "64Mi"), + Env: envVars, + Ports: ports, + VolumeMounts: volumeMounts, + ReadinessProbe: readiness, SecurityContext: &corev1.SecurityContext{ RunAsNonRoot: &nonRoot, RunAsUser: &uid, @@ -403,6 +423,11 @@ func (r *GameServerReconciler) reconcileTunnel( Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, }, }} + if tunnel.Provider == "frp" || tunnel.Provider == "playit" { + container := &dep.Spec.Template.Spec.Containers[0] + container.StartupProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(8091)}}, PeriodSeconds: 2, TimeoutSeconds: 1, FailureThreshold: 30} + container.ReadinessProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt32(8091)}}, PeriodSeconds: 2, TimeoutSeconds: 1, FailureThreshold: 1} + } return controllerutil.SetControllerReference(gs, dep, r.Scheme) }) @@ -695,14 +720,18 @@ func buildFrpRemotePortsConfig(frp *gameplanev1alpha1.FrpTunnelSpec, tmpl *gamep } // buildTailscalePortsConfig constructs the BACKING_SERVICE_PORTS env var for Tailscale. -// Format: "port_name:port,..." e.g. "java:25565,bedrock:19133" +// Format: "port_name:port:protocol,..." e.g. "java:25565:tcp,bedrock:19133:udp". func buildTailscalePortsConfig(tmpl *gameplanev1alpha1.GameTemplate) string { var entries []string for _, p := range tmpl.Spec.Ports { if !p.Advertise { continue } - entries = append(entries, fmt.Sprintf("%s:%d", p.Name, p.ContainerPort)) + protocol := strings.ToLower(string(p.Protocol)) + if protocol == "" { + protocol = "tcp" + } + entries = append(entries, fmt.Sprintf("%s:%d:%s", p.Name, p.ContainerPort, protocol)) } if len(entries) == 0 { return "" diff --git a/operator/internal/controller/gameserver_tunnel_envtest_test.go b/operator/internal/controller/gameserver_tunnel_envtest_test.go index d86582fb9..2447b2ee1 100644 --- a/operator/internal/controller/gameserver_tunnel_envtest_test.go +++ b/operator/internal/controller/gameserver_tunnel_envtest_test.go @@ -109,6 +109,32 @@ func TestTunnelReconciliation_CredentialKeyProjection(t *testing.T) { } // Verify the Secret volume has Items projection with only "authKey" + if dep.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType || dep.Spec.Strategy.RollingUpdate != nil { + t.Fatalf("Tailscale must recreate its single identity: %+v", dep.Spec.Strategy) + } + container := dep.Spec.Template.Spec.Containers[0] + if container.ReadinessProbe == nil || container.ReadinessProbe.HTTPGet == nil || container.ReadinessProbe.HTTPGet.Path != "/readyz" || container.ReadinessProbe.HTTPGet.Port.IntValue() != 8091 { + t.Fatalf("Tailscale lacks forwarding readiness probe: %+v", container.ReadinessProbe) + } + stateVolume, stateMount, protocolEnv := false, false, false + for _, volume := range dep.Spec.Template.Spec.Volumes { + if volume.Name == "tailscale-state" && volume.EmptyDir != nil { + stateVolume = true + } + } + for _, mount := range container.VolumeMounts { + if mount.Name == "tailscale-state" && mount.MountPath == "/var/lib/gameplane-tailscale" { + stateMount = true + } + } + for _, env := range container.Env { + if env.Name == "BACKING_SERVICE_PORTS" && env.Value == "game:25565:tcp" { + protocolEnv = true + } + } + if !stateVolume || !stateMount || !protocolEnv { + t.Fatal("Tailscale lost persistent pod state or advertised protocol contract") + } if len(dep.Spec.Template.Spec.Volumes) == 0 { t.Fatal("no volumes in tunnel pod template") } diff --git a/operator/specs.md b/operator/specs.md index 2db4f71cf..b640c8c82 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -10,6 +10,8 @@ The operator is a Kubernetes controller-runtime-based process that reconciles Ga ## Responsibilities +- **Relay health:** frp identities use GameServer UID without changing advertised port names. frp/Playit Deployments use one replica, Recreate, a wrapper startup probe and a provider readiness probe on TCP 8091. `TunnelReady` requires the current observed generation and updated/ready replicas with the new probe; legacy unprobed Pods cannot report success. Playit v1.0.10's cached IPC assignment cannot confirm a relay connection, so it reports unknown/not ready rather than optimistic success. Preserve `TunnelResources` defaults and override removal. See `docs/relay-health.md` for health bounds, migration and the provider limitation. + - **GameServer lifecycle:** reconcile GameServer CRs into StatefulSets, Services, PVCs, and pod-injected agent sidecars; manage pod affinity, RCON/console integration, file I/O, and graceful stop sequences. - **Module installation:** pull OCI module bundles from remote or local sources, verify signatures (cosign), and materialize GameTemplate CRs; refresh source indexes periodically. Translation files (`locales/.json`) are carried by every module source (OCI layers, directory, git, HTTP archive, upload, `gp-module package` tarball) and validated against the ids derived from the module (`i18n/modlocale`); valid entries are written to the `