From 1871b88ab3d007b2190837e9e14cb826346d0909 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 11:25:43 -0400 Subject: [PATCH 1/6] feat(operator): reserve auxiliary resources and expose management placement Signed-off-by: Baylem --- .github/workflows/ci.yaml | 12 +- charts/gameplane/templates/_helpers.tpl | 23 ++ charts/gameplane/templates/api.yaml | 1 + .../templates/audit-syslog-bridge.yaml | 1 + .../gameplane/templates/crd-apply-hook.yaml | 1 + charts/gameplane/templates/gateway.yaml | 1 + charts/gameplane/templates/mcp-server.yaml | 1 + charts/gameplane/templates/operator.yaml | 8 + .../templates/poddisruptionbudgets.yaml | 47 ++++ .../templates/telemetry-receiver.yaml | 1 + charts/gameplane/templates/validate.yaml | 3 + charts/gameplane/templates/web.yaml | 1 + .../gameplane/values-production.example.yaml | 36 +++ charts/gameplane/values.yaml | 65 ++++- charts/tests/check_workloads.py | 78 ++++++ deploy/kind/e2e.sh | 13 +- docs/production-placement.md | 140 ++++++++++ docs/release-notes.md | 21 ++ operator/cmd/main.go | 33 +++ .../controller/auxiliary_resources.go | 65 +++++ .../controller/auxiliary_resources_test.go | 105 ++++++++ .../internal/controller/backup_controller.go | 19 +- operator/internal/controller/backup_forget.go | 26 +- .../controller/gameserver_controller.go | 26 +- .../controller/gameserver_sentinel.go | 5 +- .../internal/controller/gameserver_tunnel.go | 1 + .../internal/controller/restore_controller.go | 7 +- operator/specs.md | 14 + test/e2e/buckets.sh | 8 +- test/e2e/management_placement_e2e_test.go | 251 ++++++++++++++++++ test/e2e/specs.md | 11 + 31 files changed, 979 insertions(+), 45 deletions(-) create mode 100644 charts/gameplane/templates/poddisruptionbudgets.yaml create mode 100644 charts/gameplane/values-production.example.yaml create mode 100644 charts/tests/check_workloads.py create mode 100644 docs/production-placement.md create mode 100644 docs/release-notes.md create mode 100644 operator/internal/controller/auxiliary_resources.go create mode 100644 operator/internal/controller/auxiliary_resources_test.go create mode 100644 test/e2e/management_placement_e2e_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1c9aecdd..a902708f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1115,6 +1115,8 @@ jobs: run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders run: python3 charts/tests/check_standalone.py + - name: auxiliary resources and management placement chart renders + run: python3 charts/tests/check_workloads.py - name: sanity-check rendered manifests run: | grep -q 'kind: Deployment' /tmp/rendered.yaml @@ -1664,13 +1666,17 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement] arch: [amd64] include: - bucket: operator parallel: 8 test_timeout: 20m job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: management-placement + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: api-auth parallel: 4 test_timeout: 15m @@ -1718,7 +1724,9 @@ jobs: with: arch: ${{ matrix.arch }} - name: bring up cluster + helm install - run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e + env: + GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }} + run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) run: | RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')" diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index bec3e1ffb..8085e8ac0 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -135,3 +135,26 @@ true true {{- end -}} {{- end -}} + +{{- /* Shared PodSpec placement fields; absent keys preserve older reuse-values. */ -}} +{{- define "gameplane.placement" -}} +{{- range $key := list "nodeSelector" "tolerations" "affinity" "topologySpreadConstraints" "priorityClassName" -}} +{{- with index $ $key }} +{{ $key }}: {{ toYaml . | nindent 2 }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- /* Complete auxiliary overrides; empty objects must not become defaults. */ -}} +{{- define "gameplane.auxiliaryResourceArgs" -}} +{{- range $name := list "agent" "sentinel" "tunnel" "backup" -}} +{{- $key := printf "%sResources" $name -}} +{{- $value := index $ $key -}} +{{- if and (hasKey $ $key) (not (kindIs "invalid" $value)) -}} +{{- if not (kindIs "map" $value) -}} +{{- fail (printf "operator.%s must be a resources object or null" $key) -}} +{{- end }} +- {{ printf "--%s-resources=%s" $name (toJson $value) | quote }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/gameplane/templates/api.yaml b/charts/gameplane/templates/api.yaml index a562d7c7b..f04fa99ec 100644 --- a/charts/gameplane/templates/api.yaml +++ b/charts/gameplane/templates/api.yaml @@ -258,6 +258,7 @@ spec: checksum/mtls: {{ include "gameplane.mtlsChecksum" . }} {{- end }} spec: + {{- include "gameplane.placement" .Values.api | nindent 6 }} serviceAccountName: gameplane-api {{- if .Values.api.standalone }} automountServiceAccountToken: false diff --git a/charts/gameplane/templates/audit-syslog-bridge.yaml b/charts/gameplane/templates/audit-syslog-bridge.yaml index 11bb8b279..c030d0150 100644 --- a/charts/gameplane/templates/audit-syslog-bridge.yaml +++ b/charts/gameplane/templates/audit-syslog-bridge.yaml @@ -27,6 +27,7 @@ spec: app.kubernetes.io/name: gameplane-audit-syslog-bridge {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" $b | nindent 6 }} securityContext: runAsNonRoot: true runAsUser: 65532 diff --git a/charts/gameplane/templates/crd-apply-hook.yaml b/charts/gameplane/templates/crd-apply-hook.yaml index b767b4e28..4ad3925cb 100644 --- a/charts/gameplane/templates/crd-apply-hook.yaml +++ b/charts/gameplane/templates/crd-apply-hook.yaml @@ -194,6 +194,7 @@ spec: app.kubernetes.io/name: gameplane-crd-apply {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" .Values.crds.autoApply | nindent 6 }} serviceAccountName: {{ .Release.Name }}-crd-apply restartPolicy: Never securityContext: diff --git a/charts/gameplane/templates/gateway.yaml b/charts/gameplane/templates/gateway.yaml index 82c46613e..aeab6bafc 100644 --- a/charts/gameplane/templates/gateway.yaml +++ b/charts/gameplane/templates/gateway.yaml @@ -113,6 +113,7 @@ spec: app.kubernetes.io/name: gameplane-gateway {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" $gateway | nindent 6 }} serviceAccountName: gameplane-gateway securityContext: runAsNonRoot: true diff --git a/charts/gameplane/templates/mcp-server.yaml b/charts/gameplane/templates/mcp-server.yaml index 3858878a3..40051e993 100644 --- a/charts/gameplane/templates/mcp-server.yaml +++ b/charts/gameplane/templates/mcp-server.yaml @@ -66,6 +66,7 @@ spec: app.kubernetes.io/name: gameplane-mcp-server {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" .Values.mcpServer | nindent 6 }} serviceAccountName: gameplane-mcp-server securityContext: runAsNonRoot: true diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 772e0b10e..444a60936 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -239,6 +239,12 @@ metadata: labels: {{- include "gameplane.labels" . | nindent 4 }} spec: replicas: {{ .Values.operator.replicas }} + {{- if not .Values.operator.leaderElect }} + # Without leader election, prevent a rolling update from overlapping writers. + strategy: + type: Recreate + rollingUpdate: null + {{- end }} selector: matchLabels: app.kubernetes.io/name: gameplane-operator @@ -250,6 +256,7 @@ spec: annotations: checksum/mtls: {{ include "gameplane.mtlsChecksum" . }} spec: + {{- include "gameplane.placement" .Values.operator | nindent 6 }} serviceAccountName: gameplane-operator securityContext: runAsNonRoot: true @@ -261,6 +268,7 @@ spec: image: {{ include "gameplane.operatorImage" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: + {{- include "gameplane.auxiliaryResourceArgs" .Values.operator | nindent 12 }} {{- if .Values.operator.leaderElect }} - --leader-elect {{- end }} diff --git a/charts/gameplane/templates/poddisruptionbudgets.yaml b/charts/gameplane/templates/poddisruptionbudgets.yaml new file mode 100644 index 000000000..35667e143 --- /dev/null +++ b/charts/gameplane/templates/poddisruptionbudgets.yaml @@ -0,0 +1,47 @@ +{{- $apiEnabled := eq (include "gameplane.apiEnabled" .) "true" -}} +{{- $bridge := .Values.api.audit.webhook.syslogBridge -}} +{{- $receiver := .Values.api.telemetry.receiver -}} +{{- $gateway := .Values.gateway | default dict -}} +{{- $components := list + (dict "name" "operator" "config" .Values.operator "enabled" (eq (include "gameplane.operatorEnabled" .) "true")) + (dict "name" "api" "config" .Values.api "enabled" $apiEnabled) + (dict "name" "web" "config" .Values.web "enabled" (and $apiEnabled .Values.web.enabled)) + (dict "name" "mcp-server" "config" .Values.mcpServer "enabled" .Values.mcpServer.enabled) + (dict "name" "gateway" "config" $gateway "enabled" $gateway.enabled) + (dict "name" "audit-syslog-bridge" "config" $bridge "enabled" (and $apiEnabled $bridge.enabled (not .Values.api.audit.webhook.url))) + (dict "name" "telemetry-receiver" "config" $receiver "enabled" (and $apiEnabled $receiver.enabled (not .Values.api.telemetry.endpoint))) -}} +{{- range $components -}} +{{- $pdb := .config.pdb | default dict -}} +{{- if and .enabled $pdb.enabled -}} +{{- $minimum := not (kindIs "invalid" $pdb.minAvailable) -}} +{{- $maximum := not (kindIs "invalid" $pdb.maxUnavailable) -}} +{{- if eq .name "api" -}} +{{- if or $minimum (and $maximum (ne (toString $pdb.maxUnavailable) "1")) -}} +{{- fail "api.pdb must allow the singleton API to be evicted: use maxUnavailable: 1 without minAvailable" -}} +{{- end -}} +{{- else if and $minimum $maximum -}} +{{- fail (printf "%s.pdb must set only one of minAvailable or maxUnavailable" .name) -}} +{{- end }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: gameplane-{{ .name }} + namespace: {{ $.Release.Namespace }} + labels: {{- include "gameplane.labels" $ | nindent 4 }} +spec: + {{- if eq .name "api" }} + maxUnavailable: 1 + {{- else if $minimum }} + minAvailable: {{ toYaml $pdb.minAvailable }} + {{- else if $maximum }} + maxUnavailable: {{ toYaml $pdb.maxUnavailable }} + {{- else }} + maxUnavailable: 1 + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: gameplane-{{ .name }} + app.kubernetes.io/instance: {{ $.Release.Name }} +{{- end -}} +{{- end -}} diff --git a/charts/gameplane/templates/telemetry-receiver.yaml b/charts/gameplane/templates/telemetry-receiver.yaml index 4e8255fbc..18a084d32 100644 --- a/charts/gameplane/templates/telemetry-receiver.yaml +++ b/charts/gameplane/templates/telemetry-receiver.yaml @@ -88,6 +88,7 @@ spec: app.kubernetes.io/name: gameplane-telemetry-receiver {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" $r | nindent 6 }} securityContext: runAsNonRoot: true runAsUser: 65532 diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index 79b5e17b6..c6df65ede 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,3 +1,6 @@ +{{- if and (eq (include "gameplane.operatorEnabled" .) "true") (gt (int .Values.operator.replicas) 1) (not .Values.operator.leaderElect) }} +{{- fail "operator.leaderElect must be enabled with more than one replica" }} +{{- end }} {{- if .Values.api.standalone }} {{- if ne (include "gameplane.apiEnabled" .) "true" }} {{- fail "api.standalone requires api.enabled=true" }} diff --git a/charts/gameplane/templates/web.yaml b/charts/gameplane/templates/web.yaml index 4429fc5e4..736fd7136 100644 --- a/charts/gameplane/templates/web.yaml +++ b/charts/gameplane/templates/web.yaml @@ -25,6 +25,7 @@ spec: app.kubernetes.io/name: gameplane-web {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" .Values.web | nindent 6 }} automountServiceAccountToken: false securityContext: runAsNonRoot: true diff --git a/charts/gameplane/values-production.example.yaml b/charts/gameplane/values-production.example.yaml new file mode 100644 index 000000000..229448ac3 --- /dev/null +++ b/charts/gameplane/values-production.example.yaml @@ -0,0 +1,36 @@ +# Example policy, not a capacity recommendation. Label/taint worker nodes before +# applying this profile; see docs/production-placement.md. Never remove or +# tolerate control-plane taints just to satisfy these selectors. +operator: + replicas: 2 + leaderElect: true + nodeSelector: &managementNodes { gameplane.local/pool: management } + tolerations: &managementTolerations + - { key: gameplane.local/pool, operator: Equal, value: management, effect: NoSchedule } + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: { app.kubernetes.io/name: gameplane-operator } + pdb: { enabled: true, maxUnavailable: 1 } +api: + replicas: 1 + nodeSelector: *managementNodes + tolerations: *managementTolerations + pdb: { enabled: true, maxUnavailable: 1 } +web: + replicas: 2 + nodeSelector: *managementNodes + tolerations: *managementTolerations + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: { app.kubernetes.io/name: gameplane-web } + pdb: { enabled: true, maxUnavailable: 1 } +crds: + autoApply: + nodeSelector: *managementNodes + tolerations: *managementTolerations diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 66d95af0b..9346f8e00 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -39,6 +39,12 @@ image: crds: autoApply: enabled: true + # The upgrade hook must be schedulable before new Deployments roll out. + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" # kubectl image the apply Job runs. Any image with kubectl on PATH works; # retag to a private mirror for air-gapped installs (needed at upgrade # time, and also on an install over CRDs an earlier, uninstalled release @@ -53,17 +59,28 @@ operator: # Disable the operator and bundled native CRDs for a standalone central panel. enabled: true replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } # optional minAvailable OR maxUnavailable; defaults to maxUnavailable: 1 logLevel: info # zap log level: debug|info|error (warn not supported by zap's --zap-log-level) agentLogLevel: "" # injected into every agent sidecar as GAMEPLANE_LOG_LEVEL (debug|info|warn|error); empty injects nothing, avoiding StatefulSet rollouts on upgrade - # Leader election guards against two operators reconciling at once. With a - # single replica it's unnecessary, and on a resource-starved node a missed - # lease renewal makes controller-runtime exit (the operator then crash-loops - # and stops reconciling). Leave on for HA installs; turn off for single-replica - # or constrained clusters (the e2e install sets it false). + # Required with multiple replicas and recommended during rolling upgrades. + # Disabling it with one replica selects Recreate to prevent overlapping writers. leaderElect: true resources: requests: { cpu: 100m, memory: 64Mi } limits: { cpu: 500m, memory: 256Mi } + # Complete auxiliary ResourceRequirements overrides. Null/missing uses binary + # defaults, including on --reuse-values upgrades. {} (or requests: {}, limits: + # {}) clears defaults. No new CPU/memory limits are set by default. + agentResources: null # requests 25m CPU / 64Mi memory; also config-init + sentinelResources: null # requests 25m CPU / 32Mi memory + tunnelResources: null # requests 25m CPU / 64Mi memory (every provider) + backupResources: null # backup/init/restore requests 100m CPU / 128Mi memory; + # cleanup retains 50m / 128Mi, memory limit 512Mi agentImage: "" # defaults to {image.registry}/agent:{image.tag} sentinelImage: "" # defaults to {image.registry}/sentinel:{image.tag} # Which load-balancer address manager runs in this cluster, so the operator @@ -133,6 +150,14 @@ api: # Standalone mode also skips the CRD apply hook and its RBAC. standalone: false replicas: 1 # Keep at 1. SQLite is single-writer on a RWO volume; with db.driver=postgres the user-management lock and audit hash chain are still per-process + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + # The API is a singleton: eviction causes downtime. A blocking budget would + # hang voluntary node drains, so only maxUnavailable: 1 is supported here. + pdb: { enabled: false, maxUnavailable: 1 } logLevel: info # API log verbosity (debug|info|warn|error), passed as GAMEPLANE_LOG_LEVEL # Port of the API's Prometheus metrics listener. It is separate from the # public API port (the one the ingress and the web front end route to), so @@ -241,6 +266,12 @@ api: facility: local0 # RFC 5424 facility: kern/user/daemon/auth/.../local0..local7 severity: info # emerg/alert/crit/err/warning/notice/info/debug replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: {} # Native S3-compatible audit sink. When endpoint and bucket are set, the API # batches audit events as NDJSON and uploads them to S3 (or MinIO, Backblaze, @@ -314,6 +345,12 @@ api: # Must stay 1 while persistence is enabled: the SQLite file on the PVC # has a single writer, so the template fails the render otherwise. replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: {} # Aggregate storage (SQLite on a PVC mounted at /data). With # persistence disabled an emptyDir is used instead and every @@ -422,6 +459,12 @@ api: gateway: enabled: false replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } # Must match the Cluster resource name in the central Gameplane installation. clusterID: "" # Exact URI SAN trusted in the central API client certificate. @@ -465,6 +508,12 @@ gateway: web: enabled: true replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: requests: { cpu: 10m, memory: 16Mi } limits: { cpu: 200m, memory: 128Mi } @@ -602,6 +651,12 @@ clusterOps: mcpServer: enabled: false replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: requests: { cpu: 50m, memory: 32Mi } limits: { cpu: 200m, memory: 128Mi } diff --git a/charts/tests/check_workloads.py b/charts/tests/check_workloads.py new file mode 100644 index 000000000..33fd6f89f --- /dev/null +++ b/charts/tests/check_workloads.py @@ -0,0 +1,78 @@ +"""CI-only resource override, management placement and disruption checks.""" +import json +from pathlib import Path +import shutil +import tempfile + +import yaml + +from check_gateway import CHART, one, render, settings + + +def main(): + defaults = render() + assert not any(obj["kind"] == "PodDisruptionBudget" for obj in defaults) + args = one(defaults, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert not any(arg.startswith("--tunnel-resources=") for arg in args) + placement = { + "nodeSelector": {"gameplane.local/pool": "management"}, + "tolerations": [{"key": "gameplane.local/pool", "operator": "Equal", "value": "management", "effect": "NoSchedule"}], + "affinity": {"podAntiAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [{"weight": 100, "podAffinityTerm": {"topologyKey": "kubernetes.io/hostname", "labelSelector": {"matchLabels": {"app.kubernetes.io/instance": "gameplane"}}}}]}}, + "topologySpreadConstraints": [{"maxSkew": 1, "topologyKey": "kubernetes.io/hostname", "whenUnsatisfiable": "DoNotSchedule", "labelSelector": {"matchLabels": {"app.kubernetes.io/instance": "gameplane"}}}], + "priorityClassName": "management", + } + component = dict(placement, pdb={"enabled": True, "maxUnavailable": 1}) + gateway = dict(settings(), **component) + values = { + "operator": dict(component, replicas=2, tunnelResources={"requests": {}, "limits": {}}, agentResources={"requests": {"cpu": "80m"}}), + "api": dict(component, audit={"webhook": {"syslogBridge": dict(component, enabled=True, syslog={"addr": "logs:514"})}}, telemetry={"receiver": dict(component, enabled=True)}), + "web": component, "mcpServer": dict(component, enabled=True), "gateway": gateway, + "crds": {"autoApply": placement}, + } + production = render(values, upgrade=True) + for name in ["operator", "api", "web", "mcp-server", "gateway", "audit-syslog-bridge", "telemetry-receiver"]: + deployment = one(production, "Deployment", "gameplane-" + name) + spec = deployment["spec"]["template"]["spec"] + for key, expected in placement.items(): + assert spec[key] == expected, (name, key) + pdb = one(production, "PodDisruptionBudget", "gameplane-" + name)["spec"] + assert pdb["maxUnavailable"] == 1 and "minAvailable" not in pdb + assert pdb["selector"]["matchLabels"].items() <= deployment["spec"]["template"]["metadata"]["labels"].items() + hook = one(production, "Job", "gameplane-crd-apply")["spec"]["template"]["spec"] + for key, expected in placement.items(): + assert hook[key] == expected + args = one(production, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + overrides = {arg.split("=", 1)[0]: json.loads(arg.split("=", 1)[1]) for arg in args if arg.startswith(("--agent-resources=", "--tunnel-resources="))} + assert overrides["--tunnel-resources"] == {"requests": {}, "limits": {}} + assert overrides["--agent-resources"] == {"requests": {"cpu": "80m"}} + render({"api": {"pdb": {"enabled": True, "minAvailable": 1}}}, failure="api.pdb must allow the singleton API to be evicted") + render({"api": {"pdb": {"enabled": True, "maxUnavailable": 0}}}, failure="api.pdb must allow the singleton API to be evicted") + render({"operator": {"replicas": 2, "leaderElect": False}}, failure="operator.leaderElect must be enabled") + single = render({"operator": {"leaderElect": False}}) + assert one(single, "Deployment", "gameplane-operator")["spec"]["strategy"] == {"type": "Recreate", "rollingUpdate": None} + removed = render({"operator": {"agentResources": None, "tunnelResources": {}}}) + args = one(removed, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert not any(arg.startswith("--agent-resources=") for arg in args) + assert "--tunnel-resources={}" in args + disabled = render({"api": {"enabled": False, "pdb": {"enabled": True}}, "web": {"pdb": {"enabled": True}}}) + assert not any(obj["kind"] == "PodDisruptionBudget" for obj in disabled) + profile = yaml.safe_load((CHART / "values-production.example.yaml").read_text()) + example = render(profile, upgrade=True) + assert one(example, "Deployment", "gameplane-operator")["spec"]["replicas"] == 2 + assert one(example, "PodDisruptionBudget", "gameplane-api")["spec"]["maxUnavailable"] == 1 + # Helm --reuse-values may not contain new keys. Omission must retain binary + # resource defaults and empty placement without indexing a missing map. + with tempfile.TemporaryDirectory() as directory: + old_chart = Path(directory) / "chart" + shutil.copytree(CHART, old_chart) + old_values = yaml.safe_load((old_chart / "values.yaml").read_text()) + for key in ["agentResources", "sentinelResources", "tunnelResources", "backupResources", "pdb", *placement]: + old_values["operator"].pop(key, None) + (old_chart / "values.yaml").write_text(yaml.safe_dump(old_values)) + old = render(chart=old_chart) + args = one(old, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert not any(arg.startswith("--tunnel-resources=") for arg in args) + + +if __name__ == "__main__": + main() diff --git a/deploy/kind/e2e.sh b/deploy/kind/e2e.sh index 3c2f7b12f..8f5df5211 100755 --- a/deploy/kind/e2e.sh +++ b/deploy/kind/e2e.sh @@ -6,7 +6,7 @@ # deploy/kind/e2e.sh down [cluster-name] # # Diffs from dev-up (deploy/kind/up.sh): -# - Single-node cluster (faster boot, sufficient for E2E coverage). +# - Single-node by default; management profile adds two scheduling workers. # - Skips ingress-nginx (the dashboard isn't exercised here). # - Loads pre-built gameplane-test/{operator,api,agent,sentinel,capture-sidecar,telemetry-receiver,fakeoidc}: # images (and gameplane-test/gameprobe: when present). @@ -28,9 +28,9 @@ CLUSTER="${2:-gameplane-e2e}" TAG="${3:-${GAMEPLANE_E2E_TAG:-e2e}}" PROFILE="${4:-default}" case "${PROFILE}" in -default | postgres) ;; +default | postgres | management) ;; *) - echo "unknown profile '${PROFILE}' (want default or postgres)" >&2 + echo "unknown profile '${PROFILE}' (want default, postgres or management)" >&2 exit 2 ;; esac @@ -188,12 +188,17 @@ up) echo "removing pre-existing cluster ${CLUSTER} for a clean slate" kind delete cluster --name "${CLUSTER}" || true fi - echo "creating kind cluster ${CLUSTER} (single-node)" + echo "creating kind cluster ${CLUSTER} (${PROFILE})" + extra_nodes="" + if [[ "${PROFILE}" == "management" ]]; then + extra_nodes=$' - role: worker\n - role: worker' + fi kind create cluster --name "${CLUSTER}" --wait 90s --config - </dev/null diff --git a/docs/production-placement.md b/docs/production-placement.md new file mode 100644 index 000000000..40bdb41b0 --- /dev/null +++ b/docs/production-placement.md @@ -0,0 +1,140 @@ +# Resource reservations and management placement + +New installs and upgrades reserve resources for supporting containers. These +requests are scheduling floors, not measured capacity targets: + +| Container | CPU request | Memory request | +| --- | --- | --- | +| Agent and config initializer | 25m | 64Mi | +| Wake sentinel | 25m | 32Mi | +| Tunnel, every provider | 25m | 64Mi | +| Restic backup, repository initializer and restore | 100m | 128Mi | + +These defaults add no CPU or memory limits. CPU limits can throttle bursts; +memory limits can terminate the process when its working set exceeds them. +Repository forget/prune retains its existing 50m/128Mi requests and 512Mi memory +limit unless overridden. Init containers receive requests too; Kubernetes +accounts for their peak requirements when scheduling the Pod. + +## Overrides and upgrades + +The new chart keys `operator.agentResources`, `operator.sentinelResources`, +`operator.tunnelResources` and `operator.backupResources` each accept a complete +Kubernetes ResourceRequirements object. Backup resources also apply to restore +and forget/prune workers; agent resources also apply to the config initializer. +The matching operator flags are `--agent-resources`, `--sentinel-resources`, +`--tunnel-resources`, and `--backup-resources`, with JSON values. + +Null or a missing chart key uses the binary defaults, including older +`helm upgrade --reuse-values` configurations. A supplied object replaces defaults +wholesale. Clear reservations explicitly: + +```yaml +operator: + agentResources: { requests: {}, limits: {} } + sentinelResources: { requests: {}, limits: {} } + tunnelResources: { requests: {}, limits: {} } + backupResources: { requests: {}, limits: {} } +``` + +An empty object `{}` has the same effect. Setting a key back to `null` restores +the defaults. No chart-side default map is merged into the supplied object. +For example `agentResources: {requests: {memory: 96Mi}}` reserves only memory. + +Helm still merges user-supplied maps. When replacing a previously populated +override with empty maps, export the release's values, edit that complete file, +and upgrade with `--reset-values -f `. An empty nested map +combined with `--reuse-values` can retain the release's old resource entries. +Keep all unrelated install values in the edited file; resetting to chart defaults +alone is not a resource-only change. + +Existing `GameTemplate.spec.agent.resources` takes precedence when it declares +requests or limits, replacing the install agent object wholesale. An image-only +agent override keeps install defaults. Kubernetes may omit empty maps in a +template, so use explicit zero CPU/memory requests for a per-template zero +reservation, or the install-level empty object to clear all defaults. Game +container resources continue to use the existing template defaults and complete +`GameServer.spec.resources` override. Game placement fields are unchanged. + +Changing resources changes generated Pod templates. Existing game StatefulSets, +sentinels and tunnels roll according to their normal reconciliation policy; +new Restic Jobs use the new configuration. Schedule this upgrade when game pod +restarts are acceptable. Insufficient capacity can leave replacement pods or +Jobs Pending. Check requested resources and concurrent backups before upgrading. + +## Management workers + +The chart now exposes `nodeSelector`, `tolerations`, `affinity`, +`topologySpreadConstraints`, and `priorityClassName` beneath: + +- `operator`, `api`, `web`, `gateway`, and `mcpServer`. +- `api.audit.webhook.syslogBridge` and `api.telemetry.receiver`. +- `crds.autoApply` for the upgrade hook Job. + +Each Deployment also supports `pdb.enabled` and either `pdb.minAvailable` or +`pdb.maxUnavailable`; enabled budgets default to `maxUnavailable: 1`. PDBs select +both the component name and Helm release. Disabled components get no PDB. Hook +Jobs have placement settings but no disruption budget. + +The API remains one process for both SQLite and PostgreSQL. Its optional PDB +only permits `maxUnavailable: 1`, so a voluntary node drain can evict it and will +cause API downtime. A PDB is not an availability guarantee and does not prevent +node failures. The persistent telemetry receiver is also a singleton; avoid a +zero-unavailable budget unless intentionally accepting a blocked drain. + +Keep operator leader election enabled with redundant replicas and rolling +updates. The chart rejects multiple replicas with leader election disabled; +with one replica and election disabled it uses Recreate to avoid overlapping +reconcilers during upgrades. API replica/process restrictions remain unchanged. + +The [example profile](../charts/gameplane/values-production.example.yaml) places +management services on labeled, tainted workers and spreads redundant operator +and dashboard pods. First label and taint the selected **worker** nodes: + +```sh +kubectl label node management-a management-b gameplane.local/pool=management +kubectl taint node management-a management-b gameplane.local/pool=management:NoSchedule +kubectl label node game-a game-b gameplane.local/pool=games +kubectl taint node game-a game-b gameplane.local/pool=games:NoSchedule +``` + +Use existing GameServer placement fields for game workers: + +```yaml +spec: + nodeSelector: { gameplane.local/pool: games } + tolerations: + - { key: gameplane.local/pool, operator: Equal, value: games, effect: NoSchedule } +``` + +Preserve control-plane taints. Avoid broad `operator: Exists` tolerations. +Optional management components need their corresponding placement values too. +Per-game sentinel and tunnel Deployments are operator-generated and are not +covered by these chart management placement keys. A priorityClassName must name +an existing PriorityClass. Unsatisfied hard selectors, taints or topology +constraints correctly leave pods Pending; do not silently relax them. + +Local or node-bound volumes cannot move just because a pod has a new selector. +Check API/receiver PVC node affinity before selecting management workers, and +game PVC topology before backups or moving game pods. Restic scheduling and +source preparation are separate from management placement. + +## Sizing and controlled-drain checks + +Collect per-container CPU/memory, pod requests, Pending events, game tick/query +latency, storage throughput and backup duration during busy periods and actual +backup concurrency. Use `kubectl top pods --containers`, scheduler events and +the available Prometheus metrics. Include headroom for one failed management +node, game startup, repository maintenance and replacement pods. No fixed server +count follows from these request values. + +In a disposable multi-worker environment, apply the example and verify every +management pod's node labels. Temporarily set a nonexistent selector and verify +Pending/FailedScheduling, then restore it. Cordon one management worker, evict +its redundant operator/web pods through the eviction API, and verify their PDB +allows only one unavailable replica while the remaining replica stays Ready. +Uncordon after checking replacements. A full `kubectl drain` also evicts the +singleton API and can interrupt access; account for local PVCs and validate +recovery before doing this in production. CI chart checks verify selectors, +PDB gates, resource clearing and the production profile; cluster behavior needs +the multi-node integration scenario, not just a successful render. diff --git a/docs/release-notes.md b/docs/release-notes.md new file mode 100644 index 000000000..f3da01f11 --- /dev/null +++ b/docs/release-notes.md @@ -0,0 +1,21 @@ +# Unreleased operational changes + +## Auxiliary resource requests and management placement (#724) + +Agent/config-init (25m CPU, 64Mi), sentinel (25m, 32Mi), tunnel (25m, 64Mi), and +Restic backup/init/restore (100m, 128Mi) now request resources on every install. +These are initial scheduling reservations, not workload capacity estimates. +No new CPU or memory limits are imposed. Existing forget/prune limits remain. + +Upgrades change generated Pod templates and can restart game/auxiliary pods or +leave them Pending when capacity is insufficient. Review capacity and schedule +the upgrade accordingly. The four `operator.*Resources` chart values provide +complete overrides; explicit empty objects clear defaults, while null/missing +keys use the defaults even with `--reuse-values`. + +Chart-managed pods now accept placement, topology-spread and priority settings, +with optional PDBs for Deployments. API remains a singleton and its PDB uses +`maxUnavailable: 1`; node drains can cause API downtime. Operator replicas greater +than one require leader election. Disabling election on a singleton uses Recreate. +See [production placement](production-placement.md) for upgrade controls and a +worker-pool example. diff --git a/operator/cmd/main.go b/operator/cmd/main.go index 85263e901..9625068fb 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -150,10 +150,18 @@ func main() { captureDefaultMaxSizeBytes int64 backupJobBackoffLimit int64 backupJobActiveDeadlineSeconds int64 + agentResourcesJSON string + sentinelResourcesJSON string + tunnelResourcesJSON string + backupResourcesJSON string ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "Address the metrics endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "Address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election.") + flag.StringVar(&agentResourcesJSON, "agent-resources", "", "JSON ResourceRequirements for agents and config init containers; default requests 25m/64Mi, {} clears defaults.") + flag.StringVar(&sentinelResourcesJSON, "sentinel-resources", "", "JSON ResourceRequirements for sentinels; default requests 25m/32Mi, {} clears defaults.") + flag.StringVar(&tunnelResourcesJSON, "tunnel-resources", "", "JSON ResourceRequirements for tunnel relays; default requests 25m/64Mi, {} clears defaults.") + flag.StringVar(&backupResourcesJSON, "backup-resources", "", "JSON ResourceRequirements for restic backup/init, restore and cleanup workers; default backup/restore requests 100m/128Mi, {} clears defaults.") flag.StringVar(&agentImage, "agent-image", "ghcr.io/gameplanepanel/gameplane/agent:dev", "Image to use for the Gameplane agent sidecar injected into game pods.") flag.StringVar(&agentImagePullPolicy, "agent-image-pull-policy", "", @@ -263,6 +271,26 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) setupLog := ctrl.Log.WithName("setup") + agentResources, err := controller.ParseAuxiliaryResources(agentResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --agent-resources") + os.Exit(1) + } + sentinelResources, err := controller.ParseAuxiliaryResources(sentinelResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --sentinel-resources") + os.Exit(1) + } + tunnelResources, err := controller.ParseAuxiliaryResources(tunnelResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --tunnel-resources") + os.Exit(1) + } + backupResources, err := controller.ParseAuxiliaryResources(backupResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --backup-resources") + os.Exit(1) + } backupJobBackoff32, err := boundedInt32(backupJobBackoffLimit) // rejects <0 and >MaxInt32 if err != nil { setupLog.Error(err, "invalid --backup-job-backoff-limit value (must be >= 0)", "value", backupJobBackoffLimit) @@ -380,6 +408,9 @@ func main() { APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), AgentImage: agentImage, + AgentResources: agentResources, + SentinelResources: sentinelResources, + TunnelResources: tunnelResources, AgentImagePullPolicy: agentImagePullPolicy, ConfigInitImage: configInitImage, SentinelImage: sentinelImage, @@ -419,6 +450,7 @@ func main() { Clientset: kubernetes.NewForConfigOrDie(mgr.GetConfig()), AgentClient: agentClient, ResticImage: resticImage, + BackupResources: backupResources, JobBackoffLimit: &backupJobBackoff32, JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds, EventRecorder: mgr.GetEventRecorder("backup-controller"), @@ -438,6 +470,7 @@ func main() { APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), ResticImage: resticImage, + BackupResources: backupResources, JobBackoffLimit: &backupJobBackoff32, JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds, }).SetupWithManager(mgr); err != nil { diff --git a/operator/internal/controller/auxiliary_resources.go b/operator/internal/controller/auxiliary_resources.go new file mode 100644 index 000000000..8f7bc7901 --- /dev/null +++ b/operator/internal/controller/auxiliary_resources.go @@ -0,0 +1,65 @@ +package controller + +import ( + "encoding/json" + "fmt" + "io" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation" +) + +// ParseAuxiliaryResources parses an install-level resource override. An absent +// flag returns nil (use defaults); an explicit object, including {}, replaces +// defaults wholesale. Reject typos before the operator starts creating pods. +func ParseAuxiliaryResources(value string) (*corev1.ResourceRequirements, error) { + if value == "" { + return nil, nil + } + var configured *corev1.ResourceRequirements + decoder := json.NewDecoder(strings.NewReader(value)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&configured); err != nil { + return nil, fmt.Errorf("decode resources: %w", err) + } + if configured == nil { + return nil, fmt.Errorf("resources must be a JSON object; use {} to clear defaults") + } + if err := decoder.Decode(new(any)); err != io.EOF { + return nil, fmt.Errorf("resources must contain exactly one JSON object") + } + if len(configured.Claims) != 0 { + return nil, fmt.Errorf("resource claims are not supported for auxiliary containers") + } + for _, values := range []corev1.ResourceList{configured.Requests, configured.Limits} { + for name, quantity := range values { + if len(validation.IsQualifiedName(string(name))) != 0 || quantity.Sign() < 0 { + return nil, fmt.Errorf("invalid resource %q or negative quantity", name) + } + } + } + for name, request := range configured.Requests { + if limit, exists := configured.Limits[name]; exists && request.Cmp(limit) > 0 { + return nil, fmt.Errorf("request for %s exceeds its limit", name) + } + } + return configured, nil +} + +func auxiliaryResources(configured *corev1.ResourceRequirements, cpu, memory string) corev1.ResourceRequirements { + if configured != nil { + return *configured.DeepCopy() + } + return corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(cpu), corev1.ResourceMemory: resource.MustParse(memory), + }} +} + +func optionalResources(values []*corev1.ResourceRequirements) *corev1.ResourceRequirements { + if len(values) == 0 { + return nil + } + return values[0] +} diff --git a/operator/internal/controller/auxiliary_resources_test.go b/operator/internal/controller/auxiliary_resources_test.go new file mode 100644 index 000000000..f63cef5f8 --- /dev/null +++ b/operator/internal/controller/auxiliary_resources_test.go @@ -0,0 +1,105 @@ +package controller + +import ( + "reflect" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestAuxiliaryResourceConfiguration(t *testing.T) { + for _, bad := range []string{"null", `{"request":{"cpu":"1"}}`, `{"requests":{"cpu":"-1"}}`, `{"requests":{"memory":"128Mi"},"limits":{"memory":"64Mi"}}`, `{} {}`} { + if _, err := ParseAuxiliaryResources(bad); err == nil { + t.Errorf("accepted invalid resources %s", bad) + } + } + unset, err := ParseAuxiliaryResources("") + if err != nil || unset != nil { + t.Fatalf("absent configuration: %v %v", unset, err) + } + cleared, err := ParseAuxiliaryResources(`{"requests":{},"limits":{}}`) + if err != nil || cleared == nil { + t.Fatalf("explicit clear: %v %v", cleared, err) + } + if got := auxiliaryResources(cleared, "25m", "64Mi"); len(got.Requests) != 0 || len(got.Limits) != 0 { + t.Fatal("clear restored default reservations") + } + defaults := auxiliaryResources(nil, "25m", "64Mi") + if defaults.Requests.Cpu().String() != "25m" || defaults.Requests.Memory().String() != "64Mi" || len(defaults.Limits) != 0 { + t.Fatalf("unexpected scheduling defaults: %+v", defaults) + } +} + +func TestAuxiliaryResourcesReachAgentAndResticContainers(t *testing.T) { + configured := &corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("73m")}} + gs := &gameplanev1alpha1.GameServer{} + tmpl := &gameplanev1alpha1.GameTemplate{} + agent := buildAgentContainer(gs, tmpl, nil, "agent", "", "", configured) + if !reflect.DeepEqual(agent.Resources, *configured) { + t.Fatalf("agent lost install resources: %+v", agent.Resources) + } + tmpl.Spec.Agent = &gameplanev1alpha1.AgentSpec{Image: "custom-agent"} + agent = buildAgentContainer(gs, tmpl, nil, "agent", "", "", configured) + if !reflect.DeepEqual(agent.Resources, *configured) { + t.Fatal("image-only template override removed resource defaults") + } + tmpl.Spec.Agent.Resources = corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("91Mi")}} + agent = buildAgentContainer(gs, tmpl, nil, "agent", "", "", configured) + if !reflect.DeepEqual(agent.Resources, tmpl.Spec.Agent.Resources) { + t.Fatal("template agent resources did not replace the install defaults") + } + b := &gameplanev1alpha1.Backup{Spec: gameplanev1alpha1.BackupSpec{RepoRef: &gameplanev1alpha1.SecretKeySelector{Name: "repo"}}} + br := &BackupReconciler{BackupResources: configured} + backup := br.buildBackupPodSpec(b, tmpl) + forget := br.buildForgetPodSpec(b) + rr := &RestoreReconciler{BackupResources: configured} + restore := rr.buildRestorePodSpec(&gameplanev1alpha1.Restore{}, b, tmpl) + for _, container := range append(append(append(backup.Containers, backup.InitContainers...), forget.Containers...), restore.Containers...) { + if !reflect.DeepEqual(container.Resources, *configured) { + t.Errorf("container %s lost resource configuration: %+v", container.Name, container.Resources) + } + } + if init := buildConfigInitContainer("init", tmpl, configured); !reflect.DeepEqual(init.Resources, *configured) { + t.Fatal("config init lost install resource configuration") + } + // Builders must not return the shared maps: one container's customization + // cannot mutate the defaults for another server or operation. + agent.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1Mi") + if tmpl.Spec.Agent.Resources.Requests.Memory().String() != "91Mi" { + t.Fatal("agent resources alias the template") + } +} + +func TestAuxiliaryDeploymentResourceOverrideAndRemoval(t *testing.T) { + s := wipeScheme(t) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "games", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "frp", Frp: &gameplanev1alpha1.FrpTunnelSpec{ServerAddr: "relay.example"}}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + custom := &corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("81m")}} + for _, configured := range []*corev1.ResourceRequirements{nil, custom, {}} { + r.SentinelResources, r.TunnelResources = configured, configured + if err := r.reconcileSentinel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + if err := r.reconcileTunnel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + for _, component := range []struct{ suffix, memory string }{{"waker", "32Mi"}, {"tunnel", "64Mi"}} { + var dep appsv1.Deployment + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-" + component.suffix}, &dep); err != nil { + t.Fatal(err) + } + want := auxiliaryResources(configured, "25m", component.memory) + if !reflect.DeepEqual(dep.Spec.Template.Spec.Containers[0].Resources, want) { + t.Fatalf("%s did not apply/clear resources", component.suffix) + } + } + } +} diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index b350a5448..3e5a82d2b 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -119,6 +119,9 @@ type BackupReconciler struct { // operator flag so air-gapped installs can point it at a private // registry mirror. Empty falls back to DefaultResticImage. ResticImage string + // BackupResources replaces resources for backup/init and repository cleanup + // workers. Nil uses defaults; an explicit empty object clears them. + BackupResources *corev1.ResourceRequirements // JobBackoffLimit / JobActiveDeadlineSeconds bound the restic backup Job. // Set from operator flags; nil / zero fall back to // DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds. @@ -965,9 +968,10 @@ func (r *BackupReconciler) buildBackupPodSpec(b *gameplanev1alpha1.Backup, tmpl return corev1.PodSpec{ SecurityContext: buildBackupPodSecurityContext(tmpl), InitContainers: []corev1.Container{{ - Name: "restic-init", - Image: resticImageOrDefault(r.ResticImage), - Command: []string{"/bin/sh", "-c"}, + Name: "restic-init", + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), + Command: []string{"/bin/sh", "-c"}, // `cat config` is the canonical "does the repo exist" probe; // `init` is idempotent against an empty target. Args: []string{"restic cat config --no-lock >/dev/null 2>&1 || restic init"}, @@ -978,10 +982,11 @@ func (r *BackupReconciler) buildBackupPodSpec(b *gameplanev1alpha1.Backup, tmpl SecurityContext: containerSec, }}, Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), - Args: args, - Env: env, + Name: "restic", + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), + Args: args, + Env: env, VolumeMounts: []corev1.VolumeMount{ {Name: "data", MountPath: "/data", ReadOnly: true}, {Name: "cache", MountPath: "/tmp"}, diff --git a/operator/internal/controller/backup_forget.go b/operator/internal/controller/backup_forget.go index 86f1c53cf..2d97d9410 100644 --- a/operator/internal/controller/backup_forget.go +++ b/operator/internal/controller/backup_forget.go @@ -420,6 +420,12 @@ func forgetJobName(backupName string) string { func (r *BackupReconciler) buildForgetPodSpec(b *gameplanev1alpha1.Backup) corev1.PodSpec { nonRoot := true uid := int64(65532) + resources := auxiliaryResources(r.BackupResources, "50m", "128Mi") + if r.BackupResources == nil { + // Preserve the existing bounded repository-cleanup default. Explicit + // install overrides replace it along with other restic worker resources. + resources.Limits = corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")} + } env := resticEnv(b.Spec.RepoRef.Name) env = append(env, corev1.EnvVar{Name: forgetSnapshotIDEnv, Value: b.Status.SnapshotID}) @@ -433,20 +439,12 @@ func (r *BackupReconciler) buildForgetPodSpec(b *gameplanev1alpha1.Backup) corev SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, }, Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), - Command: []string{"/bin/sh", "-c"}, - Args: []string{forgetScript}, - Env: env, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("50m"), - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - }, + Name: "restic", + Image: resticImageOrDefault(r.ResticImage), + Command: []string{"/bin/sh", "-c"}, + Args: []string{forgetScript}, + Env: env, + Resources: resources, VolumeMounts: []corev1.VolumeMount{ {Name: "cache", MountPath: "/tmp"}, }, diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 67e8be4e3..4093959d8 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -68,6 +68,12 @@ type GameServerReconciler struct { // to info and existing StatefulSets don't roll on operator upgrade. AgentLogLevel string + // Auxiliary resources are complete install-level overrides. Nil uses the + // built-in requests; a non-nil empty object explicitly clears them. + AgentResources *corev1.ResourceRequirements + SentinelResources *corev1.ResourceRequirements + TunnelResources *corev1.ResourceRequirements + // ConfigInitImage is the image for the init container that copies // operator-rendered config files onto the data volume. Set from an // operator flag so air-gapped installs can point it at a private @@ -1509,7 +1515,7 @@ func (r *GameServerReconciler) reconcileStatefulSet( ss.Spec.Template.Annotations = ann ss.Spec.Template.Spec.Containers = []corev1.Container{ buildGameContainer(gs, tmpl, image, ver, mc), - buildAgentContainer(gs, tmpl, ver, r.AgentImage, r.AgentLogLevel, r.AgentImagePullPolicy), + buildAgentContainer(gs, tmpl, ver, r.AgentImage, r.AgentLogLevel, r.AgentImagePullPolicy, r.AgentResources), } volumes := []corev1.Volume{ { @@ -1569,7 +1575,7 @@ func (r *GameServerReconciler) reconcileStatefulSet( }, }, }) - ss.Spec.Template.Spec.InitContainers = []corev1.Container{buildConfigInitContainer(r.ConfigInitImage, tmpl)} + ss.Spec.Template.Spec.InitContainers = []corev1.Container{buildConfigInitContainer(r.ConfigInitImage, tmpl, r.AgentResources)} } else { ss.Spec.Template.Spec.InitContainers = nil } @@ -1983,7 +1989,7 @@ func configInitImageOrDefault(image string) string { // When the template sets security.fsGroup, config-init makes each file // and directory it copied group-writable so a non-root game sharing that // fsGroup can update them. -func buildConfigInitContainer(image string, tmpl *gameplanev1alpha1.GameTemplate) corev1.Container { +func buildConfigInitContainer(image string, tmpl *gameplanev1alpha1.GameTemplate, defaults ...*corev1.ResourceRequirements) corev1.Container { image = configInitImageOrDefault(image) mountPath := effectiveMountPath(tmpl) cpCmd := "cp -RL " + configFilesStagingPath + "/* '" + mountPath + "/'" @@ -1996,9 +2002,10 @@ func buildConfigInitContainer(image string, tmpl *gameplanev1alpha1.GameTemplate } return corev1.Container{ - Name: "config-init", - Image: image, - Command: []string{"/bin/sh", "-c"}, + Name: "config-init", + Image: image, + Resources: auxiliaryResources(optionalResources(defaults), "25m", "64Mi"), + Command: []string{"/bin/sh", "-c"}, // -L dereferences the kubelet's per-key symlinks; the * glob // skips the ..data/.. dot-entries of the Secret mount. Args: []string{cpCmd}, @@ -2242,14 +2249,17 @@ func gamePodSecurityContext(tmpl *gameplanev1alpha1.GameTemplate) *corev1.PodSec func buildAgentContainer( gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate, ver *gameplanev1alpha1.GameVersion, fallbackImage, logLevel, pullPolicy string, + defaults ...*corev1.ResourceRequirements, ) corev1.Container { image := fallbackImage - res := corev1.ResourceRequirements{} + res := auxiliaryResources(optionalResources(defaults), "25m", "64Mi") if tmpl.Spec.Agent != nil { if tmpl.Spec.Agent.Image != "" { image = tmpl.Spec.Agent.Image } - res = tmpl.Spec.Agent.Resources + if tmpl.Spec.Agent.Resources.Requests != nil || tmpl.Spec.Agent.Resources.Limits != nil || len(tmpl.Spec.Agent.Resources.Claims) != 0 { + res = *tmpl.Spec.Agent.Resources.DeepCopy() + } } mountPath := effectiveMountPath(tmpl) nonRoot := true diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index b2c8a9edc..5415fbbf8 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -278,8 +278,9 @@ func (r *GameServerReconciler) reconcileSentinel( dep.Spec.Template.Spec.ServiceAccountName = sentinelSAName(gs) dep.Spec.Template.Spec.Containers = []corev1.Container{{ - Name: "sentinel", - Image: image, + Name: "sentinel", + Image: image, + Resources: auxiliaryResources(r.SentinelResources, "25m", "32Mi"), Env: []corev1.EnvVar{ {Name: "GAMESERVER_NAME", Value: gs.Name}, {Name: "GAMESERVER_NAMESPACE", Value: gs.Namespace}, diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index a950292b2..e5712d96b 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -392,6 +392,7 @@ func (r *GameServerReconciler) reconcileTunnel( dep.Spec.Template.Spec.Containers = []corev1.Container{{ Name: "tunnel", Image: image, + Resources: auxiliaryResources(r.TunnelResources, "25m", "64Mi"), Env: envVars, Ports: ports, VolumeMounts: volumeMounts, diff --git a/operator/internal/controller/restore_controller.go b/operator/internal/controller/restore_controller.go index 4cbf94e10..fe46011ef 100644 --- a/operator/internal/controller/restore_controller.go +++ b/operator/internal/controller/restore_controller.go @@ -36,6 +36,8 @@ type RestoreReconciler struct { // operator flag so air-gapped installs can point it at a private // registry mirror. Empty falls back to DefaultResticImage. ResticImage string + // BackupResources shares the backup worker resource configuration. + BackupResources *corev1.ResourceRequirements // JobBackoffLimit / JobActiveDeadlineSeconds bound the restic restore Job. // Set from operator flags; nil / zero fall back to // DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds. @@ -327,8 +329,9 @@ func (r *RestoreReconciler) buildRestorePodSpec( return corev1.PodSpec{ SecurityContext: buildBackupPodSecurityContext(tmpl), Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), + Name: "restic", + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), // --target / restores each snapshot entry at its original // absolute path. The companion backup runs `restic backup // /data`, so the snapshot tree is rooted at /data/...; with diff --git a/operator/specs.md b/operator/specs.md index c7aea8225..2db4f71cf 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -131,6 +131,20 @@ Nine CRD kinds under `gameplane.local/v1alpha1`: ## Reconcilers +### Auxiliary resource reservations + +Install flags `--agent-resources`, `--sentinel-resources`, `--tunnel-resources` +and `--backup-resources` accept complete JSON ResourceRequirements objects. +Absent flags use requests of 25m/64Mi for agent/config-init, 25m/32Mi for sentinel, +25m/64Mi for each tunnel provider, and 100m/128Mi for Restic backup/init/restore. +No new limits are set. Forget/prune retains 50m/128Mi and its existing 512Mi +memory limit when no override is configured. Explicit `{}` clears defaults. +Builders deep-copy resource maps. Template agent requests/limits, when declared, +replace install agent defaults; image-only overrides keep those defaults. +Game resources and GameServer placement retain their existing semantics. +Chart placement and optional PDBs are documented in +[`docs/production-placement.md`](../docs/production-placement.md). + Primary reconcilers register with the manager in `cmd/main.go` and handle CRD lifecycle: ### GameServerReconciler diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d60a26f14..4ea08f43e 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -392,10 +392,15 @@ EOF # Tests that exist in the suite but deliberately run in NO bucket. Every # entry needs a reason; `verify` fails on any unlisted stray so additions # here are a conscious, reviewed act. Currently empty. +bucket_management_placement() { cat <<'EOF' +TestManagementPlacementAndDisruption +EOF +} + unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components management-placement } list_bucket() { @@ -414,6 +419,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + management-placement) bucket_management_placement ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/management_placement_e2e_test.go b/test/e2e/management_placement_e2e_test.go new file mode 100644 index 000000000..c212de9ea --- /dev/null +++ b/test/e2e/management_placement_e2e_test.go @@ -0,0 +1,251 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// This bucket has no API logins and runs alone on the three-node management +// profile. It uses chart-rendered placement/PDBs with sleeping fixture containers +// to exercise the real scheduler and eviction controller without game downloads. +func TestManagementPlacementAndDisruption(t *testing.T) { + e := envInstance + nodes, err := e.K8s.CoreV1().Nodes().List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + var candidates []string + for _, node := range nodes.Items { + eligible := !node.Spec.Unschedulable + for _, taint := range node.Spec.Taints { + if taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute { + eligible = false + } + } + ready := false + for _, condition := range node.Status.Conditions { + ready = ready || (condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue) + } + if eligible && ready { + candidates = append(candidates, node.Name) + } + } + if len(candidates) < 2 { + t.Fatal("management placement requires two schedulable workers; use the management e2e profile") + } + candidates = candidates[:2] + ns := fmt.Sprintf("placement-%d", time.Now().UnixNano()) + if _, err := e.K8s.CoreV1().Namespaces().Create(t.Context(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = e.K8s.CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}) }) + poolLabel := "gameplane.local/" + ns + for _, name := range candidates { + patchNode := func(value any) error { + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"labels": map[string]any{poolLabel: value}}}) + if err != nil { + return err + } + _, err = e.K8s.CoreV1().Nodes().Patch(context.Background(), name, types.MergePatchType, patch, metav1.PatchOptions{}) + return err + } + if err := patchNode("management"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = patchNode(nil) }) + } + + chart, err := findRepoRel("charts/gameplane") + if err != nil { + t.Fatal(err) + } + selector := map[string]any{"app.kubernetes.io/name": "gameplane-web", "app.kubernetes.io/instance": ns} + values := map[string]any{ + "operator": map[string]any{"enabled": false}, + "api": map[string]any{"pdb": map[string]any{"enabled": true, "maxUnavailable": 1}}, + "web": map[string]any{ + "enabled": true, "replicas": 2, "nodeSelector": map[string]any{poolLabel: "management"}, + "pdb": map[string]any{"enabled": true, "maxUnavailable": 1}, + "affinity": map[string]any{"podAntiAffinity": map[string]any{"requiredDuringSchedulingIgnoredDuringExecution": []any{map[string]any{"topologyKey": "kubernetes.io/hostname", "labelSelector": map[string]any{"matchLabels": selector}}}}}, + "topologySpreadConstraints": []any{map[string]any{"maxSkew": 1, "topologyKey": "kubernetes.io/hostname", "whenUnsatisfiable": "DoNotSchedule", "labelSelector": map[string]any{"matchLabels": selector}}}, + }, + } + data, err := json.Marshal(values) + if err != nil { + t.Fatal(err) + } + valuesPath := filepath.Join(t.TempDir(), "values.json") + if err := os.WriteFile(valuesPath, data, 0o600); err != nil { + t.Fatal(err) + } + rendered, err := exec.CommandContext(t.Context(), "helm", "template", ns, chart, "--namespace", ns, "--values", valuesPath, "--show-only", "templates/web.yaml", "--show-only", "templates/poddisruptionbudgets.yaml").CombinedOutput() + if err != nil { + t.Fatalf("render placement chart: %v: %s", err, rendered) + } + fixture := corev1.Container{Name: "fixture", Image: "busybox:1.38.0", Command: []string{"sleep", "3600"}} + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 4096) + for { + var object unstructured.Unstructured + if err := decoder.Decode(&object); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + switch object.GetKind() { + case "Deployment": + var deployment appsv1.Deployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.Object, &deployment); err != nil { + t.Fatal(err) + } + deployment.Spec.Template.Spec.Containers = []corev1.Container{fixture} + if _, err := e.K8s.AppsV1().Deployments(ns).Create(t.Context(), &deployment, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + case "PodDisruptionBudget": + var pdb policyv1.PodDisruptionBudget + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.Object, &pdb); err != nil { + t.Fatal(err) + } + if _, err := e.K8s.PolicyV1().PodDisruptionBudgets(ns).Create(t.Context(), &pdb, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + } + webPods := func() ([]corev1.Pod, error) { + list, err := e.K8s.CoreV1().Pods(ns).List(t.Context(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gameplane-web"}) + if err != nil { + return nil, err + } + return list.Items, nil + } + waitReady := func() { + e.Eventually(t, 3*time.Minute, func() (bool, string) { + pods, err := webPods() + if err != nil || len(pods) != 2 { + return false, fmt.Sprintf("pods=%d error=%v", len(pods), err) + } + seen := map[string]bool{} + for _, pod := range pods { + ready, err := e.PodIsReady(t.Context(), ns, pod.Name) + if err != nil || !ready || !pod.DeletionTimestamp.IsZero() { + return false, "web fixture not ready" + } + if pod.Spec.NodeName != candidates[0] && pod.Spec.NodeName != candidates[1] { + t.Fatalf("pod escaped management selector to %s", pod.Spec.NodeName) + } + seen[pod.Spec.NodeName] = true + } + return len(seen) == 2, "web replicas were not spread across two workers" + }) + } + waitReady() + setSelector := func(value string) { + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{"nodeSelector": map[string]any{poolLabel: value}}}}}) + if err != nil { + t.Fatal(err) + } + if _, err := e.K8s.AppsV1().Deployments(ns).Patch(t.Context(), "gameplane-web", types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + } + setSelector("absent-pool") + e.Eventually(t, time.Minute, func() (bool, string) { + pods, err := webPods() + if err != nil { + return false, err.Error() + } + for _, pod := range pods { + for _, condition := range pod.Status.Conditions { + if pod.Spec.NodeSelector[poolLabel] == "absent-pool" && condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && condition.Reason == "Unschedulable" { + return true, "" + } + } + } + return false, "missing unschedulable replacement" + }) + setSelector("management") + waitReady() + cordon := func(unschedulable bool) error { + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"unschedulable": unschedulable}}) + if err != nil { + return err + } + _, err = e.K8s.CoreV1().Nodes().Patch(context.Background(), candidates[0], types.MergePatchType, patch, metav1.PatchOptions{}) + return err + } + if err := cordon(true); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cordon(false) }) + waitBudget := func(name string, allowed int32) { + e.Eventually(t, time.Minute, func() (bool, string) { + pdb, err := e.K8s.PolicyV1().PodDisruptionBudgets(ns).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return pdb.Status.ObservedGeneration >= pdb.Generation && pdb.Status.DisruptionsAllowed == allowed, fmt.Sprintf("budget status: %+v", pdb.Status) + }) + } + waitBudget("gameplane-web", 1) + pods, err := webPods() + if err != nil { + t.Fatal(err) + } + var remaining string + for _, pod := range pods { + if pod.Spec.NodeName == candidates[0] { + if err := e.K8s.CoreV1().Pods(ns).EvictV1(t.Context(), &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Name: pod.Name, Namespace: ns}}); err != nil { + t.Fatal(err) + } + } else { + remaining = pod.Name + } + } + waitBudget("gameplane-web", 0) + if err := e.K8s.CoreV1().Pods(ns).EvictV1(t.Context(), &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Name: remaining, Namespace: ns}}); !apierrors.IsTooManyRequests(err) { + t.Fatalf("second eviction bypassed web availability budget: %v", err) + } + if ready, err := e.PodIsReady(t.Context(), ns, remaining); err != nil || !ready { + t.Fatalf("remaining web replica lost readiness: %v", err) + } + if err := cordon(false); err != nil { + t.Fatal(err) + } + waitReady() + // The chart's API budget must permit the only API pod to be evicted. This + // isolated labeled fixture tests the PDB without stopping the suite's API. + apiLabels := map[string]string{"app.kubernetes.io/name": "gameplane-api", "app.kubernetes.io/instance": ns} + one := int32(1) + apiDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "gameplane-api", Namespace: ns}, Spec: appsv1.DeploymentSpec{Replicas: &one, Selector: &metav1.LabelSelector{MatchLabels: apiLabels}, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: apiLabels}, Spec: corev1.PodSpec{Containers: []corev1.Container{fixture}}}}} + if _, err := e.K8s.AppsV1().Deployments(ns).Create(t.Context(), apiDeployment, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + waitBudget("gameplane-api", 1) + apiPods, err := e.K8s.CoreV1().Pods(ns).List(t.Context(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gameplane-api"}) + if err != nil || len(apiPods.Items) != 1 { + t.Fatalf("API fixture pods: %v", err) + } + if err := e.K8s.CoreV1().Pods(ns).EvictV1(t.Context(), &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Name: apiPods.Items[0].Name, Namespace: ns}}); err != nil { + t.Fatalf("API singleton PDB blocked drain: %v", err) + } +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index abf7cda91..75fff7eb7 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -40,6 +40,17 @@ are removed through test cleanup. CI sets `GAMEPLANE_E2E_REQUIRE_GATEWAY=1`, so missing second cluster fails instead of skipping the scenario. The multicluster Go invocation has a 45-minute deadline within the existing 60-minute job budget. +### Management placement and disruption acceptance + +The `management-placement` bucket uses the three-node `management` Kind profile +and performs no API login. `TestManagementPlacementAndDisruption` applies the +chart's web placement and API/web PDBs in an isolated namespace, with sleeping +fixture containers. It verifies real scheduling across labeled workers, +Unschedulable status for an impossible selector, one-at-a-time eviction during +a cordon, readiness of the surviving replica, and singleton API eviction. Node +labels and cordons are restored in cleanup. This validates scheduler/PDB behavior, +not application HA or storage migration. + ## Non-goals / boundaries - Does **not** replace the unit or envtest tiers for any other module — it exercises integration behavior, not the exhaustive branch coverage those tiers own (this module carries no `.testcoverage.yml` coverage gate). From 94b7fd76b75a245172cb9437c60672619fd5f889 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 11:50:34 -0400 Subject: [PATCH 2/6] fix: handle wrapped decoder EOF in placement drill Signed-off-by: Baylem --- test/e2e/management_placement_e2e_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/management_placement_e2e_test.go b/test/e2e/management_placement_e2e_test.go index c212de9ea..ec4525dfa 100644 --- a/test/e2e/management_placement_e2e_test.go +++ b/test/e2e/management_placement_e2e_test.go @@ -6,6 +6,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -106,7 +107,7 @@ func TestManagementPlacementAndDisruption(t *testing.T) { decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 4096) for { var object unstructured.Unstructured - if err := decoder.Decode(&object); err == io.EOF { + if err := decoder.Decode(&object); errors.Is(err, io.EOF) { break } else if err != nil { t.Fatal(err) From 9d2c3118615e821d473a86d0b914614c7f58109c Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:43:20 -0400 Subject: [PATCH 3/6] ci: require and report every audit E2E bucket Signed-off-by: Baylem --- .github/workflows/ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a902708f0..70346b980 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2518,7 +2518,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2671,7 +2671,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); From fd5d3d5d8139b18251a09a83ba543ae5ff866ea0 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 17:13:32 -0400 Subject: [PATCH 4/6] fix(chart): migrate operator strategy before applying Recreate Signed-off-by: Baylem --- .../templates/operator-strategy-migrate.yaml | 85 +++++++++++++++++++ charts/gameplane/templates/operator.yaml | 5 ++ charts/tests/check_workloads.py | 34 ++++++++ docs/production-placement.md | 7 ++ test/e2e/upgrade_e2e_test.go | 13 +++ 5 files changed, 144 insertions(+) create mode 100644 charts/gameplane/templates/operator-strategy-migrate.yaml diff --git a/charts/gameplane/templates/operator-strategy-migrate.yaml b/charts/gameplane/templates/operator-strategy-migrate.yaml new file mode 100644 index 000000000..e376fca4b --- /dev/null +++ b/charts/gameplane/templates/operator-strategy-migrate.yaml @@ -0,0 +1,85 @@ +{{- /* SSA cannot remove a server-defaulted rollingUpdate field. Patch it before +Helm applies Recreate, including installations that manage CRDs externally. */ -}} +{{- if and .Release.IsUpgrade (eq (include "gameplane.operatorEnabled" .) "true") (not .Values.operator.leaderElect) }} +{{- $existing := lookup "apps/v1" "Deployment" .Release.Namespace "gameplane-operator" }} +{{- if $existing }} +{{- $name := printf "%s-op-roll" .Release.Name }} +{{- $annotations := dict "helm.sh/hook" "pre-upgrade" "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gameplane.labels" . | nindent 4 }} + annotations: {{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $annotations) | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gameplane.labels" . | nindent 4 }} + annotations: {{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $annotations) | nindent 4 }} +rules: + - apiGroups: [apps] + resources: [deployments] + resourceNames: [gameplane-operator] + verbs: [get, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gameplane.labels" . | nindent 4 }} + annotations: {{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $annotations) | nindent 4 }} +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $name }} } +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gameplane.labels" . | nindent 4 }} + annotations: {{- toYaml (merge (dict "helm.sh/hook-weight" "-4") $annotations) | nindent 4 }} +spec: + backoffLimit: 2 + ttlSecondsAfterFinished: 300 + template: + spec: + {{- include "gameplane.placement" .Values.crds.autoApply | nindent 6 }} + serviceAccountName: {{ $name }} + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + fsGroup: 65532 + seccompProfile: { type: RuntimeDefault } + containers: + - name: kubectl + image: {{ .Values.crds.autoApply.image | quote }} + imagePullPolicy: {{ .Values.crds.autoApply.pullPolicy }} + command: [kubectl, patch, deployment, gameplane-operator, --type=merge, -p, '{"spec":{"strategy":{"type":"Recreate","rollingUpdate":null}}}'] + env: + - { name: HOME, value: /tmp } + resources: + requests: { cpu: 50m, memory: 64Mi } + limits: { cpu: 500m, memory: 256Mi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: [ALL] } + volumeMounts: + - { name: tmp, mountPath: /tmp } + volumes: + - name: tmp + emptyDir: {} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 444a60936..0111536df 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -244,6 +244,11 @@ spec: strategy: type: Recreate rollingUpdate: null + {{- else }} + # Explicitly restore rolling updates when election is re-enabled after a + # singleton install; the migration hook may own the previous Recreate field. + strategy: + type: RollingUpdate {{- end }} selector: matchLabels: diff --git a/charts/tests/check_workloads.py b/charts/tests/check_workloads.py index 33fd6f89f..a4c433528 100644 --- a/charts/tests/check_workloads.py +++ b/charts/tests/check_workloads.py @@ -9,8 +9,42 @@ from check_gateway import CHART, one, render, settings +def check_operator_migration(): + """Exercise live-lookup hook output with the longest valid Helm name.""" + with tempfile.TemporaryDirectory() as directory: + chart = Path(directory) / "chart" + shutil.copytree(CHART, chart) + template = chart / "templates/operator-strategy-migrate.yaml" + source = template.read_text() + lookup = 'lookup "apps/v1" "Deployment" .Release.Namespace "gameplane-operator"' + assert lookup in source + # helm template has no API server. Supply only the existing-object + # lookup response; leave the actual hook gates and resources intact. + template.write_text(source.replace(lookup, 'dict "metadata" (dict "name" "gameplane-operator")')) + release = "r" * 53 + name = release + "-op-roll" + values = {"api": {"enabled": False}, "operator": {"leaderElect": False}, + "crds": {"autoApply": {"enabled": False, "nodeSelector": {"pool": "management"}}}} + objects = render(values, upgrade=True, release=release, chart=chart) + job = one(objects, "Job", name) + assert len(job["metadata"]["name"]) <= 63 + pod = job["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == name + assert pod["nodeSelector"] == {"pool": "management"} + assert one(objects, "Role", name)["rules"] == [{"apiGroups": ["apps"], "resources": ["deployments"], + "resourceNames": ["gameplane-operator"], "verbs": ["get", "patch"]}] + binding = one(objects, "RoleBinding", name) + assert binding["roleRef"]["name"] == name and binding["subjects"][0]["name"] == name + one(objects, "ServiceAccount", name) + assert not any(obj["kind"] == "Job" for obj in render(values, release=release, chart=chart)) + values["operator"]["leaderElect"] = True + assert not any(obj["kind"] == "Job" for obj in render(values, upgrade=True, release=release, chart=chart)) + + def main(): + check_operator_migration() defaults = render() + assert one(defaults, "Deployment", "gameplane-operator")["spec"]["strategy"]["type"] == "RollingUpdate" assert not any(obj["kind"] == "PodDisruptionBudget" for obj in defaults) args = one(defaults, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] assert not any(arg.startswith("--tunnel-resources=") for arg in args) diff --git a/docs/production-placement.md b/docs/production-placement.md index 40bdb41b0..175d23f3a 100644 --- a/docs/production-placement.md +++ b/docs/production-placement.md @@ -87,6 +87,13 @@ updates. The chart rejects multiple replicas with leader election disabled; with one replica and election disabled it uses Recreate to avoid overlapping reconcilers during upgrades. API replica/process restrictions remain unchanged. +When switching an existing operator to Recreate, a pre-upgrade hook clears the +server-defaulted RollingUpdate fields before Helm applies the Deployment. The +hook uses `crds.autoApply.image` and its placement settings, including when CRDs +are managed externally. It can patch only `gameplane-operator` in the release +namespace and is skipped when that Deployment does not exist. Upgrades need +access to the configured kubectl image. + The [example profile](../charts/gameplane/values-production.example.yaml) places management services on labeled, tainted workers and spreads redundant operator and dashboard pods. First label and taint the selected **worker** nodes: diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 25a1221c9..65d9007b9 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -115,6 +115,12 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { } // ---- 3. upgrade to the working tree ---------------------------------- + // Exercise the SSA migration even when the published fixture already uses + // Recreate: Kubernetes defaults rollingUpdate on this merge patch. + if out, err := envInstance.Kubectl(ctx, "-n", "gameplane-system", "patch", "deployment", "gameplane-operator", + "--type=merge", "-p", `{"spec":{"strategy":{"type":"RollingUpdate"}}}`); err != nil { + t.Fatalf("seed operator RollingUpdate strategy: %v\n%s", err, out) + } upgrade := exec.CommandContext(ctx, "helm", "upgrade", "gameplane", filepath.Join(repoRoot, "charts", "gameplane"), @@ -133,6 +139,13 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { if out, err := upgrade.CombinedOutput(); err != nil { t.Fatalf("helm upgrade to the working-tree chart failed: %v\n%s", err, out) } + operator, err := envInstance.K8s.AppsV1().Deployments("gameplane-system").Get(ctx, "gameplane-operator", metav1.GetOptions{}) + if err != nil { + t.Fatalf("read upgraded operator: %v", err) + } + if operator.Spec.Strategy.Type != "Recreate" || operator.Spec.Strategy.RollingUpdate != nil { + t.Fatalf("operator strategy migration retained RollingUpdate fields: %+v", operator.Spec.Strategy) + } // ---- 4. the CRDs were actually updated ------------------------------- From ec3c21b11ff0d0469c5340e4cfae14f99e42b9dc Mon Sep 17 00:00:00 2001 From: Baylem Date: Thu, 22 Oct 2026 20:26:46 -0400 Subject: [PATCH 5/6] test(ci): cover audit buckets in full selection fixtures Signed-off-by: Baylem --- test/e2e/ci_selection_e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index d2d6edd00..a52268ad2 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", + "telemetry", "optional-components", "management-placement", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. From e079e7f2efc72b51d11a7700b729ae90b56ae28c Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:19:17 -0400 Subject: [PATCH 6/6] test: synchronize closed RCON response fixtures Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption_test.go | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 248c6a7fb..9061d2c94 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1275,18 +1275,47 @@ type pipeFrameCase struct { forbidErr string // substring the error must NOT contain (e.g. a detail suffix) } +type responseDeadlineConn struct { + net.Conn + deadlineSet chan<- struct{} +} + +func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error { + if err := c.Conn.SetDeadline(deadline); err != nil { + return err + } + select { + case c.deadlineSet <- struct{}{}: + default: + } + return nil +} + // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe // and checks the body or the error. func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + deadlineSet := make(chan struct{}, 1) client := newPipeClient(t, func(remote net.Conn) { if _, _, err := readNuclearOptionRequest(remote); err != nil { return } + if len(tt.frame) == 0 { + // Exec sets its response deadline after sending the request. + // Closing first can fail SetDeadline instead of the status read. + select { + case <-deadlineSet: + case <-t.Context().Done(): + return + } + } writeFrameSegments(remote, tt.frame, tt.oneByte) }) + if len(tt.frame) == 0 { + client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet} + } body, err := client.Exec("test-command") if tt.wantErr != "" {