diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 26d9ee778..f901237f0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -439,7 +439,8 @@ jobs: needs: [changes] if: (needs.changes.outputs.images == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'build-images') runs-on: ubuntu-latest - timeout-minutes: 15 + # Include all image builds, cache exports, artifact upload and cleanup. + timeout-minutes: 30 permissions: contents: read steps: @@ -1793,13 +1794,17 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, wake-hostport] arch: [amd64] include: - bucket: operator parallel: 8 test_timeout: 20m job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: wake-hostport + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: api-auth parallel: 4 test_timeout: 15m @@ -1856,7 +1861,9 @@ jobs: arch: ${{ matrix.arch }} - name: bring up cluster + helm install if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' - run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e + env: + GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'wake-hostport' && 'two-node' || 'default' }} + run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) if: env.RUN_REGEX != '' run: | @@ -2660,7 +2667,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|wake-hostport) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2813,7 +2820,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|wake-hostport) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 248c6a7fb..9061d2c94 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1275,18 +1275,47 @@ type pipeFrameCase struct { forbidErr string // substring the error must NOT contain (e.g. a detail suffix) } +type responseDeadlineConn struct { + net.Conn + deadlineSet chan<- struct{} +} + +func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error { + if err := c.Conn.SetDeadline(deadline); err != nil { + return err + } + select { + case c.deadlineSet <- struct{}{}: + default: + } + return nil +} + // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe // and checks the body or the error. func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + deadlineSet := make(chan struct{}, 1) client := newPipeClient(t, func(remote net.Conn) { if _, _, err := readNuclearOptionRequest(remote); err != nil { return } + if len(tt.frame) == 0 { + // Exec sets its response deadline after sending the request. + // Closing first can fail SetDeadline instead of the status read. + select { + case <-deadlineSet: + case <-t.Context().Done(): + return + } + } writeFrameSegments(remote, tt.frame, tt.oneByte) }) + if len(tt.frame) == 0 { + client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet} + } body, err := client.Exec("test-command") if tt.wantErr != "" { diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 772e0b10e..d395f52eb 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -16,6 +16,10 @@ metadata: name: {{ .Release.Name }}-operator-watch labels: {{- include "gameplane.labels" . | nindent 4 }} rules: + # Direct node identity/readiness checks for durable Hostport wake endpoints. + - apiGroups: [""] + resources: [nodes] + verbs: [get] - apiGroups: ["gameplane.local"] resources: [gameservers, gametemplates, backups, backupschedules, restores, modules, modulesources, networkcaptures] verbs: [get, list, watch] diff --git a/deploy/kind/e2e.sh b/deploy/kind/e2e.sh index 3c2f7b12f..c6f1d8e72 100755 --- a/deploy/kind/e2e.sh +++ b/deploy/kind/e2e.sh @@ -6,7 +6,7 @@ # deploy/kind/e2e.sh down [cluster-name] # # Diffs from dev-up (deploy/kind/up.sh): -# - Single-node cluster (faster boot, sufficient for E2E coverage). +# - Single-node cluster by default; two-node profile tests Hostport handoff. # - Skips ingress-nginx (the dashboard isn't exercised here). # - Loads pre-built gameplane-test/{operator,api,agent,sentinel,capture-sidecar,telemetry-receiver,fakeoidc}: # images (and gameplane-test/gameprobe: when present). @@ -28,9 +28,9 @@ CLUSTER="${2:-gameplane-e2e}" TAG="${3:-${GAMEPLANE_E2E_TAG:-e2e}}" PROFILE="${4:-default}" case "${PROFILE}" in -default | postgres) ;; +default | postgres | two-node) ;; *) - echo "unknown profile '${PROFILE}' (want default or postgres)" >&2 + echo "unknown profile '${PROFILE}' (want default, postgres or two-node)" >&2 exit 2 ;; esac @@ -188,12 +188,17 @@ up) echo "removing pre-existing cluster ${CLUSTER} for a clean slate" kind delete cluster --name "${CLUSTER}" || true fi - echo "creating kind cluster ${CLUSTER} (single-node)" + echo "creating kind cluster ${CLUSTER} (${PROFILE})" + extra_nodes="" + if [[ "${PROFILE}" == "two-node" ]]; then + extra_nodes=" - role: worker" + fi kind create cluster --name "${CLUSTER}" --wait 90s --config - </dev/null diff --git a/docs/archive/superpowers/specs/2026-07-24-wake-on-connect-design.md b/docs/archive/superpowers/specs/2026-07-24-wake-on-connect-design.md index c8632d253..51f5288e6 100644 --- a/docs/archive/superpowers/specs/2026-07-24-wake-on-connect-design.md +++ b/docs/archive/superpowers/specs/2026-07-24-wake-on-connect-design.md @@ -79,7 +79,45 @@ A new Service `-game-direct` — an always-ClusterIP Service selecting the g 4. Game Service selector flips back to `gameserver: ` 5. Sentinel Deployment deleted last -Connections already held in the proxy survive the flip-back because they connect through `-game-direct`, not the re-pointed game Service. In Hostport mode the sentinel relinquishes the host port on deletion; the game pod briefly Pending self-heals via scheduler retry. +Connections already held in the proxy survive the flip-back because they connect through `-game-direct`, not the re-pointed game Service. + +**Hostport correction (#730, October 2026):** Hostport uses a separate handoff. +Before its first sleep, the operator saves the live game's node name and Node UID +in `gameplane.local/hostport-node` and `gameplane.local/hostport-node-uid`. These +annotations survive Pod deletion and operator restart; the dashboard's current +`gameplane.local/node` annotation remains a separate, transient observation. +The game completes its normal graceful stop and scales to zero. Only after the +StatefulSet acknowledges that generation and a direct API list confirms all game +Pods have disappeared does the operator create the sentinel. The sentinel has +required node affinity for that identity, the game's tolerations and scheduling +constraints, and a Recreate strategy. It never bypasses the scheduler with +`spec.nodeName`. + +On wake, foreground deletion removes the sentinel Deployment and its children. +The game remains at zero until a direct API read confirms the Deployment is gone +and no sentinel Pods remain, including terminating Pods. The game then starts +with the same required endpoint-node affinity. Hostport clients still reconnect; +the old "briefly Pending, self-heals" sequencing is superseded. Service-backed +exposure retains the hold-and-proxy drain described above. + +`HostportWakeReady=False` explains `GameDraining`, `SentinelDraining`, +`EndpointNodeUnknown`, `EndpointNodeUnavailable`, `EndpointUnschedulable`, or +`ListenerPending`. A full node stays pinned and reports the scheduler's reason; +a missing, replaced or unready node makes wake unavailable. The operator never +silently changes the advertised node. Unrelated child reconciliation continues +during these waits. After restoring node capacity/readiness, reconciliation +resumes automatically. For intentional migration, suspend the server, wait for +both game and sentinel Pods to disappear, clear both durable endpoint annotations, +update scheduling/storage placement and the client-facing node address, then +resume the game before sleeping it again. Do not clear the annotations while a +listener remains active. An installation upgraded while already asleep without +a recorded game node must first start its game to establish placement. + +**Wake budget correction (#729):** The join deadline starts before the Kubernetes +PATCH, which has a separate five-second maximum within that budget. Concurrent +callers share the actual result of the in-flight PATCH. UDP packet reads enqueue +coalesced requests to one bounded worker and never wait on the PATCH. Successful +TCP proxy sessions use the longer session/drain context after upstream connects. ### Connection handling state machine diff --git a/hack/ci_scope.py b/hack/ci_scope.py index 91da39131..86884a383 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -29,7 +29,8 @@ FORCE_ALL_PREFIXES = (".github/actions/",) # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml. E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components"] + "telemetry", "optional-components", "wake-hostport"] +E2E_AMD64_ONLY = {"wake-hostport"} SUITES_FILE = "test/e2e/suite-components.json" # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. NO_E2E_FILES = { @@ -936,7 +937,8 @@ def e2e_row(suite, gate_on, picked, key, feed): for bucket in E2E_BUCKETS: name = "e2e api-auth + ratelimit tail" if bucket == "api-auth" else f"e2e {bucket}" picked = bucket in e2e_run or (bucket == "api-auth" and "ratelimit" in e2e_run) - e2e_row(f"{name} (amd64, arm64)", gate("e2e"), picked, bucket, e2e_feed()) + arches = "amd64" if bucket in E2E_AMD64_ONLY else "amd64, arm64" + e2e_row(f"{name} ({arches})", gate("e2e"), picked, bucket, e2e_feed()) e2e_row("e2e multicluster (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-multicluster"), "multicluster", e2e_feed()) e2e_row("e2e upgrade (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-upgrade"), "upgrade", diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index c8dd10dc1..4bc8d8d49 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -9,7 +9,7 @@ from unittest.mock import patch from subprocess import CompletedProcess -from ci_scope import (E2E_BUCKETS, FORCE_ALL_FILES, ROOT, affected_modules, bot_matrix, +from ci_scope import (E2E_AMD64_ONLY, E2E_BUCKETS, FORCE_ALL_FILES, ROOT, affected_modules, bot_matrix, bucket_lists, changed_paths, e2e_outputs, full_scope_reason, image_inputs, is_unmapped, load_suites, main, publish_paths, select_images, summarize, verify_suites, go_packages, web_tests, CI_FILE, ci_blocks, ci_jobs, @@ -347,10 +347,10 @@ def test_every_e2e_bucket_and_bot_test_has_a_row(self): """Check every e2e bucket and bot test has a selection row.""" rows, _ = self.web_only() suites = {row["suite"] for row in rows} - self.assertEqual(len(E2E_BUCKETS), 8) for bucket in E2E_BUCKETS: name = "e2e api-auth + ratelimit tail" if bucket == "api-auth" else f"e2e {bucket}" - self.assertIn(f"{name} (amd64, arm64)", suites) + arches = "amd64" if bucket in E2E_AMD64_ONLY else "amd64, arm64" + self.assertIn(f"{name} ({arches})", suites) for test in SUMMARY_BOTS: self.assertIn(f"e2e game bot / {test}", suites) @@ -408,6 +408,17 @@ def select(self, *paths): """Return the e2e outputs for a partial-scope change to paths.""" return e2e_outputs(ROOT, list(paths), None) + def test_audit_wake_hostport_has_a_run_pattern(self): + """Affected components and the owning test file select the audit bucket.""" + for path in ("operator/internal/controller/gameserver_sentinel.go", + "sentinel/main.go", + "gameproto/minecraft.go", + "test/e2e/hostport_wake_e2e_test.go"): + with self.subTest(path=path): + values = self.select(path) + self.assertIn("TestGameServer_HostportWakeStableEndpoint", + values["e2e-run"].get("wake-hostport", "")) + def test_agent_only_change_selects_agent_tests_and_skips_auth_and_telemetry(self): values = self.select("agent/internal/console/console.go") self.assertEqual(values["reason"], "") @@ -504,6 +515,20 @@ def test_repository_suite_map_is_valid(self): class E2ERunTests(unittest.TestCase): """The e2e-run regexes that replace buckets.sh regex in the e2e jobs (T016).""" + def test_bucket_registry_matches_workflow_matrices(self): + """Every matrix bucket emits a run pattern and reports its real architectures.""" + text = (ROOT / CI_FILE).read_text(encoding="utf-8") + owner, _ = ci_blocks(text) + matrices = {} + for number, line in enumerate(text.splitlines(), 1): + job = owner[number] + if job in {"e2e-go", "e2e-go-arm64"} and line.strip().startswith("bucket: ["): + self.assertNotIn(job, matrices) + matrices[job] = {name.strip() for name in line.split("[", 1)[1].split("]", 1)[0].split(",")} + self.assertEqual(set(matrices), {"e2e-go", "e2e-go-arm64"}) + self.assertEqual(set(E2E_BUCKETS), matrices["e2e-go"] | matrices["e2e-go-arm64"]) + self.assertEqual(E2E_AMD64_ONLY, matrices["e2e-go"] - matrices["e2e-go-arm64"]) + def test_full_scope_regex_equals_buckets_sh_regex(self): values = e2e_outputs(ROOT, None, "master push") self.assertEqual(values["reason"], "master push") diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index 059470fd2..80079e373 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -263,6 +263,34 @@ "defaultMessage": "The state of the server is not known yet.", "description": "Healthy condition before the operator has observed the server." }, + "condition.hostportwakeready.endpointnodeunavailable": { + "defaultMessage": "The Hostport endpoint node is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", + "description": "Hostport endpoint handoff status: EndpointNodeUnavailable." + }, + "condition.hostportwakeready.endpointnodeunknown": { + "defaultMessage": "No game endpoint node has been recorded; start the game before enabling sleep.", + "description": "Hostport endpoint handoff status: EndpointNodeUnknown." + }, + "condition.hostportwakeready.endpointunschedulable": { + "defaultMessage": "The Hostport listener cannot schedule on its endpoint node. Inspect the Pod scheduling events for details.", + "description": "Hostport endpoint handoff status: EndpointUnschedulable." + }, + "condition.hostportwakeready.gamedraining": { + "defaultMessage": "Waiting for the game Pod to disappear before acquiring its Hostport.", + "description": "Hostport endpoint handoff status: GameDraining." + }, + "condition.hostportwakeready.listenerpending": { + "defaultMessage": "Waiting for a Ready listener on the Hostport endpoint node.", + "description": "Hostport endpoint handoff status: ListenerPending." + }, + "condition.hostportwakeready.listenerready": { + "defaultMessage": "The listener is Ready on the Hostport endpoint node.", + "description": "Hostport endpoint handoff status: ListenerReady." + }, + "condition.hostportwakeready.sentineldraining": { + "defaultMessage": "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.", + "description": "Hostport endpoint handoff status: SentinelDraining." + }, "condition.idlescheduleinvalid.wakewindowunparseable": { "defaultMessage": "A wake window in the idle schedule cannot be parsed: {details}", "description": "Condition when spec.idle has an invalid wake window. details is the English parse error." @@ -423,10 +451,26 @@ "defaultMessage": "The module content does not match its pinned digest.", "description": "Module condition when the digest differs from the pin." }, + "condition.ready.endpointnodeunavailable": { + "defaultMessage": "The Hostport endpoint node is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", + "description": "Hostport endpoint handoff status: EndpointNodeUnavailable." + }, + "condition.ready.endpointnodeunknown": { + "defaultMessage": "No game endpoint node has been recorded; start the game before enabling sleep.", + "description": "Hostport endpoint handoff status: EndpointNodeUnknown." + }, + "condition.ready.endpointunschedulable": { + "defaultMessage": "The Hostport listener cannot schedule on its endpoint node. Inspect the Pod scheduling events for details.", + "description": "Hostport endpoint handoff status: EndpointUnschedulable." + }, "condition.ready.failed": { "defaultMessage": "{detail}", "description": "Ready condition when the server cannot start. detail is the English reason from the operator, for example a missing template or an invalid configuration." }, + "condition.ready.gamedraining": { + "defaultMessage": "Waiting for the game Pod to disappear before acquiring its Hostport.", + "description": "Hostport endpoint handoff status: GameDraining." + }, "condition.ready.idleasleep": { "defaultMessage": "The server is asleep because no players are online.", "description": "Ready condition for a server put to sleep by idle auto-sleep." @@ -443,6 +487,14 @@ "defaultMessage": "The module cannot be removed while game servers use its template.", "description": "Ready condition when deletion is blocked." }, + "condition.ready.listenerpending": { + "defaultMessage": "Waiting for a Ready listener on the Hostport endpoint node.", + "description": "Hostport endpoint handoff status: ListenerPending." + }, + "condition.ready.listenerready": { + "defaultMessage": "The listener is Ready on the Hostport endpoint node.", + "description": "Hostport endpoint handoff status: ListenerReady." + }, "condition.ready.noversionavailable": { "defaultMessage": "The module source has no available versions.", "description": "Module condition when no version can be chosen." @@ -463,6 +515,10 @@ "defaultMessage": "The server is running and the agent is reporting heartbeats.", "description": "Ready condition while the game server is up." }, + "condition.ready.sentineldraining": { + "defaultMessage": "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.", + "description": "Hostport endpoint handoff status: SentinelDraining." + }, "condition.ready.servingstalecatalog": { "defaultMessage": "Serving the cached catalog. The last index attempt failed.", "description": "Ready condition while an older catalog stays available." diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index e9d3d9337..666956650 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -26,6 +26,12 @@ rules: - list - patch - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get - apiGroups: - "" resources: diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index e644cf041..0fb74217b 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -46,6 +46,8 @@ type row struct { // guardCalls maps a helper that takes a condition reason as a string literal // to the index of that argument. var guardCalls = map[string]int{ + "hostportCondition": 1, + "hostportPending": 0, "markUnhealthy": 2, "markFailed": 2, "markPending": 2, @@ -120,6 +122,15 @@ var rows = []row{ {types: []string{"AddressAssignment"}, reason: "AssignmentPending", english: "Requested $REQ: the address manager has not assigned an address yet.", description: "AddressAssignment condition while the address manager works."}, {types: []string{"AddressAssignment"}, reason: "Assigned", english: "{assignedFrom, select, pool {Address {address} assigned from pool ''{pool}''.} other {Address {address} assigned.}}", description: "AddressAssignment condition once an address is assigned. assignedFrom is pool or default."}, + // Hostport endpoint handoff; Ready can mirror the same reason. + {types: []string{"HostportWakeReady", "Ready"}, reason: "EndpointNodeUnknown", english: "No game endpoint node has been recorded; start the game before enabling sleep.", description: "Hostport endpoint handoff status: EndpointNodeUnknown."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "EndpointNodeUnavailable", english: "The Hostport endpoint node is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", description: "Hostport endpoint handoff status: EndpointNodeUnavailable."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "GameDraining", english: "Waiting for the game Pod to disappear before acquiring its Hostport.", description: "Hostport endpoint handoff status: GameDraining."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "SentinelDraining", english: "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.", description: "Hostport endpoint handoff status: SentinelDraining."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "EndpointUnschedulable", english: "The Hostport listener cannot schedule on its endpoint node. Inspect the Pod scheduling events for details.", description: "Hostport endpoint handoff status: EndpointUnschedulable."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "ListenerReady", english: "The listener is Ready on the Hostport endpoint node.", description: "Hostport endpoint handoff status: ListenerReady."}, + {types: []string{"HostportWakeReady", "Ready"}, reason: "ListenerPending", english: "Waiting for a Ready listener on the Hostport endpoint node.", description: "Hostport endpoint handoff status: ListenerPending."}, + // GameServer: tunnel and other {types: []string{"TunnelReady"}, reason: "InvalidConfig", english: "{provider, select, none {The tunnel is enabled but the server has no tunnel configuration.} other {The ''{provider}'' tunnel provider is selected but its configuration is missing.}}", description: "TunnelReady condition for an incomplete tunnel setup. provider is the selected provider, or none."}, {types: []string{"TunnelReady"}, reason: "DeploymentNotReady", english: "Waiting for the tunnel deployment to become ready.", description: "TunnelReady condition while the tunnel pod is not ready."}, diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 67e8be4e3..5e5906e38 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -386,6 +386,9 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) logger.Error(err, "compute desired replicas") return ctrl.Result{}, err } + if plan.blockGame { + replicas = 0 + } if err := r.reconcileStatefulSet(ctx, &gs, &tmpl, ver, mc, replicas, &wipeState); err != nil { if apierrors.IsConflict(err) { return ctrl.Result{Requeue: true}, nil @@ -442,7 +445,7 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } } - requeue, err := r.reconcileStatus(ctx, &gs, idle, idleStatus, idleScheduleErr, tunnelPlan, &tmpl, svcEvents, conflictingServer) + requeue, err := r.reconcileStatus(ctx, &gs, idle, idleStatus, idleScheduleErr, tunnelPlan, &tmpl, svcEvents, conflictingServer, plan.hostportCondition) if err != nil { return ctrl.Result{}, err } @@ -452,7 +455,7 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) // or nothing fires the transition until an unrelated event happens to // wake the reconciler. hints := []time.Duration{stopRequeue, idleRequeue} - if plan.wantSentinel { + if plan.wantSentinel || plan.hostportCondition != nil { // Bounded backstop for planSentinel's wait on the sentinel or the // game pod. Both are owned (Owns(&appsv1.Deployment{}), // Owns(&appsv1.StatefulSet{})) so a readiness transition normally @@ -1592,7 +1595,22 @@ func (r *GameServerReconciler) reconcileStatefulSet( // a removed scheduling pin active on the StatefulSet. ss.Spec.Template.Spec.NodeSelector = gs.Spec.NodeSelector ss.Spec.Template.Spec.Tolerations = gs.Spec.Tolerations - ss.Spec.Template.Spec.Affinity = gs.Spec.Affinity + // Do not change a live Pod's template merely because its node was + // recorded while a graceful stop is still saving the game. Apply the + // endpoint pin at zero replicas, then retain it on every future wake. + if actualReplicas == 0 || ss.ResourceVersion == "" || ss.Spec.Template.Annotations[hostportNodeAnnotation] != "" { + ss.Spec.Template.Spec.Affinity = hostportAffinity(gs) + if gs.Spec.Networking.Expose == "Hostport" && gs.Annotations[hostportNodeAnnotation] != "" { + if ss.Spec.Template.Annotations == nil { + ss.Spec.Template.Annotations = map[string]string{} + } + ss.Spec.Template.Annotations[hostportNodeAnnotation] = gs.Annotations[hostportNodeAnnotation] + } else { + delete(ss.Spec.Template.Annotations, hostportNodeAnnotation) + } + } else { + ss.Spec.Template.Spec.Affinity = gs.Spec.Affinity.DeepCopy() + } // Same unconditional-assign reasoning: removing template.spec.security // must clear a previously-set pod-level FSGroup, not leave it pinned. ss.Spec.Template.Spec.SecurityContext = gamePodSecurityContext(tmpl) diff --git a/operator/internal/controller/gameserver_hostport.go b/operator/internal/controller/gameserver_hostport.go new file mode 100644 index 000000000..0ff7464a8 --- /dev/null +++ b/operator/internal/controller/gameserver_hostport.go @@ -0,0 +1,187 @@ +package controller + +import ( + "context" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + // Unlike nodeAnnotation (the current Pod's dashboard location), these + // survive sleep, Pod deletion and operator restarts. Reusing a node name + // after replacement must not silently redirect the advertised endpoint. + hostportNodeAnnotation = "gameplane.local/hostport-node" + hostportNodeUIDAnnotation = "gameplane.local/hostport-node-uid" + hostportWakeCondition = "HostportWakeReady" +) + +// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get + +func (r *GameServerReconciler) hostportReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +func hostportCondition(gs *gameplanev1alpha1.GameServer, reason, message string, ready bool) *metav1.Condition { + status := metav1.ConditionFalse + if ready { + status = metav1.ConditionTrue + } + return &metav1.Condition{Type: hostportWakeCondition, Status: status, Reason: reason, Message: message, ObservedGeneration: gs.Generation} +} + +// planHostportSentinel never transfers a port based on replica status or a +// cached Pod absence. Producers must first stop, then their actual Pods must +// disappear. All waits return a plan so the remaining reconciliation proceeds. +func (r *GameServerReconciler) planHostportSentinel(ctx context.Context, gs *gameplanev1alpha1.GameServer, idle idleState) (sentinelPlan, error) { + reader := r.hostportReader() + var gamePods, sentinelPods corev1.PodList + for _, entry := range []struct { + name string + pods *corev1.PodList + }{{"gameplane-game", &gamePods}, {sentinelValue, &sentinelPods}} { + if err := reader.List(ctx, entry.pods, client.InNamespace(gs.Namespace), client.MatchingLabels{"app.kubernetes.io/name": entry.name, "app.kubernetes.io/instance": gs.Name}); err != nil { + return sentinelPlan{}, err + } + } + plan := sentinelPlan{} + hostportPending := func(reason, message string) { + plan.hostportCondition = hostportCondition(gs, reason, message, false) + } + nodeName := gs.Annotations[hostportNodeAnnotation] + // Record the endpoint before the first sleep scales down the game. Doing + // this during sleep also avoids rolling an already-running game merely to + // add the new affinity after an operator upgrade. + if nodeName == "" && idle == idleAsleep { + for _, pod := range gamePods.Items { + if pod.Name == gs.Name+"-0" && pod.Spec.NodeName != "" { + nodeName = pod.Spec.NodeName + break + } + } + if nodeName != "" { + var node corev1.Node + if err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node); err != nil { + if !apierrors.IsNotFound(err) { + return sentinelPlan{}, err + } + } + base := gs.DeepCopy() + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[hostportNodeAnnotation] = nodeName + gs.Annotations[hostportNodeUIDAnnotation] = string(node.UID) + if err := r.Patch(ctx, gs, client.MergeFrom(base)); err != nil { + return sentinelPlan{}, err + } + } + } + if nodeName == "" && idle == idleAsleep { + hostportPending("EndpointNodeUnknown", "No game endpoint node has been recorded; start the game before enabling sleep.") + return plan, nil + } + if nodeName != "" { + var node corev1.Node + err := reader.Get(ctx, client.ObjectKey{Name: nodeName}, &node) + nodeMissing := apierrors.IsNotFound(err) + if err != nil && !nodeMissing { + return plan, err + } + ready := false + for _, condition := range node.Status.Conditions { + if condition.Type == corev1.NodeReady { + ready = condition.Status == corev1.ConditionTrue + } + } + if nodeMissing || !node.DeletionTimestamp.IsZero() || !ready || string(node.UID) != gs.Annotations[hostportNodeUIDAnnotation] { + plan.blockGame = len(gamePods.Items) == 0 + hostportPending("EndpointNodeUnavailable", fmt.Sprintf("Hostport endpoint node %q is missing, replaced or not Ready; restore that node or explicitly migrate the endpoint.", nodeName)) + return plan, nil + } + } + if idle == idleAsleep { + var ss appsv1.StatefulSet + err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if err != nil && !apierrors.IsNotFound(err) { + return plan, err + } + if len(gamePods.Items) != 0 || (err == nil && (ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation || ss.Status.Replicas != 0)) { + hostportPending("GameDraining", "Waiting for the game Pod to disappear before acquiring its Hostport.") + return plan, nil + } + plan.wantSentinel = true + plan.hostportCondition = hostportListenerCondition(gs, sentinelPods.Items, nodeName) + plan.routeToSentinel = plan.hostportCondition.Status == metav1.ConditionTrue + return plan, nil + } + var dep appsv1.Deployment + err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-waker"}, &dep) + if err != nil && !apierrors.IsNotFound(err) { + return plan, err + } + if err == nil || len(sentinelPods.Items) != 0 { + plan.blockGame = true + hostportPending("SentinelDraining", "Waiting for the sentinel Deployment and Pods to disappear before scheduling the game on its endpoint node.") + return plan, nil + } + plan.hostportCondition = hostportListenerCondition(gs, gamePods.Items, nodeName) + return plan, nil +} + +func hostportListenerCondition(gs *gameplanev1alpha1.GameServer, pods []corev1.Pod, nodeName string) *metav1.Condition { + for _, pod := range pods { + if !pod.DeletionTimestamp.IsZero() { + continue + } + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && condition.Reason == corev1.PodReasonUnschedulable { + return hostportCondition(gs, "EndpointUnschedulable", fmt.Sprintf("Hostport listener cannot schedule on endpoint node %q: %s", nodeName, condition.Message), false) + } + if condition.Type == corev1.PodReady && condition.Status == corev1.ConditionTrue && (nodeName == "" || pod.Spec.NodeName == nodeName) { + return hostportCondition(gs, "ListenerReady", "The listener is Ready on the Hostport endpoint node.", true) + } + } + } + return hostportCondition(gs, "ListenerPending", fmt.Sprintf("Waiting for a Ready listener on endpoint node %q.", nodeName), false) +} + +// hostportAffinity ANDs node identity into every OR alternative, retaining +// user selectors, pod affinity and anti-affinity. Empty selectors still match +// no nodes; setting PodSpec.NodeName would bypass the scheduler's port checks. +func hostportAffinity(gs *gameplanev1alpha1.GameServer) *corev1.Affinity { + affinity := gs.Spec.Affinity.DeepCopy() + node := gs.Annotations[hostportNodeAnnotation] + if gs.Spec.Networking.Expose != "Hostport" || node == "" { + return affinity + } + if affinity == nil { + affinity = &corev1.Affinity{} + } + if affinity.NodeAffinity == nil { + affinity.NodeAffinity = &corev1.NodeAffinity{} + } + if affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{}}} + // This newly created term intentionally matches the endpoint node. + affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields = []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{node}}} + return affinity + } + for i := range affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + term := &affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[i] + if len(term.MatchExpressions) == 0 && len(term.MatchFields) == 0 { + continue + } + term.MatchFields = append(term.MatchFields, corev1.NodeSelectorRequirement{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{node}}) + } + return affinity +} diff --git a/operator/internal/controller/gameserver_hostport_test.go b/operator/internal/controller/gameserver_hostport_test.go new file mode 100644 index 000000000..fc19f5116 --- /dev/null +++ b/operator/internal/controller/gameserver_hostport_test.go @@ -0,0 +1,132 @@ +package controller + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func hostportFixture(t *testing.T) (*GameServerReconciler, *gameplanev1alpha1.GameServer, *gameplanev1alpha1.GameTemplate) { + t.Helper() + s := wipeScheme(t) + if err := appsv1.AddToScheme(s); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "game-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Expose: "Hostport"}, Idle: &gameplanev1alpha1.IdleSpec{Enabled: true, WakeOnConnect: true}}} + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a", UID: "node-uid"}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}} + zero := int32(0) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace}, Spec: appsv1.StatefulSetSpec{Replicas: &zero}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, node, ss).Build() + return &GameServerReconciler{Client: c, APIReader: c, Scheme: s}, gs, singlePortTemplate(true, "generic") +} + +func TestHostportSleepCapturesNodeAndWaitsForActualPodDeletion(t *testing.T) { + r, gs, tmpl := hostportFixture(t) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "game-0", Namespace: gs.Namespace, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-game", "app.kubernetes.io/instance": gs.Name}, Finalizers: []string{"test/drain"}}, Spec: corev1.PodSpec{NodeName: "node-a"}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAsleep) + if err != nil { + t.Fatal(err) + } + if plan.wantSentinel || plan.hostportCondition.Reason != "GameDraining" { + t.Fatalf("plan before actual deletion: %+v", plan) + } + var saved gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &saved); err != nil { + t.Fatal(err) + } + if saved.Annotations[hostportNodeAnnotation] != "node-a" || saved.Annotations[hostportNodeUIDAnnotation] != "node-uid" { + t.Fatalf("durable endpoint not saved: %v", saved.Annotations) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(pod), pod); err != nil { + t.Fatal(err) + } + pod.Finalizers = nil + if err := r.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + plan, err = r.planSentinel(t.Context(), &saved, tmpl, idleAsleep) + if err != nil || !plan.wantSentinel { + t.Fatalf("plan after actual deletion: %+v, %v", plan, err) + } +} + +func TestHostportWakeUsesUncachedPodsAndWaitsForProducerDeletion(t *testing.T) { + r, gs, tmpl := hostportFixture(t) + gs.Annotations = map[string]string{hostportNodeAnnotation: "node-a", hostportNodeUIDAnnotation: "node-uid"} + dep := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-waker", Namespace: gs.Namespace}} + if err := r.Create(t.Context(), dep); err != nil { + t.Fatal(err) + } + // A cached view with no Pods must not authorize transfer while the live + // API still has a terminating listener, even after its Deployment is gone. + live := fake.NewClientBuilder().WithScheme(r.Scheme).WithObjects(dep, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "old-waker", Namespace: gs.Namespace, Labels: map[string]string{sentinelLabel: sentinelValue, "app.kubernetes.io/instance": gs.Name}}}, &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a", UID: "node-uid"}, Status: corev1.NodeStatus{Conditions: []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}}}}).Build() + r.APIReader = live + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAwake) + if err != nil || !plan.blockGame || plan.wantSentinel { + t.Fatalf("producer still live: %+v, %v", plan, err) + } + if err := live.Delete(t.Context(), dep); err != nil { + t.Fatal(err) + } + plan, err = r.planSentinel(t.Context(), gs, tmpl, idleAwake) + if err != nil || !plan.blockGame { + t.Fatalf("live Pod bypassed by cache: %+v, %v", plan, err) + } + if err := live.Delete(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "old-waker", Namespace: gs.Namespace}}); err != nil { + t.Fatal(err) + } + plan, err = r.planSentinel(t.Context(), gs, tmpl, idleAwake) + if err != nil || plan.blockGame { + t.Fatalf("drained listener still blocks: %+v, %v", plan, err) + } +} + +func TestHostportNodeLossAndUnschedulableListenerStayUnavailable(t *testing.T) { + r, gs, tmpl := hostportFixture(t) + gs.Annotations = map[string]string{hostportNodeAnnotation: "node-a", hostportNodeUIDAnnotation: "node-uid"} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "waker", Namespace: gs.Namespace, Labels: map[string]string{sentinelLabel: sentinelValue, "app.kubernetes.io/instance": gs.Name}}, Status: corev1.PodStatus{Conditions: []corev1.PodCondition{{Type: corev1.PodScheduled, Status: corev1.ConditionFalse, Reason: "Unschedulable", Message: "Insufficient memory"}}}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + plan, err := r.planSentinel(t.Context(), gs, tmpl, idleAsleep) + if err != nil || plan.routeToSentinel || plan.hostportCondition.Reason != "EndpointUnschedulable" { + t.Fatalf("full node: %+v, %v", plan, err) + } + if err := r.Delete(t.Context(), &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "node-a"}}); err != nil { + t.Fatal(err) + } + plan, err = r.planSentinel(t.Context(), gs, tmpl, idleAsleep) + if err != nil || plan.wantSentinel || plan.hostportCondition.Reason != "EndpointNodeUnavailable" { + t.Fatalf("lost node: %+v, %v", plan, err) + } +} + +func TestHostportAffinityConjoinsEveryAlternativeAndPreservesOtherConstraints(t *testing.T) { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{hostportNodeAnnotation: "node-a"}}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Expose: "Hostport"}, Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "pool", Operator: corev1.NodeSelectorOpIn, Values: []string{"a"}}}}, {MatchExpressions: []corev1.NodeSelectorRequirement{{Key: "pool", Operator: corev1.NodeSelectorOpIn, Values: []string{"b"}}}}}}}, PodAntiAffinity: &corev1.PodAntiAffinity{}}}} + got := hostportAffinity(gs) + for _, term := range got.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + if len(term.MatchFields) != 1 || term.MatchFields[0].Key != "metadata.name" || term.MatchFields[0].Values[0] != "node-a" { + t.Fatalf("unpinned alternative: %+v", term) + } + } + if got.PodAntiAffinity == nil || len(gs.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields) != 0 { + t.Fatal("lost constraints or mutated CR") + } + // An explicitly empty selector matches no nodes; pinning must not widen it. + gs.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = nil + if len(hostportAffinity(gs).NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms) != 0 { + t.Fatal("empty selector widened") + } +} diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index b2c8a9edc..bb145738e 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -102,8 +102,10 @@ func wakeOnConnectEligible(gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1al // whether it should exist, and whether the game Service should route to it // instead of the game pod. type sentinelPlan struct { - wantSentinel bool - routeToSentinel bool + wantSentinel bool + routeToSentinel bool + blockGame bool + hostportCondition *metav1.Condition } // planSentinel is the single source of truth for the sentinel/game-Service @@ -132,9 +134,10 @@ type sentinelPlan struct { // terminationGracePeriodSeconds plus the sentinel's shutdown drain // (sentinelShutdownDrainTimeout) keep any handed-through session alive // until it ends on its own; only new connections stop reaching it. -// - Hostport: the sentinel and the game pod bind the exact same host -// port, so the sentinel cannot outlive the wake decision — the game -// pod cannot even schedule until the sentinel releases that port. The +// - Hostport: planHostportSentinel records the endpoint node and waits +// for actual game Pod deletion before creating a pinned sentinel. On +// wake it waits for foreground sentinel deletion and actual Pod drain +// before permitting the game to start on that same node. The // sentinel is expected to bounce any clients it is holding with a // protocol-native "waking up, try again shortly" response before the // operator tears it down here; players simply reconnect once the game @@ -142,6 +145,22 @@ type sentinelPlan struct { func (r *GameServerReconciler) planSentinel( ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate, idle idleState, ) (sentinelPlan, error) { + if gs.Spec.Networking.Expose == "Hostport" { + eligible := wakeOnConnectEligible(gs, tmpl) + if !eligible { + idle = idleAwake + } + plan, err := r.planHostportSentinel(ctx, gs, idle) + if err != nil { + return plan, err + } + // A Hostport server that has never used wake keeps its existing + // status behavior. Still drain a leftover sentinel when disabling it. + if !eligible && gs.Annotations[hostportNodeAnnotation] == "" && !plan.blockGame { + return sentinelPlan{}, nil + } + return plan, nil + } if !wakeOnConnectEligible(gs, tmpl) { return sentinelPlan{}, nil } @@ -157,10 +176,6 @@ func (r *GameServerReconciler) planSentinel( // idle == idleAwake: either a wake was just decided, or the server was // never asleep to begin with — sentinelExists is false in the latter // case, so we return immediately without the extra StatefulSet Get. - if gs.Spec.Networking.Expose == "Hostport" { - // See the Hostport bullet above: no hold-open window, ever. - return sentinelPlan{}, nil - } exists, err := r.sentinelExists(ctx, gs.Namespace, gs.Name) if err != nil || !exists { return sentinelPlan{}, err @@ -206,7 +221,10 @@ func (r *GameServerReconciler) reconcileSentinel( want bool, ) error { if !want { - return r.deleteSentinel(ctx, gs.Namespace, gs.Name) + return r.deleteSentinel(ctx, gs.Namespace, gs.Name, gs.Spec.Networking.Expose == "Hostport") + } + if gs.Spec.Networking.Expose == "Hostport" && gs.Annotations[hostportNodeAnnotation] == "" { + return fmt.Errorf("hostport sentinel requires a persisted endpoint node") } // Create or update the sentinel Deployment. @@ -277,6 +295,16 @@ func (r *GameServerReconciler) reconcileSentinel( dep.Spec.Template.Spec.TerminationGracePeriodSeconds = &graceSeconds dep.Spec.Template.Spec.ServiceAccountName = sentinelSAName(gs) + if hostPort { + dep.Spec.Template.Spec.NodeSelector = gs.Spec.NodeSelector + dep.Spec.Template.Spec.Tolerations = gs.Spec.Tolerations + dep.Spec.Template.Spec.Affinity = hostportAffinity(gs) + dep.Spec.Strategy = appsv1.DeploymentStrategy{Type: appsv1.RecreateDeploymentStrategyType} + } else { + dep.Spec.Template.Spec.NodeSelector = nil + dep.Spec.Template.Spec.Tolerations = nil + dep.Spec.Template.Spec.Affinity = nil + } dep.Spec.Template.Spec.Containers = []corev1.Container{{ Name: "sentinel", Image: image, @@ -406,12 +434,15 @@ func (r *GameServerReconciler) sentinelIsReady(ctx context.Context, namespace, g } // deleteSentinel removes the sentinel Deployment if it exists. -func (r *GameServerReconciler) deleteSentinel(ctx context.Context, namespace, gsName string) error { +func (r *GameServerReconciler) deleteSentinel(ctx context.Context, namespace, gsName string, foreground ...bool) error { dep, exists, err := r.getSentinel(ctx, namespace, gsName) if err != nil || !exists { return err } policy := metav1.DeletePropagationBackground + if len(foreground) != 0 && foreground[0] { + policy = metav1.DeletePropagationForeground + } return client.IgnoreNotFound(r.Delete(ctx, dep, &client.DeleteOptions{PropagationPolicy: &policy})) } diff --git a/operator/internal/controller/gameserver_sentinel_envtest_test.go b/operator/internal/controller/gameserver_sentinel_envtest_test.go index e805d58c8..f756311b6 100644 --- a/operator/internal/controller/gameserver_sentinel_envtest_test.go +++ b/operator/internal/controller/gameserver_sentinel_envtest_test.go @@ -12,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -182,7 +183,7 @@ func TestSentinelReconciliation(t *testing.T) { deleteCleanup(t, tmpl) gs := &gameplanev1alpha1.GameServer{ - ObjectMeta: metav1.ObjectMeta{Name: "test-sentinel-hostport", Namespace: ns}, + ObjectMeta: metav1.ObjectMeta{Name: "test-sentinel-hostport", Namespace: ns, Annotations: map[string]string{hostportNodeAnnotation: "node-a"}}, Spec: gameplanev1alpha1.GameServerSpec{ TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: tmpl.Name}, Idle: armedIdleSpec(), @@ -550,8 +551,40 @@ func TestGameServer_WakeOnConnectHostportNoHoldOpen(t *testing.T) { return ssReplicas(t, ns, "wakehp") == 1, "statefulset not scaled up yet" }) setSSStatus(t, ns, "wakehp", 1, 1) + node := &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: uniqueName("hostport-node")}} + if err := k8sClient.Create(ctx, node); err != nil { + t.Fatal(err) + } + deleteCleanup(t, node) + node.Status.Conditions = []corev1.NodeCondition{{Type: corev1.NodeReady, Status: corev1.ConditionTrue}} + if err := k8sClient.Status().Update(ctx, node); err != nil { + t.Fatal(err) + } + gamePod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "wakehp-0", Namespace: ns, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-game", "app.kubernetes.io/instance": gs.Name}}, Spec: corev1.PodSpec{NodeName: node.Name, Containers: []corev1.Container{{Name: "game", Image: "busybox"}}}} + if err := k8sClient.Create(ctx, gamePod); err != nil { + t.Fatal(err) + } setAsleep(t, ns, "wakehp") + eventually(t, func() (bool, string) { return ssReplicas(t, ns, gs.Name) == 0, "game not scaled down" }) + consistently(t, time.Second, func() (bool, string) { + return !deploymentExists(t, ns, "wakehp-waker"), "sentinel overlaps the live game Pod" + }) + zero := int64(0) + if err := k8sClient.Delete(ctx, gamePod, &client.DeleteOptions{GracePeriodSeconds: &zero}); err != nil { + t.Fatal(err) + } + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var ss appsv1.StatefulSet + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: gs.Name}, &ss); err != nil { + return err + } + ss.Status.Replicas, ss.Status.ReadyReplicas = 0, 0 + ss.Status.ObservedGeneration = ss.Generation + return k8sClient.Status().Update(ctx, &ss) + }); err != nil { + t.Fatal(err) + } eventually(t, func() (bool, string) { return deploymentExists(t, ns, "wakehp-waker"), "sentinel deployment not created" }) @@ -565,6 +598,23 @@ func TestGameServer_WakeOnConnectHostportNoHoldOpen(t *testing.T) { clearAsleep(t, ns, "wakehp") eventually(t, func() (bool, string) { - return !deploymentExists(t, ns, "wakehp-waker"), "sentinel not deleted immediately on Hostport wake" + var dep appsv1.Deployment + err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "wakehp-waker"}, &dep) + return err == nil && !dep.DeletionTimestamp.IsZero(), "sentinel foreground deletion not requested" + }) + consistently(t, time.Second, func() (bool, string) { + return ssReplicas(t, ns, gs.Name) == 0, "game restarted before sentinel producer disappeared" + }) + // envtest has no garbage collector: finish foreground deletion explicitly. + var dep appsv1.Deployment + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "wakehp-waker"}, &dep); err != nil { + t.Fatal(err) + } + dep.Finalizers = nil + if err := k8sClient.Update(ctx, &dep); err != nil { + t.Fatal(err) + } + eventually(t, func() (bool, string) { + return ssReplicas(t, ns, gs.Name) == 1, "game did not resume after sentinel deletion" }) } diff --git a/operator/internal/controller/gameserver_status.go b/operator/internal/controller/gameserver_status.go index 54393de0b..4a470c122 100644 --- a/operator/internal/controller/gameserver_status.go +++ b/operator/internal/controller/gameserver_status.go @@ -43,6 +43,7 @@ func (r *GameServerReconciler) reconcileStatus( ctx context.Context, gs *gameplanev1alpha1.GameServer, idle idleState, idleStatus *gameplanev1alpha1.IdleStatus, idleScheduleErr error, tunnelPlan tunnelPlan, tmpl *gameplanev1alpha1.GameTemplate, svcEvents []corev1.Event, conflictingServer string, + hostportConditions ...*metav1.Condition, ) (time.Duration, error) { // base captures the object as fetched so we can issue a JSON merge // patch of only the fields this reconciler owns. The agent sidecar @@ -194,6 +195,16 @@ func (r *GameServerReconciler) reconcileStatus( // Compute tunnel conditions. gs.Status.Conditions = computeTunnelConditions(gs, tunnelPlan, tunnelDep) + if len(hostportConditions) != 0 && hostportConditions[0] != nil { + condition := *hostportConditions[0] + gs.Status.Conditions = upsertCondition(gs.Status.Conditions, condition) + if condition.Status != metav1.ConditionTrue { + condition.Type = "Ready" + gs.Status.Conditions = upsertCondition(gs.Status.Conditions, condition) + } + } else { + gs.Status.Conditions = removeCondition(gs.Status.Conditions, hostportWakeCondition) + } // Validate and merge playit tunnel endpoints if present. var playitErr string diff --git a/operator/specs.md b/operator/specs.md index c7aea8225..0fc1414e4 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -30,6 +30,24 @@ start/stop and expiry remain reconciler responsibilities; the gateway cannot bypass them or operate an arbitrary pod. Existing game resources and storage are not recreated to enable remote parity. +### Hostport wake handoff + +Hostport wake-on-connect keeps a durable endpoint node name and Node UID in +`gameplane.local/hostport-node` / `gameplane.local/hostport-node-uid`, captured from +the live game Pod before the first sleep. The current-node dashboard annotation +is independent. Sleep waits for the zero-replica StatefulSet generation to be +observed and for actual game Pod disappearance through the uncached API reader. +Wake foreground-deletes the sentinel and waits for its Deployment and all Pods +to disappear before scheduling the game. Both successor templates use required +node affinity for the recorded endpoint, combined with the game's scheduling +constraints; the sentinel carries its tolerations. Node gets are read-only RBAC. +`HostportWakeReady` reports listener readiness, each drain and unavailable or +unschedulable endpoints without blocking unrelated reconciliation. There is no +automatic endpoint migration. See the updated +[July handoff design](../docs/superpowers/specs/2026-07-24-wake-on-connect-design.md#sequencing-and-port-handover) +for recovery/migration steps. Service-backed wake retains its established-session +drain behavior. Auxiliary resource defaults are defined separately. + ## Non-goals / boundaries The operator **does not** handle: diff --git a/sentinel/main.go b/sentinel/main.go index edc563b4d..2eaec4afa 100644 --- a/sentinel/main.go +++ b/sentinel/main.go @@ -72,7 +72,8 @@ const ( // defaultPollInterval is how often waitForUpstream retries dialing the // game-direct Service while holding a connection open. - defaultPollInterval = 250 * time.Millisecond + defaultPollInterval = 250 * time.Millisecond + defaultWakePatchTimeout = 5 * time.Second // defaultShutdownDrainTimeout bounds how long the sentinel keeps // in-flight sessions alive after it is told to shut down (see @@ -354,44 +355,68 @@ type wakeRequester interface { // waker patches the GameServer's idle-wake-requested annotation to signal // the operator that this server should wake. // -// It coalesces bursts: while a patch is in flight, or within minInterval of -// the last one landing, RequestWake is a cheap no-op instead of issuing -// another apiserver call. This is what keeps a port scan, or a pile of +// It coalesces bursts: callers share the result of an in-flight patch, and +// successful patches suppress another call for minInterval. This keeps a port scan, or a pile of // players reconnecting at once, from hammering the apiserver with patches — // the operator only needs to see the annotation change once to act on it. type waker struct { - res dynamic.ResourceInterface - name string - minInterval time.Duration + res dynamic.ResourceInterface + name string + minInterval time.Duration + patchTimeout time.Duration mu sync.Mutex - inFlight bool + inFlight *wakeAttempt lastPatch time.Time } +type wakeAttempt struct { + done chan struct{} + err error +} + func newWaker(res dynamic.ResourceInterface, name string, minInterval time.Duration) *waker { - return &waker{res: res, name: name, minInterval: minInterval} + return &waker{res: res, name: name, minInterval: minInterval, patchTimeout: defaultWakePatchTimeout} } // RequestWake stamps a fresh token onto the GameServer's // idle-wake-requested annotation via a JSON merge patch (rather than a full // object update), so any other annotations already on the object survive. -func (w *waker) RequestWake(ctx context.Context) error { +func (w *waker) RequestWake(ctx context.Context) (result error) { + if err := ctx.Err(); err != nil { + return err + } w.mu.Lock() now := time.Now() - if w.inFlight || (!w.lastPatch.IsZero() && now.Sub(w.lastPatch) < w.minInterval) { + if attempt := w.inFlight; attempt != nil { + w.mu.Unlock() + select { + case <-ctx.Done(): + return ctx.Err() + case <-attempt.done: + return attempt.err + } + } + if !w.lastPatch.IsZero() && now.Sub(w.lastPatch) < w.minInterval { w.mu.Unlock() return nil } - w.inFlight = true + attempt := &wakeAttempt{done: make(chan struct{})} + w.inFlight = attempt w.mu.Unlock() defer func() { w.mu.Lock() - w.inFlight = false - w.lastPatch = time.Now() + attempt.err = result + w.inFlight = nil + if result == nil { + w.lastPatch = time.Now() + } + close(attempt.done) w.mu.Unlock() }() + patchCtx, cancel := context.WithTimeout(ctx, w.patchTimeout) + defer cancel() patch, err := json.Marshal(map[string]any{ "metadata": map[string]any{ @@ -404,7 +429,7 @@ func (w *waker) RequestWake(ctx context.Context) error { return fmt.Errorf("encode wake patch: %w", err) } - if _, err := w.res.Patch(ctx, w.name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + if _, err := w.res.Patch(patchCtx, w.name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { return fmt.Errorf("patch %s annotation: %w", idleWakeRequestedAnnotation, err) } return nil @@ -714,17 +739,20 @@ type joinParams struct { // bidirectionally; on timeout, the connection is bounced (if a bounce // function is set) and closed. func handleJoin(ctx context.Context, conn net.Conn, br *bufio.Reader, w wakeRequester, p joinParams) { - if err := w.RequestWake(ctx); err != nil { + joinCtx, cancelJoin := context.WithTimeout(ctx, p.deadline) + defer cancelJoin() + if err := w.RequestWake(joinCtx); err != nil { log.Printf("request wake: %v", err) } - upstream, err := waitForUpstream(ctx, p.upstreamAddr, p.deadline, p.pollInterval) + upstream, err := waitForUpstream(joinCtx, p.upstreamAddr, p.deadline, p.pollInterval) if err != nil { if p.bounce != nil { p.bounce(conn) } return } + cancelJoin() defer func() { if err := upstream.Close(); err != nil && !errors.Is(err, net.ErrClosed) { log.Printf("close upstream connection: %v", err) @@ -842,6 +870,11 @@ func serveUDP(ctx context.Context, port PortConfig, w wakeRequester, cfg Config, } return } + defer func() { _ = pc.Close() }() + serveUDPPackets(ctx, pc, port, w, cfg, errCh) +} + +func serveUDPPackets(ctx context.Context, pc net.PacketConn, port PortConfig, w wakeRequester, cfg Config, errCh chan<- error) { // Belt-and-suspenders shutdown: Close() unblocks a ReadFrom that's // already parked waiting for a packet, and the read deadline below @@ -858,6 +891,11 @@ func serveUDP(ctx context.Context, port PortConfig, w wakeRequester, cfg Config, sweepCtx, cancelSweep := context.WithCancel(ctx) defer cancelSweep() go heuristic.sweepLoop(sweepCtx, cfg.UDPWindow) + wakeWorker := newUDPWakeWorker(sweepCtx, w) + defer func() { + cancelSweep() + <-wakeWorker.done + }() buf := make([]byte, 2048) for { @@ -882,9 +920,7 @@ func serveUDP(ctx context.Context, port PortConfig, w wakeRequester, cfg Config, } if heuristic.shouldWake(addr.String(), time.Now()) { - if err := w.RequestWake(ctx); err != nil { - log.Printf("request wake: %v", err) - } + wakeWorker.request() } } } diff --git a/sentinel/specs.md b/sentinel/specs.md index e39166621..3d1a6305d 100644 --- a/sentinel/specs.md +++ b/sentinel/specs.md @@ -59,7 +59,7 @@ Single executable module; no subdirectories or packages. - `none`: Operator still advertises the port (container spec, Service), but the sentinel doesn't listen on it (port scan sink). - **Startup validation:** At startup, all configured `wakeProtocol` values (except `generic` and `none`) are validated against the gameproto registry. Unknown protocol names cause a fatal startup error listing available registered protocols. -- **`WAKE_DEADLINE`** (optional, default `25s`): How long to hold a join connection waiting for the game pod to come up. Must be less than the game client's own timeout (Minecraft ~30s, Terraria ~30s) or the client gives up first. +- **`WAKE_DEADLINE`** (optional, default `25s`): Total join budget, starting before the wake PATCH and including upstream dialing. The PATCH is bounded by the lesser of five seconds and the remaining join budget. Must be less than the game client's own timeout (Minecraft ~30s, Terraria ~30s) or the client gives up first. After a successful upstream connection, proxying uses the session/drain context and outlives this budget. - **`UDP_PACKET_THRESHOLD`** (optional, default `3`): Number of packets from one source within the window before triggering a wake. - **`UDP_PACKET_WINDOW`** (optional, default `10s`): Time window for counting UDP packets. - **`UDP_MAX_SOURCES`** (optional, default `4096`): Maximum number of distinct UDP sources (keyed by `ip:port`, not just IP) to track. Older sources are evicted when this limit is reached. @@ -246,7 +246,7 @@ Each accepted TCP connection runs as a *session* under its own context, separate 5. **No data duplication or loss.** The complete original stream is: `Consumed + remaining(bufio.Reader) + new-data-from-client`. Both classifiers and proxying preserve this boundary. -6. **Annotation patch coalesces bursts.** Bursts of players reconnecting at once don't hammer the apiserver; RequestWake is a no-op if a patch is already in flight or within `wakePatchInterval` of the last one. +6. **Annotation patch coalesces bursts.** Callers awaiting an in-flight PATCH observe that attempt's completion and error, unless their own context expires first. Only successful attempts start the `wakePatchInterval` cooldown; failures release the shared attempt so a later request can retry. One worker per UDP listener accepts a one-entry coalescing queue; packet reception and bounded source tracking continue while Kubernetes is stalled. Listener shutdown cancels and joins that worker. 7. **UDP sources are evicted fairly.** The least-recently-active source is evicted (LRU), not a random or oldest source, so brief bursts of scanners don't unfairly starve legitimate players trying to rejoin. @@ -293,6 +293,7 @@ Each accepted TCP connection runs as a *session* under its own context, separate - **Shutdown drain (F-179):** `TestRunDrainsProxiedSessionAfterContextCancel` (replaces `TestProxyBidirectionalStopsOnContextCancel`, which asserted the old cut-on-cancel behaviour) verifies that after cancel new connections are refused while a handed-through session keeps working in both directions and `run` returns only once it ends; `TestRunForceClosesSessionsAfterDrainTimeout` verifies the drain is bounded; `TestLoadConfigShutdownDrainTimeout` covers `SHUTDOWN_DRAIN_TIMEOUT` parsing. - **Listener failure (F-180):** `TestRunSurfacesUDPListenFailureWhileTCPHealthy` verifies a UDP bind failure is returned promptly while a TCP listener is still healthy. - **Waker (annotation patching):** `TestWakerRequestWakeSetsAnnotation`, `TestWakerRequestWakePreservesOtherAnnotations`, `TestWakerRequestWakeCoalescesBursts` verify merge patch isolation, burst coalescing. +- **Wake availability:** `wake_budget_test.go` covers a hanging HTTP API, shared failure and retry, the total join deadline, worker cancellation/coalescing, successful gameplay beyond the join deadline, and receiving a 101-datagram burst while wake is stalled. CI logs elapsed receive time; this fixture does not establish a production capacity ceiling. - **Registry dispatch:** `TestRegistryProtocolDispatchMinecraftJoinWakes`, `TestRegistryProtocolDispatchMinecraftStatusPingNoWake`, `TestRegistryProtocolDispatchTerrariaJoinWakes`, `TestRegistryProtocolDispatchUnknownNoWake`, `TestRegistryProtocolReplayConsumedBytes` verify registry lookup, Classifier.Classify invocation, protocol-agnostic join/status/unknown handling, and consumed-byte replay. - **Startup validation:** `TestParsePortsConfigRejectsUnknownProtocol` verifies that unknown protocol names cause a fatal startup error with a clear message listing available protocols. - **Generic handlers:** `TestHandleGenericWakesOnConnect`, `TestHandleGenericClosesSilentlyOnDeadline` verify generic TCP (always join), generic UDP bounce (no message). diff --git a/sentinel/udp_wake.go b/sentinel/udp_wake.go new file mode 100644 index 000000000..3b8a9b29a --- /dev/null +++ b/sentinel/udp_wake.go @@ -0,0 +1,41 @@ +package main + +import ( + "context" + "log" +) + +type udpWakeWorker struct { + requests chan struct{} + done chan struct{} +} + +func newUDPWakeWorker(ctx context.Context, w wakeRequester) *udpWakeWorker { + worker := &udpWakeWorker{requests: make(chan struct{}, 1), done: make(chan struct{})} + go func() { + defer close(worker.done) + for { + select { + case <-ctx.Done(): + return + case <-worker.requests: + if ctx.Err() != nil { + return + } + wakeCtx, cancel := context.WithTimeout(ctx, defaultWakePatchTimeout) + if err := w.RequestWake(wakeCtx); err != nil { + log.Printf("request UDP wake: %v", err) + } + cancel() + } + } + }() + return worker +} + +func (w *udpWakeWorker) request() { + select { + case w.requests <- struct{}{}: + default: + } +} diff --git a/sentinel/wake_budget_test.go b/sentinel/wake_budget_test.go new file mode 100644 index 000000000..1b518742f --- /dev/null +++ b/sentinel/wake_budget_test.go @@ -0,0 +1,264 @@ +package main + +import ( + "bufio" + "context" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "testing" + "time" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + ktesting "k8s.io/client-go/testing" +) + +func TestWakeSharedFailureAndImmediateRetry(t *testing.T) { + w, client := newFakeGameServerWaker(t, "alpha", "games", time.Minute, nil) + started, release := make(chan struct{}), make(chan struct{}) + var patches atomic.Int32 + failure := errors.New("API unavailable") + client.PrependReactor("patch", "gameservers", func(ktesting.Action) (bool, runtime.Object, error) { + if patches.Add(1) == 1 { + close(started) + <-release + return true, nil, failure + } + return false, nil, nil + }) + first, second := make(chan error, 1), make(chan error, 1) + go func() { first <- w.RequestWake(t.Context()) }() + <-started + waiting := &observedWaitContext{Context: t.Context(), observed: make(chan struct{})} + go func() { second <- w.RequestWake(waiting) }() + select { + case err := <-second: + close(release) + t.Fatalf("in-flight waiter returned before wake completed: %v", err) + case <-waiting.observed: + case <-time.After(time.Second): + close(release) + t.Fatal("second wake request never reached its wait") + } + close(release) + for _, result := range []<-chan error{first, second} { + select { + case err := <-result: + if !errors.Is(err, failure) { + t.Fatalf("got %v, want shared failure", err) + } + case <-time.After(time.Second): + t.Fatal("wake waiter did not finish") + } + } + if err := w.RequestWake(t.Context()); err != nil { + t.Fatalf("retry: %v", err) + } + if patches.Load() != 2 { + t.Fatalf("got %d patches, want failed attempt and retry", patches.Load()) + } +} + +// Done is evaluated when RequestWake waits on the shared result. This handshake +// avoids relying on the scheduler running the second goroutine within a sleep. +type observedWaitContext struct { + context.Context + observed chan struct{} + once sync.Once +} + +func (c *observedWaitContext) Done() <-chan struct{} { + c.once.Do(func() { close(c.observed) }) + return c.Context.Done() +} + +func TestWakePatchHasIndependentTimeout(t *testing.T) { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) { + _, _ = io.Copy(io.Discard, req.Body) + select { + case <-req.Context().Done(): + case <-release: + } + })) + defer server.Close() + defer close(release) + client, err := dynamic.NewForConfig(&rest.Config{Host: server.URL}) + if err != nil { + t.Fatal(err) + } + w := newWaker(client.Resource(gameServerGVR).Namespace("games"), "alpha", time.Minute) + w.patchTimeout = 25 * time.Millisecond + started := time.Now() + if err := w.RequestWake(t.Context()); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("got %v, want deadline exceeded", err) + } + if time.Since(started) > time.Second { + t.Fatal("wake PATCH exceeded its own timeout") + } +} + +type contextBoundWake struct{ started chan struct{} } + +func (w *contextBoundWake) RequestWake(ctx context.Context) error { + close(w.started) + <-ctx.Done() + return ctx.Err() +} + +func TestJoinBudgetIncludesWakeRequest(t *testing.T) { + server, client := net.Pipe() + defer server.Close() + defer client.Close() + wake := &contextBoundWake{started: make(chan struct{})} + done := make(chan struct{}) + go func() { + defer close(done) + handleJoin(t.Context(), server, bufio.NewReader(server), wake, joinParams{upstreamAddr: "127.0.0.1:1", deadline: 25 * time.Millisecond, pollInterval: time.Millisecond}) + }() + <-wake.started + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("join deadline did not bound wake request") + } +} + +func TestUDPWakeWorkerCoalescesAndStops(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + wake := &contextBoundWake{started: make(chan struct{})} + worker := newUDPWakeWorker(ctx, wake) + worker.request() + <-wake.started + for range 1000 { + worker.request() + } + if len(worker.requests) > 1 { + t.Fatal("unbounded UDP wake backlog") + } + cancel() + select { + case <-worker.done: + case <-time.After(time.Second): + t.Fatal("UDP wake worker did not stop") + } +} + +func TestSuccessfulGameplayOutlivesJoinBudget(t *testing.T) { + upstream, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer upstream.Close() + accepted := make(chan struct{}) + go func() { + conn, acceptErr := upstream.Accept() + if acceptErr != nil { + return + } + defer conn.Close() + close(accepted) + _, _ = io.Copy(conn, conn) + }() + server, player := tcpPipe(t) + defer player.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan struct{}) + go func() { + defer close(done) + defer server.Close() + handleJoin(ctx, server, bufio.NewReader(server), newCountingWaker(nil), joinParams{upstreamAddr: upstream.Addr().String(), deadline: 100 * time.Millisecond, pollInterval: time.Millisecond}) + }() + select { + case <-accepted: + case <-time.After(time.Second): + t.Fatal("join did not reach game") + } + // The session must retain the drain context after the short join context + // is cancelled, including after its original deadline has elapsed. + time.Sleep(200 * time.Millisecond) + _ = player.SetDeadline(time.Now().Add(time.Second)) + if _, err := player.Write([]byte("still playing")); err != nil { + t.Fatal(err) + } + got := make([]byte, len("still playing")) + if _, err := io.ReadFull(player, got); err != nil || string(got) != "still playing" { + t.Fatalf("gameplay after join budget: %q, %v", got, err) + } + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("session did not stop on drain cancellation") + } +} + +type observedPacketConn struct { + net.PacketConn + reads chan struct{} +} + +func (c observedPacketConn) ReadFrom(p []byte) (int, net.Addr, error) { + n, addr, err := c.PacketConn.ReadFrom(p) + if err == nil { + c.reads <- struct{}{} + } + return n, addr, err +} + +func TestUDPReceiveContinuesWhileWakeIsStalled(t *testing.T) { + listener, err := (&net.ListenConfig{}).ListenPacket(t.Context(), "udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + client, err := (&net.Dialer{}).DialContext(t.Context(), "udp", listener.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + defer client.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + wake := &contextBoundWake{started: make(chan struct{})} + reads := make(chan struct{}, 128) + done := make(chan struct{}) + go func() { + defer close(done) + serveUDPPackets(ctx, observedPacketConn{listener, reads}, PortConfig{}, wake, Config{UDPThreshold: 1, UDPWindow: time.Minute, UDPMaxSources: 10}, make(chan error, 1)) + }() + if _, err := client.Write([]byte("wake")); err != nil { + t.Fatal(err) + } + select { + case <-wake.started: + case <-time.After(time.Second): + t.Fatal("wake did not start") + } + started := time.Now() + for range 100 { + if _, err := client.Write([]byte("game packet")); err != nil { + t.Fatal(err) + } + } + for range 101 { + select { + case <-reads: + case <-time.After(time.Second): + t.Fatal("UDP receive blocked behind stalled wake PATCH") + } + } + t.Logf("received 101 datagrams while wake blocked in %s", time.Since(started)) + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("UDP listener/worker did not stop") + } +} diff --git a/test/e2e/api_events_roles_scope_e2e_test.go b/test/e2e/api_events_roles_scope_e2e_test.go index 2ab19b2b4..4c3b388c0 100644 --- a/test/e2e/api_events_roles_scope_e2e_test.go +++ b/test/e2e/api_events_roles_scope_e2e_test.go @@ -12,6 +12,9 @@ import ( "strings" "testing" "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions checks two @@ -64,19 +67,37 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { applyBusyboxTemplate(t, tmpl) createGameServerViaAPI(t, admin, ns, gsName, tmpl) + // /events starts at each collection's current resourceVersion. Drive real + // updates throughout both streams instead of depending on initial replay + // or other parallel tests to happen to mutate a readable resource kind. + changes := func(ctx context.Context) error { + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": map[string]string{ + "gameplane.local/e2e-event-sequence": time.Now().UTC().Format(time.RFC3339Nano), + }}}) + if err != nil { + return fmt.Errorf("encode event trigger: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, tmpl, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger template event: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, gsName, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger server event: %w", err) + } + return nil + } user := envInstance.APIClient(t, userName, userPW) defer user.Close() t.Run("admin event stream includes templates", func(t *testing.T) { - kinds, found := streamEventKinds(t, admin, ns, func(kind, _ string) bool { return kind == "templates" }, 0) + kinds, found := streamEventKinds(t, admin, ns, func(kind, name string) bool { return kind == "templates" && name == tmpl }, 0, changes) if !found { t.Fatalf("admin stream sent no templates event; kinds=%v", kinds) } }) t.Run("event stream sends only kinds the caller may read", func(t *testing.T) { - kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second) + kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second, changes) if !found { t.Fatalf("user stream never sent server %s; kinds=%v", gsName, kinds) } @@ -126,11 +147,41 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { // streamEventKinds reads GET /events as cli and returns the set of event // kinds it received. When match reports true for a frame, it keeps // reading for grace (0 = stop at once); otherwise it stops after 30s. +// changes keeps both permitted and forbidden kinds active while connecting and +// reading; its worker is cancelled and joined before the helper returns. // found reports whether match was ever true. -func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration) (map[string]bool, bool) { +func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration, changes func(context.Context) error) (map[string]bool, bool) { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() + changesDone := make(chan struct{}) + var changesErr error + go func() { + defer close(changesDone) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + if err := changes(ctx); err != nil { + if ctx.Err() == nil { + changesErr = err + } + cancel() + return + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + defer func() { + cancel() + <-changesDone + if changesErr != nil { + t.Errorf("event trigger failed: %v", changesErr) + } + }() req, err := http.NewRequestWithContext(ctx, http.MethodGet, cli.BaseURL+"/events?namespace="+ns, nil) if err != nil { t.Fatalf("new events request: %v", err) diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index a32d38dea..d5dd59799 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -406,10 +406,15 @@ EOF # Tests that exist in the suite but deliberately run in NO bucket. Every # entry needs a reason; `verify` fails on any unlisted stray so additions # here are a conscious, reviewed act. Currently empty. +bucket_wake_hostport() { cat <<'EOF' +TestGameServer_HostportWakeStableEndpoint +EOF +} + unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components wake-hostport } list_bucket() { @@ -428,6 +433,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + wake-hostport) bucket_wake_hostport ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index d2d6edd00..1775ef3d2 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", + "telemetry", "optional-components", "wake-hostport", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. diff --git a/test/e2e/hostport_wake_e2e_test.go b/test/e2e/hostport_wake_e2e_test.go new file mode 100644 index 000000000..d0461dd76 --- /dev/null +++ b/test/e2e/hostport_wake_e2e_test.go @@ -0,0 +1,238 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// This bucket owns its two-node cluster. It deliberately retains terminating +// Pods and changes node scheduling; it must not run with other game tests. +func TestGameServer_HostportWakeStableEndpoint(t *testing.T) { + ctx := t.Context() + const ns, name, tmpl, port = "gameplane-games", "hostport-wake", "hostport-wake-template", 12345 + nodes, err := envInstance.K8s.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(nodes.Items) < 2 { + t.Fatal("wake-hostport requires the two-node kind profile") + } + var endpoint corev1.Node + for _, node := range nodes.Items { + if _, control := node.Labels["node-role.kubernetes.io/control-plane"]; !control { + endpoint = node + break + } + } + if endpoint.Name == "" { + t.Fatal("no worker node") + } + for _, node := range nodes.Items { + patch := []byte(`{"metadata":{"labels":{"gameplane.local/wake-pool":"test"}}}`) + if _, err := envInstance.K8s.CoreV1().Nodes().Patch(ctx, node.Name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + } + // The game and sentinel both need this dedicated-node toleration. + taints := append(append([]corev1.Taint{}, endpoint.Spec.Taints...), corev1.Taint{Key: "gameplane.local/dedicated", Value: "games", Effect: corev1.TaintEffectNoSchedule}) + patchHostportNode(t, endpoint.Name, map[string]any{"taints": taints}) + t.Cleanup(func() { + patchHostportNode(t, endpoint.Name, map[string]any{"unschedulable": false, "taints": endpoint.Spec.Taints}) + }) + address := "" + for _, addr := range endpoint.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + address = net.JoinHostPort(addr.Address, fmt.Sprint(port)) + break + } + } + if address == "" { + t.Fatal("worker has no InternalIP") + } + + applyBusyboxTemplate(t, tmpl) + // During deletion the shell keeps httpd alive for 12 seconds. This makes + // the old port genuinely occupied while the sleep handoff is observed. + templatePatch := map[string]any{"spec": map[string]any{ + "command": []string{"sh", "-c", "mkdir -p /tmp/site; echo endpoint-ok >/tmp/site/index.html; httpd -p 12345 -h /tmp/site; trap 'sleep 12; exit 0' TERM; while :; do sleep 1; done"}, + "ports": []any{map[string]any{"name": "game", "containerPort": port, "advertise": true, "protocol": "TCP", "wakeProtocol": "generic"}}, + }} + data, err := json.Marshal(templatePatch) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, tmpl, types.MergePatchType, data, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + gs := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", + "metadata": map[string]any{"name": name, "namespace": ns}, + "spec": map[string]any{"templateRef": map[string]any{"name": tmpl}, "networking": map[string]any{"expose": "Hostport"}, + "nodeSelector": map[string]any{"kubernetes.io/hostname": endpoint.Labels["kubernetes.io/hostname"]}, + "tolerations": []any{map[string]any{"key": "gameplane.local/dedicated", "operator": "Equal", "value": "games", "effect": "NoSchedule"}, map[string]any{"key": "node-role.kubernetes.io/control-plane", "operator": "Exists", "effect": "NoSchedule"}}, + "idle": map[string]any{"enabled": true, "afterMinutes": int64(60), "wakeOnConnect": true}}, + }} + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Create(ctx, gs, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + waitPVCBound(t, ns, name+"-data", 90*time.Second) + httpClient := &http.Client{Timeout: 2 * time.Second, Transport: &http.Transport{DisableKeepAlives: true}} + defer httpClient.CloseIdleConnections() + gameReachable := func() (bool, string) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://"+address, nil) + if err != nil { + return false, err.Error() + } + resp, err := httpClient.Do(request) + if err != nil { + return false, err.Error() + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + return err == nil && strings.Contains(string(body), "endpoint-ok"), string(body) + } + for cycle := range 2 { + envInstance.Eventually(t, 2*time.Minute, gameReachable) + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil || pod.Spec.NodeName != endpoint.Name { + t.Fatalf("game moved: pod=%v err=%v", pod, err) + } + patchHostportServer(t, ns, name, map[string]any{"metadata": map[string]any{"annotations": map[string]any{"gameplane.local/idle-asleep-since": time.Now().UTC().Format(time.RFC3339)}}}) + envInstance.Eventually(t, 20*time.Second, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + return err == nil && !pod.DeletionTimestamp.IsZero(), fmt.Sprint(err) + }) + if ok, msg := gameReachable(); !ok { + t.Fatalf("fixture did not retain occupied old port during termination: %s", msg) + } + if _, err := envInstance.K8s.AppsV1().Deployments(ns).Get(ctx, name+"-waker", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("sentinel created before old game released port: %v", err) + } + assertHostportCondition(t, ns, name, "GameDraining", false) + if cycle == 0 { + // Once the endpoint is recorded, broaden the worker pool to both + // nodes. Only the persisted required affinity can retain identity. + patchHostportServer(t, ns, name, map[string]any{"spec": map[string]any{"nodeSelector": map[string]any{"kubernetes.io/hostname": nil, "gameplane.local/wake-pool": "test"}}}) + patchHostportNode(t, endpoint.Name, map[string]any{"unschedulable": true}) + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "EndpointUnschedulable", false) }) + // The alternate node is available; the listener must stay Pending + // on the pinned endpoint instead of silently moving there. + assertHostportPodsOnNode(t, ns, name, endpoint.Name) + patchHostportNode(t, endpoint.Name, map[string]any{"unschedulable": false}) + } + envInstance.Eventually(t, 90*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "ListenerReady", true) }) + assertHostportPodsOnNode(t, ns, name, endpoint.Name) + wakers, err := envInstance.K8s.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gameplane-waker,app.kubernetes.io/instance=" + name}) + if err != nil || len(wakers.Items) != 1 { + t.Fatalf("sentinel Pods: %v, %v", wakers, err) + } + waker := wakers.Items[0] + finalizer := []byte(`{"metadata":{"finalizers":["test.gameplane.local/hold-handoff"]}}`) + if _, err := envInstance.K8s.CoreV1().Pods(ns).Patch(ctx, waker.Name, types.MergePatchType, finalizer, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _, _ = envInstance.K8s.CoreV1().Pods(ns).Patch(context.Background(), waker.Name, types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{}) + }) + // A connection to the same node address must reach the sentinel and + // wake the game. Hostport clients reconnect after this intentional bounce. + conn, err := (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "tcp", address) + if err != nil { + t.Fatalf("sleeping endpoint unreachable: %v", err) + } + _ = conn.Close() + envInstance.Eventually(t, 30*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "SentinelDraining", false) }) + envInstance.Consistently(t, 3*time.Second, 200*time.Millisecond, func() (bool, string) { + _, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + return apierrors.IsNotFound(err), "game scheduled before terminating sentinel Pod disappeared" + }) + if _, err := envInstance.K8s.CoreV1().Pods(ns).Patch(ctx, waker.Name, types.MergePatchType, []byte(`{"metadata":{"finalizers":[]}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 2*time.Minute, gameReachable) + assertHostportPodsOnNode(t, ns, name, endpoint.Name) + t.Logf("cycle %d kept reachable endpoint %s on %s", cycle+1, address, endpoint.Name) + } + // Real node deletion (including replacement under the same name) must + // report unavailable instead of migrating to the otherwise eligible node. + patchHostportServer(t, ns, name, map[string]any{"metadata": map[string]any{"annotations": map[string]any{"gameplane.local/idle-asleep-since": time.Now().UTC().Format(time.RFC3339)}}}) + if err := envInstance.K8s.CoreV1().Nodes().Delete(ctx, endpoint.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 45*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, "EndpointNodeUnavailable", false) }) + assertHostportPodsOnNode(t, ns, name, endpoint.Name) +} + +func patchHostportServer(t *testing.T, ns, name string, patch map[string]any) { + t.Helper() + data, err := json.Marshal(patch) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(context.Background(), name, types.MergePatchType, data, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } +} + +func patchHostportNode(t *testing.T, name string, spec map[string]any) { + t.Helper() + data, err := json.Marshal(map[string]any{"spec": spec}) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.K8s.CoreV1().Nodes().Patch(context.Background(), name, types.MergePatchType, data, metav1.PatchOptions{}); err != nil && !apierrors.IsNotFound(err) { + t.Error(err) + } +} + +func hostportConditionIs(t *testing.T, ns, name, reason string, ready bool) (bool, string) { + t.Helper() + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + conditions, _, _ := unstructured.NestedSlice(gs.Object, "status", "conditions") + for _, item := range conditions { + condition, ok := item.(map[string]any) + if ok && condition["type"] == "HostportWakeReady" { + return condition["reason"] == reason && (condition["status"] == "True") == ready, fmt.Sprint(condition) + } + } + return false, "HostportWakeReady not yet present" +} + +func assertHostportCondition(t *testing.T, ns, name, reason string, ready bool) { + t.Helper() + envInstance.Eventually(t, 15*time.Second, func() (bool, string) { return hostportConditionIs(t, ns, name, reason, ready) }) +} + +func assertHostportPodsOnNode(t *testing.T, ns, name, node string) { + t.Helper() + pods, err := envInstance.K8s.CoreV1().Pods(ns).List(t.Context(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/instance=" + name}) + if err != nil { + t.Fatal(err) + } + for _, pod := range pods.Items { + if pod.Spec.NodeName != "" && pod.Spec.NodeName != node { + t.Fatalf("listener %s moved to %s; endpoint remains %s", pod.Name, pod.Spec.NodeName, node) + } + } +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index abf7cda91..b5bd411a4 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -19,6 +19,17 @@ It was one of three `go.work` modules without CI lint coverage until this featur - Validate multi-cluster dispatch, upgrade-from-previous-release, wake-on-connect, module signing/SSRF rejection, and backup/restore round trips end to end. - Enforce, via `buckets.sh`, that every test in the suite is assigned to exactly one CI bucket so nothing silently drops out of coverage. +### Hostport wake acceptance + +The dedicated `wake-hostport` bucket runs on the `two-node` kind profile. It uses +a real Hostport HTTP listener and dedicated-node taint, broadens the scheduling +pool after recording the endpoint, and exercises two sleep/wake cycles at the +same node address. It observes the old game still serving during termination, +holds a terminating sentinel Pod to prove the reverse handoff waits, checks a +cordoned endpoint remains unavailable rather than relocating, and deletes the +node to exercise the explicit unavailable/replaced-node condition. This bucket +runs alone because it changes node scheduling. Runtime results come from CI. + ### Remote gateway parity acceptance The multicluster bucket runs `TestMultiCluster_GatewayParity` against two real @@ -56,6 +67,11 @@ The golangci-lint job mirrors this: it must be invoked with `--build-tags=e2e` o ## The bucket contract — the module's most important invariant +The `wake-hostport` audit bucket is registered in the change-based selector, +with explicit component mapping and an amd64-only selection-summary row. Its +matrix leg skips cluster setup only when its selected test pattern is empty; +full-scope and affected-component runs must include it. + `buckets.sh` is the single source of truth for how CI's e2e suite is sharded into parallel jobs (`operator`, `api-auth`, `api-roles`, `api-rbac`, `api-agent`, `api-mods`, `ratelimit`, `bot-fast`, `bot-heavy`, `multicluster`, `upgrade`). Each bucket is a flat list of **exact Go function names** (e.g. `TestGameServer_MinecraftJavaBot_Joined`), and `buckets.sh regex ` turns that list into an **anchored** `^(A|B|...)$` regex passed to `go test -run`. This is a name-matching contract, not a structural one: diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index 017d1f88f..f29a7a281 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -16,6 +16,7 @@ "test/e2e/suite-components.json" ], "suites": [ + { "tests": "^TestGameServer_HostportWakeStableEndpoint$", "components": ["operator", "api", "agent", "sentinel", "gameproto"] }, { "tests": "^TestHelmInstall_", "components": ["*"] }, { "tests": "^TestCRD_Validation_", "components": ["operator"] }, { "tests": "^TestGameServer_", "components": ["operator", "api", "agent"] },