diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 26d9ee778..82e6358b8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -439,7 +439,8 @@ jobs: needs: [changes] if: (needs.changes.outputs.images == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'build-images') runs-on: ubuntu-latest - timeout-minutes: 15 + # Include all image builds, cache exports, artifact upload and cleanup. + timeout-minutes: 30 permissions: contents: read steps: @@ -1232,6 +1233,10 @@ jobs: run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders run: python3 charts/tests/check_standalone.py + - name: workload security chart contract + run: python3 charts/tests/check_workload_security.py + - name: auxiliary resources and management placement chart renders + run: python3 charts/tests/check_workloads.py - name: sanity-check rendered manifests run: | grep -q 'kind: Deployment' /tmp/rendered.yaml @@ -1793,13 +1798,17 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, management-placement] arch: [amd64] include: - bucket: operator parallel: 8 test_timeout: 20m job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: management-placement + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: api-auth parallel: 4 test_timeout: 15m @@ -1829,6 +1838,10 @@ jobs: parallel: 1 test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: restricted-workloads + parallel: 1 + test_timeout: 25m # Includes real projected-token rotation and node capture cleanup. + job_timeout: 60 env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e @@ -1856,7 +1869,9 @@ jobs: arch: ${{ matrix.arch }} - name: bring up cluster + helm install if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' - run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e + env: + GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }} + run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) if: env.RUN_REGEX != '' run: | @@ -1900,7 +1915,7 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # Mirrors e2e-go's steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads] arch: [arm64] include: - bucket: operator @@ -1936,6 +1951,10 @@ jobs: parallel: 1 test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: restricted-workloads + parallel: 1 + test_timeout: 25m # Includes real projected-token rotation and node capture cleanup. + job_timeout: 60 env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e @@ -2660,8 +2679,8 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '), - 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), 'e2e-upgrade': (n) => n.startsWith('e2e upgrade /') && n.includes('/ amd64 '), @@ -2813,7 +2832,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/agent/cmd/main.go b/agent/cmd/main.go index 8ce7f15f8..0b5c86d44 100644 --- a/agent/cmd/main.go +++ b/agent/cmd/main.go @@ -209,6 +209,13 @@ func main() { protected.Use(authCheck.Middleware, target.Guard(serverUID)) mountProtected(protected) }) + // Backups bind save-off/save-on to the process they prepared. The service + // may already route to a replacement pod, which must reject the old UID. + r.Route("/v1/pods/{uid}", func(protected chi.Router) { + protected.Use(authCheck.Middleware, target.Guard(envOr("GAMEPLANE_POD_UID", "")), middleware.Timeout(30*time.Second)) + protected.Get("/identity", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + quiesce.Mount(protected, rconClient, gameName, quiesceSpec) + }) srv := &http.Server{ Addr: addr, diff --git a/agent/internal/quiesce/quiesce_test.go b/agent/internal/quiesce/quiesce_test.go index 777ba9ade..62aca43a4 100644 --- a/agent/internal/quiesce/quiesce_test.go +++ b/agent/internal/quiesce/quiesce_test.go @@ -12,6 +12,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/GameplanePanel/gameplane/agent/internal/caps" + "github.com/GameplanePanel/gameplane/agent/internal/target" ) type fakeRcon struct { @@ -86,6 +87,33 @@ func newTestRouter(rc Rcon, spec *caps.Quiesce) *chi.Mux { return r } +func TestPodBoundRoutesRejectReplacementBeforeRCON(t *testing.T) { + for _, actual := range []string{"original", "replacement", ""} { + t.Run("pod="+actual, func(t *testing.T) { + rc := &fakeRcon{} + r := chi.NewRouter() + r.Route("/v1/pods/{uid}", func(bound chi.Router) { + bound.Use(target.Guard(actual)) + Mount(bound, rc, "test", minecraftQuiesceSpec()) + }) + for _, path := range []string{"quiesce", "unquiesce"} { + response := httptest.NewRecorder() + r.ServeHTTP(response, httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/v1/pods/original/"+path, nil)) + if actual == "original" { + if response.Code != http.StatusOK { + t.Fatalf("matching process rejected: %d", response.Code) + } + } else if response.Code != http.StatusNotFound { + t.Fatalf("wrong process admitted: %d", response.Code) + } + } + if (len(rc.calls()) > 0) != (actual == "original") { + t.Fatalf("commands reached wrong process: %v", rc.calls()) + } + }) + } +} + func decodeResponse(t *testing.T, body []byte) response { t.Helper() var got response diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 248c6a7fb..9061d2c94 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1275,18 +1275,47 @@ type pipeFrameCase struct { forbidErr string // substring the error must NOT contain (e.g. a detail suffix) } +type responseDeadlineConn struct { + net.Conn + deadlineSet chan<- struct{} +} + +func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error { + if err := c.Conn.SetDeadline(deadline); err != nil { + return err + } + select { + case c.deadlineSet <- struct{}{}: + default: + } + return nil +} + // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe // and checks the body or the error. func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + deadlineSet := make(chan struct{}, 1) client := newPipeClient(t, func(remote net.Conn) { if _, _, err := readNuclearOptionRequest(remote); err != nil { return } + if len(tt.frame) == 0 { + // Exec sets its response deadline after sending the request. + // Closing first can fail SetDeadline instead of the status read. + select { + case <-deadlineSet: + case <-t.Context().Done(): + return + } + } writeFrameSegments(remote, tt.frame, tt.oneByte) }) + if len(tt.frame) == 0 { + client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet} + } body, err := client.Exec("test-command") if tt.wantErr != "" { diff --git a/agent/internal/target/target.go b/agent/internal/target/target.go index 6a1300631..3e79a3944 100644 --- a/agent/internal/target/target.go +++ b/agent/internal/target/target.go @@ -1,4 +1,4 @@ -// Package target binds versioned agent routes to an immutable GameServer UID. +// Package target binds versioned agent routes to an immutable resource UID. package target import ( @@ -7,12 +7,13 @@ import ( "github.com/go-chi/chi/v5" ) -// Guard rejects a route for any server other than the one that started this -// agent. Legacy routes stay available for local callers during rolling upgrades. -func Guard(serverUID string) func(http.Handler) http.Handler { +// Guard rejects a route for any resource other than the one that started this +// agent (GameServer or Pod, according to the route). Legacy routes stay available +// for local callers during rolling upgrades. +func Guard(resourceUID string) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if serverUID == "" || chi.URLParam(req, "uid") != serverUID { + if resourceUID == "" || chi.URLParam(req, "uid") != resourceUID { http.NotFound(w, req) return } diff --git a/agent/specs.md b/agent/specs.md index af1f57529..b549c155f 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -5,6 +5,16 @@ ## Purpose +### Kubernetes token projection (#714) + +Game Pods disable automatic credentials. The agent alone mounts a rotating +projected token plus kube-root CA and namespace at client-go's standard path; +`rest.InClusterConfig` preserves `BearerTokenFile` refresh. No token is mounted +into game/config-init. Shared PID remains intentional for process metrics and +is residual in-Pod trust; mount isolation does not make the agent safe from +all processes in its Pod. CI exercises live projection rotation and a heartbeat +after the original short token expires. + The agent is a per-pod HTTP/HTTPS sidecar that runs inside every game pod to expose the game server's operations to the control plane and dashboard. It translates dashboard requests into game-protocol actions (RCON, file I/O, container logs, player queries) and reports liveness, resource usage, and player metrics back to the operator via Kubernetes API patches, decoupling game-specific behavior from the cluster control plane. ## Responsibilities @@ -177,6 +187,8 @@ Resource usage env vars (set by the operator): | `/players/banned` | GET | Currently banned players; response: `[ { name, reason?, source? } ]`; 502 if RCON unavailable | | `/quiesce` | POST | Pause auto-saves before snapshot; response: `{ quiesced, reason? }` (boolean, reason optional); unsupported games return `quiesced: false` with a reason | | `/unquiesce` | POST | Resume auto-saves after snapshot; response: `{ quiesced, reason? }` | +| `/v1/pods/{uid}/identity` | GET | Operator preflight: 204 only when `{uid}` matches `GAMEPLANE_POD_UID`; never mutates game state | +| `/v1/pods/{uid}/quiesce`, `/v1/pods/{uid}/unquiesce` | POST | Pod-bound backup RPCs with the same response contract as the legacy routes; reject missing/mismatched Pod UID with 404 before RCON | | `/lifecycle/stop` | POST | Run stop sequence before scale-to-zero; RCON connection drop is the expected outcome | | `/actions/run` | POST | Execute module-declared operator action; request: `{ id, params }`; response: `{ ok, raw? }` | | `/status` | GET | Live game metrics (module-declared); response: bare JSON array `[ { id, displayName?, value, unit? } ]` | @@ -209,6 +221,7 @@ All endpoints on the `--addr` control mux, except `/healthz`, return `401 Unauth - **Extracted mod files stay world-readable**: `mods.moduleFileMode` is `0o644`, not a tighter `0o600`, because the mods volume is shared with the game container, which runs as whatever uid its image needs — a different uid than the agent's own. At `0o600` the game process couldn't read its own mods. This is the rationale behind the `.golangci.yml` gosec G302 exclusion scoped to `agent/internal/mods/mods.go`. - **Resource usage is in-pod**: The `usage` package reads from `/proc` or cgroups; no external metrics pipeline required. Cgroup mode is a fallback for older clusters; proc mode (default in production) requires the operator to set `ShareProcessNamespace: true`. - **Module capabilities drive behavior**: Every game-specific handler (players, quiesce, lifecycle, status, actions) reads its config from `--capabilities` (JSON unmarshaled into `caps.Spec`). New games require no agent code change. +- **Backup process identity**: The operator injects `GAMEPLANE_POD_UID` from `metadata.uid` through the DownwardAPI. Pod-bound backup routes require the existing mTLS authentication and check this UID at the final network hop, including when the Service resolves to a replacement pod. Missing UID fails closed. Deploy the updated agent image and allow pod rollout when upgrading the operator; older agents return 404 and the operator does not fall back to unbound routes. Legacy endpoints remain available to existing callers during rollout. - **RCON connection errors are graceful**: A lost RCON connection does not crash the agent. `console`, `players`, `quiesce`, and `lifecycle` handlers catch connection errors and return appropriate HTTP status (e.g., `502 Bad Gateway`). - **Source RCON sends complete frames**: Each AUTH or command frame is submitted in one socket write, with its existing byte-size limit, request ID, type, body, and two NUL terminators preserved. This avoids separate header-only writes rejected by vanilla Minecraft's RCON reader. A short write is an error and drops the connection without replaying the command; response grace and healthy connection reuse remain unchanged. - **Log streams are tail-only**: The `logs` package does not support random-access reads. It streams from the current end (live mode) or from file start (backlog mode); clients must handle partial output and reconnection. diff --git a/api/internal/gateway/capture.go b/api/internal/gateway/capture.go index 9ead2ed3e..7ecf476d6 100644 --- a/api/internal/gateway/capture.go +++ b/api/internal/gateway/capture.go @@ -52,6 +52,12 @@ func (h *handler) serveCapture(w http.ResponseWriter, req *http.Request, target } defer transport.CloseIdleConnections() host := target.Name + "-agent." + target.Namespace + ".svc.cluster.local:9091" + if name, namespace, node, err := kube.NodeCaptureService(nc); err != nil { + http.NotFound(w, req) + return + } else if node { + host = name + "-agent." + namespace + ".svc.cluster.local:9091" + } proxy := &httputil.ReverseProxy{ Transport: transport, Rewrite: func(pr *httputil.ProxyRequest) { diff --git a/api/internal/handlers/capture.go b/api/internal/handlers/capture.go index 1f3171bcf..36583adf2 100644 --- a/api/internal/handlers/capture.go +++ b/api/internal/handlers/capture.go @@ -72,6 +72,8 @@ func MountCapture(r chi.Router, reg *kube.Registry, auditor *audit.Auditor, cfg } r.Post("/servers/{name}:capture-enable", h.captureEnable) r.Post("/servers/{name}:capture-disable", h.captureDisable) + r.Post("/servers/{name}:capture-node-enable", h.captureNodeEnable) + r.Post("/servers/{name}:capture-node-disable", h.captureNodeDisable) r.Post("/servers/{name}:capture-start", h.captureStart) r.Post("/servers/{name}:capture-stop", h.captureStop) r.Get("/servers/{name}:captures", h.captureList) @@ -957,7 +959,9 @@ func (h *captureHandler) captureDelete(w http.ResponseWriter, req *http.Request) } deleteErr = deleteCaptureWithUID(req.Context(), k, nc) } else { - h.deleteSidecarCaptureFile(req.Context(), gs.Name, gs.Namespace, captureID) + if nc.Annotations[kube.NodeCaptureNamespaceAnnotation] == "" { + h.deleteSidecarCaptureFile(req.Context(), gs.Name, gs.Namespace, captureID) + } deleteErr = k.DeleteNetworkCapture(req.Context(), ns, captureID) } @@ -1099,7 +1103,11 @@ func (h *captureHandler) captureDownload(w http.ResponseWriter, req *http.Reques if isRemoteCluster(req) { status = h.remoteCaptureDownload(w, req, gs, nc) } else { - status = h.proxyFileDownload(w, req, host, captureID) + if _, _, node, _ := kube.NodeCaptureService(nc); node { + status = h.proxyNodeCaptureDownload(w, req, gs, nc) + } else { + status = h.proxyFileDownload(w, req, host, captureID) + } } if status < http.StatusOK || (status >= http.StatusMultipleChoices && status != http.StatusNotModified) { // A disconnect must not also erase the corrective audit outcome. diff --git a/api/internal/handlers/capture_node.go b/api/internal/handlers/capture_node.go new file mode 100644 index 000000000..6025037ab --- /dev/null +++ b/api/internal/handlers/capture_node.go @@ -0,0 +1,111 @@ +package handlers + +import ( + "encoding/json" + "errors" + "net/http" + "net/url" + + "github.com/GameplanePanel/gameplane/api/internal/auth" + "github.com/GameplanePanel/gameplane/api/internal/httperr" + "github.com/GameplanePanel/gameplane/api/internal/kube" + "github.com/GameplanePanel/gameplane/api/internal/scope" + "github.com/go-chi/chi/v5" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func (h *captureHandler) captureNodeEnable(w http.ResponseWriter, r *http.Request) { + h.captureNodeOptIn(w, r, true) +} +func (h *captureHandler) captureNodeDisable(w http.ResponseWriter, r *http.Request) { + h.captureNodeOptIn(w, r, false) +} + +func (h *captureHandler) captureNodeOptIn(w http.ResponseWriter, req *http.Request, enabled bool) { + k, ok := resolveCluster(w, req, h.reg) + if !ok { + return + } + ns, ok := resolveNS(w, req) + if !ok { + return + } + name := chi.URLParam(req, "name") + u := auth.UserFromContext(req.Context()) + if u == nil || (u.Role != "admin" && !u.Can("*", true, scope.RequestedCluster(req), "*")) { + if h.auditWriteOrFail(w, req, req.Method, req.URL.Path, name, "node_diagnostics_admin_required", http.StatusForbidden) { + httperr.WriteCode(w, req, http.StatusForbidden, errors.New("node diagnostic exception requires administrator privileges")) + } + return + } + gs, err := k.GetServer(req.Context(), ns, name) + if err != nil { + httperr.Write(w, req, err) + return + } + if gs.GetDeletionTimestamp() != nil { + httperr.WriteCode(w, req, http.StatusConflict, errors.New("server is terminating")) + return + } + reason := "node_diagnostics_opt_out" + if enabled { + reason = "node_diagnostics_opt_in_host_privileges" + } + // Record intent synchronously before granting host capabilities. A failed + // audit write prevents the exception; Kubernetes status remains separate. + if !h.auditWriteOrFail(w, req, req.Method, req.URL.Path, name+":"+string(gs.GetUID()), reason, http.StatusOK) { + return + } + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"uid": string(gs.GetUID()), "resourceVersion": gs.GetResourceVersion()}, "spec": map[string]any{"capture": map[string]any{"nodeDiagnostic": enabled}}}) + if err != nil { + httperr.Write(w, req, err) + return + } + _, err = k.Dynamic.Resource(kube.GVRs["servers"]).Namespace(ns).Patch(req.Context(), name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + if h.auditWriteOrFail(w, req, req.Method, req.URL.Path, name, "node_diagnostics_update_failed", http.StatusInternalServerError) { + httperr.Write(w, req, err) + } + return + } + if !enabled { + if err := h.stopActiveCaptures(req.Context(), k, ns, name); err != nil { + httperr.Write(w, req, err) + return + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]bool{"nodeDiagnostic": enabled}) +} + +func (h *captureHandler) proxyNodeCaptureDownload(w http.ResponseWriter, req *http.Request, gs *kube.GameServer, nc *kube.NetworkCapture) int { + bound, err := captureTarget(scope.RequestedCluster(req), gs, nc) + if err != nil { + http.NotFound(w, req) + return http.StatusNotFound + } + name, ns, _, err := kube.NodeCaptureService(nc) + if err != nil { + httperr.Write(w, req, err) + return http.StatusInternalServerError + } + host, ok := newValidatedHost(name, ns) + if !ok || h.tlsClient == nil { + http.Error(w, "capture artifacts unavailable", http.StatusServiceUnavailable) + return http.StatusServiceUnavailable + } + u := url.URL{Scheme: "https", Host: string(host), Path: "/v1/targets/" + bound.UID + "/captures/" + bound.Capture + "/uids/" + bound.CaptureUID + "/file"} + up, err := http.NewRequestWithContext(req.Context(), http.MethodGet, u.String(), nil) + if err != nil { + httperr.Write(w, req, err) + return http.StatusInternalServerError + } + copyProxyHeaders(up.Header, req.Header) + resp, err := h.tlsClient.Do(up) + if err != nil { + return writeUpstreamError(w, req, err) + } + defer func() { _ = resp.Body.Close() }() + return streamCaptureResponse(w, req, resp, nc.Name) +} diff --git a/api/internal/handlers/capture_node_test.go b/api/internal/handlers/capture_node_test.go new file mode 100644 index 000000000..28063868c --- /dev/null +++ b/api/internal/handlers/capture_node_test.go @@ -0,0 +1,188 @@ +package handlers + +import ( + "net/http" + "net/url" + "strings" + "testing" + + "github.com/GameplanePanel/gameplane/api/internal/audit" + "github.com/GameplanePanel/gameplane/api/internal/auth" + "github.com/GameplanePanel/gameplane/api/internal/kube" + "github.com/GameplanePanel/gameplane/api/internal/scope" + "github.com/go-chi/chi/v5" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestNodeCaptureExceptionRequiresAuditedAdministrator(t *testing.T) { + for _, admin := range []bool{false, true} { + t.Run(map[bool]string{false: "capture-manager", true: "administrator"}[admin], func(t *testing.T) { + gs := newCaptureServerObj("game", true) + gs.SetUID("server-uid") + k := fakeCaptureClient(gs) + store := newTestStore(t) + router := mountCaptureTestRouter(k, CaptureConfig{FeatureEnabled: true}, audit.New(store)) + user := &auth.User{Username: "manager", Role: "custom", Perms: map[string]map[string]map[string]struct{}{"local": {"gameplane-games": {"captures:manage": {}}}}} + want := http.StatusForbidden + if admin { + user = testAdminUser() + want = http.StatusOK + } + rr := doWithUser(t, router, http.MethodPost, "/servers/game:capture-node-enable", nil, user) + if rr.Code != want { + t.Fatalf("status %d %s", rr.Code, rr.Body) + } + live, err := k.GetServer(t.Context(), scope.DefaultNamespace, "game") + if err != nil { + t.Fatal(err) + } + enabled, _, _ := unstructured.NestedBool(live.Object, "spec", "capture", "nodeDiagnostic") + if enabled != admin { + t.Fatalf("opt-in=%t", enabled) + } + if admin { + var count int + if err := store.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_events WHERE reason = ?", "node_diagnostics_opt_in_host_privileges").Scan(&count); err != nil || count != 1 { + t.Fatalf("missing explicit audit event count=%d err=%v", count, err) + } + } + }) + } +} + +func TestNodeExceptionRequiresAdministratorOnSelectedCluster(t *testing.T) { + for _, test := range []struct { + name, target, grantCluster, grantNamespace string + allowed bool + }{ + {"other-cluster", "cluster-b", "cluster-a", "*", false}, + {"namespace-only", "cluster-b", "cluster-b", "gameplane-games", false}, + {"selected-cluster", "cluster-b", "cluster-b", "*", true}, + {"default-local", "", "local", "*", true}, + {"explicit-local", "local", "local", "*", true}, + {"normalized-local", " local ", "local", "*", true}, + {"other-cluster-local-target", "local", "cluster-a", "*", false}, + } { + t.Run(test.name, func(t *testing.T) { + gs := newCaptureServerObj("game", true) + gs.SetUID("server-uid") + k := fakeCaptureClient(gs) + reg := kube.NewRegistry(scope.DefaultCluster) + reg.Set("local", k) + reg.Set("cluster-b", k) + router := chi.NewRouter() + MountCapture(router, reg, audit.New(newTestStore(t)), CaptureConfig{FeatureEnabled: true}, "", "", "") + user := &auth.User{Username: "custom-admin", Role: "custom", Perms: map[string]map[string]map[string]struct{}{test.grantCluster: {test.grantNamespace: {"*": {}}}}} + target := strings.TrimSpace(test.target) + if target == "" { + target = "local" + } + if user.Perms[target] == nil { + user.Perms[target] = map[string]map[string]struct{}{} + } + if user.Perms[target][scope.DefaultNamespace] == nil { + user.Perms[target][scope.DefaultNamespace] = map[string]struct{}{} + } + user.Perms[target][scope.DefaultNamespace]["captures:manage"] = struct{}{} + path := "/servers/game:capture-node-enable" + if test.target != "" { + path += "?cluster=" + url.QueryEscape(test.target) + } + rr := doWithUser(t, router, http.MethodPost, path, nil, user) + want := http.StatusForbidden + if test.allowed { + want = http.StatusOK + } + if rr.Code != want { + t.Fatalf("status %d want %d: %s", rr.Code, want, rr.Body) + } + live, err := k.GetServer(t.Context(), scope.DefaultNamespace, "game") + if err != nil { + t.Fatal(err) + } + enabled, _, _ := unstructured.NestedBool(live.Object, "spec", "capture", "nodeDiagnostic") + if enabled != test.allowed { + t.Fatal("wrong scope granted host exception") + } + }) + } +} + +func TestNodeExceptionCannotBeGrantedByGenericEditOrClone(t *testing.T) { + src := newServerObj("gameplane-games", "alpha") + if err := unstructured.SetNestedField(src.Object, true, "spec", "capture", "nodeDiagnostic"); err != nil { + t.Fatal(err) + } + k := fakeKubeClient(src) + err := validateAndProtectGameServer(auth.WithUser(t.Context(), testAdminUser()), k, "local", "gameplane-games", "new", src, nil) + if err == nil || !strings.Contains(err.Error(), "capture-node-enable") { + t.Fatalf("generic create granted exception: %v", err) + } + rr := doWithUser(t, mountLifecycleRouter(k), http.MethodPost, "/servers/alpha:clone", map[string]any{"newName": "beta"}, testAdminUser()) + if rr.Code != http.StatusOK { + t.Fatalf("clone: %d %s", rr.Code, rr.Body) + } + clone, err := k.GetServer(t.Context(), "gameplane-games", "beta") + if err != nil { + t.Fatal(err) + } + if enabled, _, _ := unstructured.NestedBool(clone.Object, "spec", "capture", "nodeDiagnostic"); enabled { + t.Fatal("clone inherited node exception") + } +} + +func TestNodeExceptionGenericEditPreservesOmittedOptIn(t *testing.T) { + manager := &auth.User{Username: "manager", Role: "custom", Perms: map[string]map[string]map[string]struct{}{"local": {"gameplane-games": {"captures:manage": {}}}}} + for _, user := range []*auth.User{testAdminUser(), manager} { + for _, enabled := range []bool{false, true} { + live := newServerObj("gameplane-games", "alpha") + if err := unstructured.SetNestedField(live.Object, enabled, "spec", "capture", "nodeDiagnostic"); err != nil { + t.Fatal(err) + } + for _, omitCapture := range []bool{false, true} { + desired := live.DeepCopy() + unstructured.RemoveNestedField(desired.Object, "spec", "capture", "nodeDiagnostic") + if omitCapture { + unstructured.RemoveNestedField(desired.Object, "spec", "capture") + } + if err := validateAndProtectGameServer(auth.WithUser(t.Context(), user), fakeKubeClient(live), "local", "gameplane-games", "alpha", desired, live); err != nil { + t.Fatalf("omitted exception rejected for %s, enabled=%t: %v", user.Role, enabled, err) + } + got, _, _ := unstructured.NestedBool(desired.Object, "spec", "capture", "nodeDiagnostic") + if got != enabled { + t.Fatal("generic edit changed an omitted exception") + } + } + desired := live.DeepCopy() + if err := unstructured.SetNestedField(desired.Object, !enabled, "spec", "capture", "nodeDiagnostic"); err != nil { + t.Fatal(err) + } + if err := validateAndProtectGameServer(auth.WithUser(t.Context(), user), fakeKubeClient(live), "local", "gameplane-games", "alpha", desired, live); err == nil || !strings.Contains(err.Error(), "capture-node-enable") { + t.Fatalf("generic edit changed explicit exception for %s: %v", user.Role, err) + } + } + } +} + +func TestNodeCaptureExceptionAuditFailureDoesNotGrant(t *testing.T) { + gs := newCaptureServerObj("game", true) + gs.SetUID("server-uid") + k := fakeCaptureClient(gs) + store := newTestStore(t) + auditor := audit.New(store) + if err := store.Close(); err != nil { + t.Fatal(err) + } + rr := doWithUser(t, mountCaptureTestRouter(k, CaptureConfig{FeatureEnabled: true}, auditor), http.MethodPost, "/servers/game:capture-node-enable", nil, testAdminUser()) + if rr.Code != http.StatusInternalServerError { + t.Fatalf("status %d", rr.Code) + } + live, err := k.GetServer(t.Context(), scope.DefaultNamespace, "game") + if err != nil { + t.Fatal(err) + } + enabled, _, _ := unstructured.NestedBool(live.Object, "spec", "capture", "nodeDiagnostic") + if enabled { + t.Fatal("granted exception without audit") + } +} diff --git a/api/internal/handlers/lifecycle.go b/api/internal/handlers/lifecycle.go index 5d10e517a..ebe1fe9b2 100644 --- a/api/internal/handlers/lifecycle.go +++ b/api/internal/handlers/lifecycle.go @@ -229,6 +229,9 @@ func cloneHandler(reg *kube.Registry) http.HandlerFunc { clone.SetFinalizers(nil) clone.SetManagedFields(nil) clone.SetCreationTimestamp(metav1.Time{}) + // A node diagnostic exception belongs to one server identity. Cloning + // never grants host-level diagnostic privileges to the new server. + unstructured.RemoveNestedField(clone.Object, "spec", "capture", "nodeDiagnostic") stampOwner(clone, req) cl := req.URL.Query().Get("cluster") diff --git a/api/internal/handlers/resources.go b/api/internal/handlers/resources.go index 2e892ff5d..8c16dd7f0 100644 --- a/api/internal/handlers/resources.go +++ b/api/internal/handlers/resources.go @@ -581,6 +581,27 @@ func validateAndProtectGameServer( } // 2. Validate spec.serviceAccountName + // A host-capable diagnostic exception must be a separate, synchronously + // audited administrator action. Generic edits may preserve an existing + // opt-in, but cannot grant/revoke it or inherit it on a clone. + nodeDiagnostic, hasNodeDiagnostic, err := unstructured.NestedBool(desired.Object, "spec", "capture", "nodeDiagnostic") + if err != nil { + return errors.New("spec.capture.nodeDiagnostic must be a boolean") + } + liveNodeDiagnostic := false + if live != nil { + liveNodeDiagnostic, _, _ = unstructured.NestedBool(live.Object, "spec", "capture", "nodeDiagnostic") + if !hasNodeDiagnostic && liveNodeDiagnostic { + if err := unstructured.SetNestedField(desired.Object, true, "spec", "capture", "nodeDiagnostic"); err != nil { + return fmt.Errorf("preserve node diagnostic exception: %w", err) + } + nodeDiagnostic = true + } + } + if nodeDiagnostic != liveNodeDiagnostic { + return errors.New("change the node diagnostic exception through the administrator capture-node-enable/capture-node-disable action") + } + sa, hasSA, _ := unstructured.NestedString(desired.Object, "spec", "serviceAccountName") if !isAdmin { if live == nil { diff --git a/api/internal/handlers/resources_annotations_test.go b/api/internal/handlers/resources_annotations_test.go index 457d5ebfb..a615dc2fe 100644 --- a/api/internal/handlers/resources_annotations_test.go +++ b/api/internal/handlers/resources_annotations_test.go @@ -39,6 +39,7 @@ func TestGenericServerWritesCannotInjectLifecycleAnnotations(t *testing.T) { wipeRequestedAnnotation: "injected", "gameplane.local/wipe-data-completed": "forged", "gameplane.local/restore-guard": "forged", + "operations.gameplane.local/owner": "forged", "gameserver.gameplane.local/stop-requested-at": "forged", "restore.gameplane.local/created-by": "forged", ownerIDAnnotation: "999", @@ -51,7 +52,7 @@ func TestGenericServerWritesCannotInjectLifecycleAnnotations(t *testing.T) { t.Fatalf("write failed: %d %s", response.Code, response.Body) } got := serverAnnotations(t, k, "alpha") - for _, key := range []string{wipeRequestedAnnotation, "gameplane.local/wipe-data-completed", "gameplane.local/restore-guard", "gameserver.gameplane.local/stop-requested-at", "restore.gameplane.local/created-by"} { + for _, key := range []string{wipeRequestedAnnotation, "gameplane.local/wipe-data-completed", "gameplane.local/restore-guard", "operations.gameplane.local/owner", "gameserver.gameplane.local/stop-requested-at", "restore.gameplane.local/created-by"} { if _, found := got[key]; found { t.Errorf("generic write injected %s", key) } @@ -98,6 +99,7 @@ func TestGenericServerUpdatePreservesControllerState(t *testing.T) { wipeRequestedAnnotation: "pending", "gameplane.local/wipe-data-completed": "previous", "gameplane.local/backup-guard": "active-backup", + "operations.gameplane.local/owner": `{"kind":"Backup","name":"copy","uid":"copy-uid","targetUID":"server-uid","generation":1}`, "gameserver.gameplane.local/stop-requested-at": "original-time", "restore.gameplane.local/created-by": "original-restore", } diff --git a/api/internal/kube/capture.go b/api/internal/kube/capture.go index 7fe33ba4d..9b6d04f26 100644 --- a/api/internal/kube/capture.go +++ b/api/internal/kube/capture.go @@ -100,7 +100,8 @@ type NetworkCapture struct { // CaptureConfiguration mirrors operator/api/v1alpha1.CaptureConfiguration // (only the fields the API needs to read). type CaptureConfiguration struct { - Enabled bool `json:"enabled,omitempty"` + Enabled bool `json:"enabled,omitempty"` + NodeDiagnostic bool `json:"nodeDiagnostic,omitempty"` // RetentionSeconds is the per-GameServer override of the cluster-default // retention window applied when a capture-start request omits its own // ttlSecondsAfterFinished. Still bounded by the cluster maximum, the same diff --git a/api/internal/kube/capture_node.go b/api/internal/kube/capture_node.go new file mode 100644 index 000000000..038fad49b --- /dev/null +++ b/api/internal/kube/capture_node.go @@ -0,0 +1,27 @@ +package kube + +import ( + "errors" + "regexp" + + "k8s.io/apimachinery/pkg/util/validation" +) + +// NodeCaptureNamespaceAnnotation is written only by the operator when it +// binds a NetworkCapture to a dedicated diagnostic artifact service. +const NodeCaptureNamespaceAnnotation = "gameplane.local/capture-node-namespace" + +var nodeCaptureUID = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +// NodeCaptureService derives a fixed in-cluster service from the immutable +// capture UID. An annotation can never supply a URL, arbitrary host or path. +func NodeCaptureService(nc *NetworkCapture) (name, namespace string, node bool, err error) { + namespace = nc.Annotations[NodeCaptureNamespaceAnnotation] + if namespace == "" { + return "", "", false, nil + } + if !nodeCaptureUID.MatchString(string(nc.UID)) || len(validation.IsDNS1123Label(namespace)) != 0 { + return "", "", true, errors.New("invalid diagnostic artifact identity") + } + return "capture-" + string(nc.UID), namespace, true, nil +} diff --git a/api/internal/kube/capture_node_test.go b/api/internal/kube/capture_node_test.go new file mode 100644 index 000000000..332c03c31 --- /dev/null +++ b/api/internal/kube/capture_node_test.go @@ -0,0 +1,26 @@ +package kube + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestNodeCaptureServiceDerivesOnlyBoundInternalName(t *testing.T) { + nc := &NetworkCapture{ObjectMeta: metav1.ObjectMeta{UID: "11111111-2222-3333-4444-555555555555", Annotations: map[string]string{NodeCaptureNamespaceAnnotation: "diagnostics"}}} + name, ns, node, err := NodeCaptureService(nc) + if err != nil || !node || ns != "diagnostics" || name != "capture-11111111-2222-3333-4444-555555555555" { + t.Fatalf("unexpected target: %s %s %t %v", name, ns, node, err) + } + for _, invalid := range []string{"https://host", "../games", "host:9091", "UPPER"} { + nc.Annotations[NodeCaptureNamespaceAnnotation] = invalid + if _, _, _, err := NodeCaptureService(nc); err == nil { + t.Fatalf("accepted namespace %q", invalid) + } + } + nc.Annotations[NodeCaptureNamespaceAnnotation] = "diagnostics" + nc.UID = "../../host" + if _, _, _, err := NodeCaptureService(nc); err == nil { + t.Fatal("accepted invalid capture UID") + } +} diff --git a/api/internal/rbac/rbac.go b/api/internal/rbac/rbac.go index 3d5563cfa..9cff2bd14 100644 --- a/api/internal/rbac/rbac.go +++ b/api/internal/rbac/rbac.go @@ -246,6 +246,8 @@ var rules = []rule{ // that moves these rules after the servers catch-all silently grants // capture access to the operator role, defeating FR-005/SC-005. {method: "POST", segment: "servers", suffix: ":capture-enable", perm: "captures:manage"}, + {method: "POST", segment: "servers", suffix: ":capture-node-enable", perm: "captures:manage"}, + {method: "POST", segment: "servers", suffix: ":capture-node-disable", perm: "captures:manage"}, {method: "POST", segment: "servers", suffix: ":capture-disable", perm: "captures:manage"}, {method: "POST", segment: "servers", suffix: ":capture-start", perm: "captures:manage"}, {method: "POST", segment: "servers", suffix: ":capture-stop", perm: "captures:manage"}, diff --git a/api/specs.md b/api/specs.md index 1dccc8987..ad20f6f22 100644 --- a/api/specs.md +++ b/api/specs.md @@ -5,6 +5,22 @@ ## Purpose +### Node diagnostic exception (#714) + +`POST /servers/{name}:capture-node-enable` and `:capture-node-disable` require +cluster-wide administrator authority in the selected cluster in addition to +capture-management routing; a namespace-only wildcard grant is insufficient. +Enable synchronously audits the server UID and +host-privilege intent before a UID/resourceVersion-fenced patch; audit failure +grants nothing. Generic create/update cannot change `capture.nodeDiagnostic`, +and clones clear it. Generic updates that omit the field preserve its live +value, including clients that omit the entire capture object. Explicit changes +still require the dedicated audited action. Disable requests cancellation of active captures. Other +capture operations retain capture permissions and existing scope/provenance +checks. A node capture's operator annotation and immutable capture UID derive +its diagnostic Service; local and gateway file downloads use UID-bound mTLS +routes. Capture deletion delegates node artifact cleanup to the CR finalizer. + The api module is the REST + WebSocket gateway for Gameplane, serving the web dashboard and external integrations. It exposes the Kubernetes game-server CRDs (GameServer, GameTemplate, Backup, BackupSchedule, Restore, Module, ModuleSource, Cluster) and operator-reconciled state through a type-safe HTTP surface with multi-cluster support, role-based access control, and audit logging. ## Responsibilities diff --git a/capture-sidecar/Dockerfile b/capture-sidecar/Dockerfile index 05ecbfc50..a97d12ba7 100644 --- a/capture-sidecar/Dockerfile +++ b/capture-sidecar/Dockerfile @@ -28,11 +28,14 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ # only mechanism that survives a non-root execve. The final stage relies on this # grant and runs as nonroot, accepting allowPrivilegeEscalation: true to permit # file capabilities to take effect (they are ignored under no_new_privs). -RUN setcap cap_net_raw+ep /out/capture-sidecar +RUN cp /out/capture-sidecar /out/capture-tool && setcap cap_net_raw+ep /out/capture-sidecar FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=build /out/capture-sidecar /capture-sidecar +# No file capabilities: the artifact server is Restricted; the node worker +# receives its explicit root capability set from its dedicated Job spec. +COPY --from=build /out/capture-tool /capture-tool USER 65532:65532 EXPOSE 9091 ENTRYPOINT ["/capture-sidecar"] diff --git a/capture-sidecar/cmd/main.go b/capture-sidecar/cmd/main.go index 337f157d9..c1219037c 100644 --- a/capture-sidecar/cmd/main.go +++ b/capture-sidecar/cmd/main.go @@ -12,6 +12,7 @@ package main import ( "context" + "encoding/json" "errors" "flag" "log/slog" @@ -24,6 +25,7 @@ import ( "github.com/GameplanePanel/gameplane/capture-sidecar/internal/auth" "github.com/GameplanePanel/gameplane/capture-sidecar/internal/httpserver" + "github.com/GameplanePanel/gameplane/capture-sidecar/internal/nodecapture" ) // Version is overridden at build time via -ldflags (see Dockerfile). @@ -68,6 +70,8 @@ func main() { keyFile := flag.String("tls-key", envOrDefault("TLS_KEY_FILE", "/etc/tls/tls.key"), "Server key file") caFile := flag.String("tls-client-ca", envOrDefault("TLS_CA_FILE", "/etc/tls/ca.crt"), "Client CA certificate file") captureDataDir := flag.String("capture-dir", "/tmp/captures", "Directory for capture files") + nodeWorker := flag.Bool("node-worker", false, "Run one operator-bound node diagnostic capture and exit") + artifactOnly := flag.Bool("artifacts-only", false, "Serve completed node capture artifacts without capture controls") // The operator sets CAPTURE_VOLUME_BUDGET_BYTES on the capture ephemeral // container (see buildCaptureEphemeralContainer in @@ -89,6 +93,18 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + if *nodeWorker { + var cfg nodecapture.Config + if *artifactOnly || json.Unmarshal([]byte(os.Getenv("NODE_CAPTURE_CONFIG")), &cfg) != nil { + slog.Error("invalid node capture configuration") + os.Exit(1) + } + if err := nodecapture.Run(ctx, cfg, *captureDataDir); err != nil { + slog.Error("node capture failed", "err", err) + os.Exit(1) + } + return + } // Create capture server. ctx (cancelled on SIGTERM/SIGINT) is the parent // for every capture goroutine, so shutdown cancels in-flight captures @@ -111,6 +127,9 @@ func main() { // (see HandleHealthz's doc comment for why nothing actually calls this // route today; F-193). mux := captureServer.Routes(auth.Middleware) + if *artifactOnly { + mux = captureServer.ArtifactRoutes(auth.Middleware) + } server := &http.Server{ Addr: *listenAddr, diff --git a/capture-sidecar/go.mod b/capture-sidecar/go.mod index 3815530bf..2834a12db 100644 --- a/capture-sidecar/go.mod +++ b/capture-sidecar/go.mod @@ -8,4 +8,4 @@ require ( golang.org/x/net v0.61.0 ) -require golang.org/x/sys v0.49.0 // indirect +require golang.org/x/sys v0.49.0 diff --git a/capture-sidecar/internal/httpserver/artifacts.go b/capture-sidecar/internal/httpserver/artifacts.go new file mode 100644 index 000000000..8eeafa1b7 --- /dev/null +++ b/capture-sidecar/internal/httpserver/artifacts.go @@ -0,0 +1,57 @@ +package httpserver + +import ( + "encoding/json" + "io" + "net/http" + + "github.com/GameplanePanel/gameplane/capture-sidecar/internal/nodecapture" +) + +// ArtifactRoutes exposes retained node-worker results without any operation +// that can open a raw socket or start privileged work. The Pod is Restricted. +func (s *Server) ArtifactRoutes(mw func(http.Handler) http.Handler) *http.ServeMux { + mux := http.NewServeMux() + if mw == nil { + mw = func(h http.Handler) http.Handler { return h } + } + mux.HandleFunc("GET /healthz", HandleHealthz) + mux.Handle("GET /captures/{id}/status", mw(http.HandlerFunc(s.handleArtifactStatus))) + mux.Handle("GET /v1/targets/{serverUID}/captures/{id}/uids/{captureUID}/file", mw(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + result, ok := s.artifactResult(r.PathValue("id")) + if !ok || result.Status != "completed" { + http.NotFound(w, r) + return + } + s.HandleBoundDownload(w, r) + }))) + mux.Handle("DELETE /v1/targets/{serverUID}/captures/{id}/uids/{captureUID}/file", mw(http.HandlerFunc(s.HandleBoundDelete))) + return mux +} + +func (s *Server) artifactResult(id string) (nodecapture.Result, bool) { + var result nodecapture.Result + if !validCaptureID(id) { + return result, false + } + f, err := s.openCaptureRegular("capture-" + id + ".result.json") + if err != nil { + return result, false + } + defer func() { _ = f.Close() }() + raw, err := io.ReadAll(io.LimitReader(f, 4097)) + if err != nil || len(raw) > 4096 || json.Unmarshal(raw, &result) != nil || result.CaptureID != id || result.ServerUID != s.serverUID || !captureUIDPattern.MatchString(result.CaptureUID) { + return result, false + } + return result, true +} + +func (s *Server) handleArtifactStatus(w http.ResponseWriter, r *http.Request) { + result, ok := s.artifactResult(r.PathValue("id")) + if !ok || r.Header.Get("X-Gameplane-Server-UID") != result.ServerUID || r.Header.Get("X-Gameplane-Capture-UID") != result.CaptureUID { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(result) +} diff --git a/capture-sidecar/internal/httpserver/artifacts_test.go b/capture-sidecar/internal/httpserver/artifacts_test.go new file mode 100644 index 000000000..959f4d66d --- /dev/null +++ b/capture-sidecar/internal/httpserver/artifacts_test.go @@ -0,0 +1,31 @@ +package httpserver + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestArtifactServerCannotStartCaptureAndBindsResult(t *testing.T) { + dir := t.TempDir() + s := NewServer(t.Context(), dir, 0, "server-uid") + routes := s.ArtifactRoutes(nil) + if err := os.WriteFile(filepath.Join(dir, "capture-cap-one.result.json"), []byte(`{"serverUID":"server-uid","captureUID":"capture-uid","captureId":"cap-one","status":"completed"}`), 0o600); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + method, path, uid string + want int + }{{http.MethodPost, "/captures/cap-one/start", "capture-uid", http.StatusNotFound}, {http.MethodGet, "/captures/cap-one/status", "other-uid", http.StatusNotFound}, {http.MethodGet, "/captures/cap-one/status", "capture-uid", http.StatusOK}} { + req := httptest.NewRequestWithContext(t.Context(), test.method, test.path, nil) + req.Header.Set("X-Gameplane-Server-UID", "server-uid") + req.Header.Set("X-Gameplane-Capture-UID", test.uid) + rr := httptest.NewRecorder() + routes.ServeHTTP(rr, req) + if rr.Code != test.want { + t.Fatalf("%s %s: %d %s", test.method, test.path, rr.Code, rr.Body) + } + } +} diff --git a/capture-sidecar/internal/nodecapture/worker.go b/capture-sidecar/internal/nodecapture/worker.go new file mode 100644 index 000000000..4a8e8acaf --- /dev/null +++ b/capture-sidecar/internal/nodecapture/worker.go @@ -0,0 +1,277 @@ +// Package nodecapture runs one bounded capture against an operator-selected +// containerd game process. It never accepts a host PID or filesystem path. +package nodecapture + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + "github.com/GameplanePanel/gameplane/capture-sidecar/internal/capture" + "golang.org/x/sys/unix" +) + +var identityPattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,128}$`) +var podPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) +var containerPattern = regexp.MustCompile(`^containerd://[0-9a-f]{64}$`) +var capturePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + +// The separate artifact server reads finalized files through the dedicated PVC group. +const artifactMode os.FileMode = 0o640 + +// Config contains immutable identities copied from live Kubernetes objects. +type Config struct { + ServerUID string `json:"serverUID"` + CaptureUID string `json:"captureUID"` + CaptureID string `json:"captureID"` + PodUID string `json:"podUID"` + ContainerID string `json:"containerID"` + Node string `json:"node"` + Filter string `json:"filter"` + DurationSeconds int64 `json:"durationSeconds"` + MaxSizeBytes int64 `json:"maxSizeBytes"` +} + +// Validate rejects unsupported runtimes, unconstrained captures and paths. +func (c Config) Validate() error { + if !identityPattern.MatchString(c.ServerUID) || !identityPattern.MatchString(c.CaptureUID) || !capturePattern.MatchString(c.CaptureID) || !podPattern.MatchString(c.PodUID) || !containerPattern.MatchString(c.ContainerID) || c.Node == "" || c.Filter == "" || len(c.Filter) > 1024 || c.DurationSeconds < 1 || c.DurationSeconds > 3600 || c.MaxSizeBytes < 1 || c.MaxSizeBytes > 900*1024*1024 { + return errors.New("invalid node capture identity, runtime, filter or limits") + } + return nil +} + +// Result is written atomically only after the PCAP has been closed. +type Result struct { + ServerUID string `json:"serverUID"` + CaptureUID string `json:"captureUID"` + CaptureID string `json:"captureId"` + Status string `json:"status"` + PacketsWritten int64 `json:"packetsWritten"` + BytesWritten int64 `json:"bytesWritten"` + Message string `json:"message"` +} + +func matchesCgroup(body, pod, id string) bool { + for _, line := range strings.Split(body, "\n") { + parts := strings.SplitN(line, ":", 3) + if len(parts) != 3 { + continue + } + p := parts[2] + podMatch := strings.Contains(p, "/pod"+pod+"/") || strings.Contains(p, "-pod"+strings.ReplaceAll(pod, "-", "_")+".slice/") + containerMatch := strings.HasSuffix(p, "/"+id) || strings.HasSuffix(p, "/cri-containerd-"+id+".scope") + if podMatch && containerMatch { + return true + } + } + return false +} + +type targetProcess struct { + root *os.Root + namespace *os.File + pod, id string +} + +func (t *targetProcess) close() { _ = t.namespace.Close(); _ = t.root.Close() } +func (t *targetProcess) verify() error { + body, err := t.root.ReadFile("cgroup") + if err != nil || !matchesCgroup(string(body), t.pod, t.id) { + return errors.New("target container identity changed") + } + return nil +} + +func findTarget(c Config) (*targetProcess, error) { + entries, err := os.ReadDir("/proc") + if err != nil { + return nil, fmt.Errorf("read host processes: %w", err) + } + id := strings.TrimPrefix(c.ContainerID, "containerd://") + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if _, err := strconv.Atoi(entry.Name()); err != nil { + continue + } + root, err := os.OpenRoot(filepath.Join("/proc", entry.Name())) + if err != nil { + continue + } + body, err := root.ReadFile("cgroup") + if err != nil || !matchesCgroup(string(body), c.PodUID, id) { + _ = root.Close() + continue + } + // /proc namespace descriptors are magic symlinks, so os.Root correctly + // refuses Open on them. Open through this already-verified numeric PID, + // then verify the pinned proc directory again before and after setns. + ns, err := os.Open(filepath.Join("/proc", entry.Name(), "ns/net")) + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("open target network namespace: %w", err) + } + t := &targetProcess{root: root, namespace: ns, pod: c.PodUID, id: id} + if err := t.verify(); err != nil { + t.close() + return nil, err + } + return t, nil + } + return nil, errors.New("containerd target process not found on selected node") +} + +func (t *targetProcess) source(filter string) (capture.PacketSource, error) { + type opened struct { + s capture.PacketSource + err error + } + done := make(chan opened, 1) + go func() { + // Never return a thread in the target netns to Go's thread pool. + // A goroutine exiting with its OS thread locked terminates that thread; + // the AF_PACKET descriptor remains bound to the original target netns. + runtime.LockOSThread() + if err := t.verify(); err != nil { + done <- opened{err: err} + return + } + if err := unix.Setns(int(t.namespace.Fd()), unix.CLONE_NEWNET); err != nil { + done <- opened{err: fmt.Errorf("enter target network namespace: %w", err)} + return + } + if err := t.verify(); err != nil { + done <- opened{err: err} + return + } + s, err := capture.NewAFPacketSource("eth0", 0, filter) + done <- opened{s: s, err: err} + }() + r := <-done + return r.s, r.err +} + +// Run captures once and exits. SIGTERM cancels ctx and finalizes the file; +// controller completion waits for worker Pods to disappear before publishing. +func Run(ctx context.Context, c Config, dir string) error { + if err := c.Validate(); err != nil { + return err + } + target, err := findTarget(c) + if err != nil { + return err + } + defer target.close() + source, err := target.source(c.Filter) + if err != nil { + return err + } + defer func() { _ = source.Close() }() + return runCapture(ctx, c, dir, source, target.verify) +} + +func runCapture(ctx context.Context, c Config, dir string, source capture.PacketSource, verify func() error) error { + identity, err := json.Marshal(map[string]string{"serverUID": c.ServerUID, "captureUID": c.CaptureUID}) + if err != nil { + return err + } + output, err := os.OpenRoot(dir) + if err != nil { + return fmt.Errorf("open capture output directory: %w", err) + } + defer func() { _ = output.Close() }() + identityName := "capture-" + c.CaptureID + ".identity.json" + f, err := output.OpenFile(identityName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("reserve capture identity: %w", err) + } + if _, err = f.Write(identity); err != nil { + _ = f.Close() + return err + } + if err = f.Close(); err != nil { + return err + } + // fsGroup grants the nonroot artifact server access after the worker exits. + if err = output.Chmod(identityName, artifactMode); err != nil { + return err + } + writer, err := capture.NewWriter(filepath.Join(dir, "capture-"+c.CaptureID+".pcapng"), c.DurationSeconds, c.MaxSizeBytes, 0, c.Filter) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, time.Duration(c.DurationSeconds)*time.Second) + defer cancel() + identityChanged := make(chan struct{}, 1) + go func() { + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if verify() != nil { + identityChanged <- struct{}{} + cancel() + return + } + } + } + }() + result := Result{ServerUID: c.ServerUID, CaptureUID: c.CaptureUID, CaptureID: c.CaptureID, Status: "completed"} + for { + packet, err := source.ReadPacket(ctx) + if err != nil { + if ctx.Err() == nil { + result.Status = "failed" + result.Message = "packet source failed" + } + break + } + if err := writer.WritePacket(packet); err != nil { + if !errors.Is(err, capture.ErrLimitReached) { + result.Status = "failed" + result.Message = "capture write failed" + } + break + } + } + select { + case <-identityChanged: + result.Status = "failed" + result.Message = "target container identity changed" + default: + } + if verify() != nil { + result.Status = "failed" + result.Message = "target container identity changed" + } + if err := writer.Close(); err != nil { + result.Status = "failed" + result.Message = "capture finalization failed" + } + result.PacketsWritten = writer.PacketsWritten() + result.BytesWritten = writer.BytesWritten() + if err := output.Chmod("capture-"+c.CaptureID+".pcapng", artifactMode); err != nil { + return err + } + raw, err := json.Marshal(result) + if err != nil { + return err + } + path := "capture-" + c.CaptureID + ".result.json" + if err := output.WriteFile(path+".tmp", raw, artifactMode); err != nil { + return err + } + return output.Rename(path+".tmp", path) +} diff --git a/capture-sidecar/internal/nodecapture/worker_test.go b/capture-sidecar/internal/nodecapture/worker_test.go new file mode 100644 index 000000000..ccb349f13 --- /dev/null +++ b/capture-sidecar/internal/nodecapture/worker_test.go @@ -0,0 +1,97 @@ +package nodecapture + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/GameplanePanel/gameplane/capture-sidecar/internal/capture" +) + +func TestCgroupIdentityMatchesExactPodAndContainer(t *testing.T) { + pod := "11111111-2222-3333-4444-555555555555" + id := strings.Repeat("a", 64) + for _, test := range []struct { + name, cgroup string + want bool + }{ + {"systemd", "0::/kubepods.slice/kubepods-burstable-pod" + strings.ReplaceAll(pod, "-", "_") + ".slice/cri-containerd-" + id + ".scope\n", true}, + {"cgroupfs", "0::/kubepods/burstable/pod" + pod + "/" + id + "\n", true}, + {"other-pod", "0::/kubepods/pod99999999-2222-3333-4444-555555555555/" + id + "\n", false}, + {"container-prefix", "0::/kubepods/pod" + pod + "/" + id + "a\n", false}, + {"host", "0::/system.slice/containerd.service\n", false}, + } { + t.Run(test.name, func(t *testing.T) { + if got := matchesCgroup(test.cgroup, pod, id); got != test.want { + t.Fatalf("identity match=%t want %t", got, test.want) + } + }) + } +} + +type endingSource struct{ err error } + +func (s endingSource) ReadPacket(context.Context) (*capture.RawPacket, error) { return nil, s.err } +func (endingSource) Close() error { return nil } + +func TestRunCapturePublishesOnlyFinalizedIdentityBoundResult(t *testing.T) { + for _, changed := range []bool{false, true} { + t.Run(map[bool]string{false: "canceled", true: "identity-changed"}[changed], func(t *testing.T) { + dir := t.TempDir() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + c := Config{ServerUID: "server", CaptureUID: "capture", CaptureID: "cap", DurationSeconds: 1, MaxSizeBytes: 4096, Filter: "tcp port 25565"} + verify := func() error { + if changed { + return errors.New("replaced") + } + return nil + } + if err := runCapture(ctx, c, dir, endingSource{context.Canceled}, verify); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(dir, "capture-cap.result.json")) + if err != nil { + t.Fatal(err) + } + var result Result + if err := json.Unmarshal(raw, &result); err != nil { + t.Fatal(err) + } + want := "completed" + if changed { + want = "failed" + } + if result.Status != want || result.ServerUID != "server" || result.CaptureUID != "capture" || result.BytesWritten <= 0 { + t.Fatalf("invalid result: %+v", result) + } + for _, name := range []string{"capture-cap.identity.json", "capture-cap.pcapng", "capture-cap.result.json"} { + stat, err := os.Stat(filepath.Join(dir, name)) + if err != nil || stat.Mode().Perm() != 0o640 { + t.Fatalf("artifact %s mode: %v %v", name, stat, err) + } + } + if err := runCapture(ctx, c, dir, endingSource{context.Canceled}, verify); err == nil { + t.Fatal("reused existing capture identity") + } + }) + } +} + +func TestWorkerRejectsUnsafeConfiguration(t *testing.T) { + c := Config{ServerUID: "server-uid", CaptureUID: "capture-uid", CaptureID: "cap-test", PodUID: "11111111-2222-3333-4444-555555555555", ContainerID: "containerd://" + strings.Repeat("a", 64), Node: "node", Filter: "tcp port 25565", DurationSeconds: 60, MaxSizeBytes: 1024} + if err := c.Validate(); err != nil { + t.Fatal(err) + } + for _, mutate := range []func(*Config){func(c *Config) { c.ContainerID = "docker://" + strings.Repeat("a", 64) }, func(c *Config) { c.CaptureID = "../../host" }, func(c *Config) { c.PodUID = "" }, func(c *Config) { c.DurationSeconds = 3601 }, func(c *Config) { c.MaxSizeBytes = 1 << 30 }, func(c *Config) { c.Filter = "" }} { + bad := c + mutate(&bad) + if bad.Validate() == nil { + t.Fatalf("accepted unsafe config: %+v", bad) + } + } +} diff --git a/capture-sidecar/specs.md b/capture-sidecar/specs.md index 5485d2def..10d8bb994 100644 --- a/capture-sidecar/specs.md +++ b/capture-sidecar/specs.md @@ -6,6 +6,27 @@ ## Purpose +### Separate node diagnostic worker (#714) + +The image also contains `/capture-tool` without file capabilities. Its +`--node-worker` mode accepts only an operator-generated JSON identity tuple +(GameServer/NetworkCapture/Pod UIDs, node, exact containerd ID), filter and +bounded limits. It discovers a process through exact Pod/container cgroup +matching, pins proc/network namespace descriptors, verifies cgroups immediately +before/after setns on a dedicated locked OS thread, then opens an eth0 AF_PACKET +socket. The locked thread exits; it is never returned to the thread pool. +Ongoing identity checks cancel capture, including idle packet streams. No host +PID/path, socket, mount or Kubernetes token is supplied. Linux/containerd, +systemd/cgroupfs paths are supported; other runtimes and host networking fail. + +PCAP, identity and atomically published result files are group-readable only +after finalization. A separate nonroot `--artifacts-only` process exposes +UID-bound download/delete and header-bound status, never start/stop/raw socket +operations. The controller removes the privileged Job/Pods before publishing +completion. SIGTERM finalizes a partial capture; changed identity publishes a +failed result. Host capability blast radius remains node-wide despite the +per-server authorization and exact target selection. + Optional packet capture sidecar injected into game pods to record network traffic for protocol reverse-engineering. The sidecar will use AF_PACKET live capture to collect packets matching a BPF filter expression, write them to a PCAPNG file on a pre-provisioned emptyDir volume, and expose a mTLS-authenticated HTTP control endpoint (`:9091`) for the operator and API to start/stop/monitor captures and download the resulting files. > **Current state:** Full implementation with AF_PACKET socket handling, PCAPNG writing, BPF filter compilation, mTLS authentication, and HTTP control endpoint. Real dependencies (gopacket, go-pcap) are pinned in go.mod. Unit tests for capture, filter, writer, and handlers are co-located with their implementations. TTL-based expiry reconciliation (`NetworkCaptureReconciler.expireCapture`, `operator/internal/controller/networkcapture_controller.go`) and the dashboard UI (`web/src/components/CaptureWidget.tsx`, `web/src/routes/tabs/settings/NetworkCapture.tsx`) are both implemented — see "Planned" below for the one remaining gap (F-190). diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index d902ea995..7d088cf5d 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: @@ -129,6 +129,75 @@ spec: status: description: BackupStatus is the observed state of a Backup. properties: + captureManifest: + description: |- + CaptureManifest is recorded once, before the first capture worker starts. + It describes persistent-volume coverage independently of copy success. + Missing on pre-upgrade backups; never reconstruct it from current layout. + properties: + layoutSource: + description: LayoutSource identifies the frozen workload used + for mount paths. + type: string + quiesceOutcome: + description: |- + QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + agent-unavailable, or unknown, independent of subsequent copy success. + type: string + recordedAt: + format: date-time + type: string + serverUID: + type: string + state: + description: |- + State is Recorded for new captures, LegacyUnknown for an existing worker + whose original layout was never recorded. + enum: + - Recorded + - LegacyUnknown + type: string + strategy: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + mountPaths: + items: + type: string + type: array + reason: + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - state + type: object completionTime: format: date-time type: string diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index d85406090..9aa20db87 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 4d6aadcbb..9e2410aca 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index 95178e56e..12108a345 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1040,10 +1040,18 @@ spec: description: |- Enabled: opt-in flag for packet-capture capability on this GameServer. If false or omitted, no capture sidecar is injected; the game container is unchanged. - If true, the operator injects the capture sidecar as an ephemeral container via + In the legacy profile, the operator injects the capture sidecar via pods/ephemeralcontainers, live, without restarting the game container. Disabling (true→false) does not remove the sidecar from a running pod — Kubernetes has no API to remove ephemeral containers — but stops accepting new captures. + Restricted workloads require NodeDiagnostic instead and never inject a sidecar. + type: boolean + nodeDiagnostic: + description: |- + NodeDiagnostic permits the separately deployed node capture worker for + this server. Application administrators must explicitly opt in. The + worker has host-level capabilities; it never weakens the game Pod. + Required for capture on Restricted workloads. Off by default. type: boolean retentionSeconds: description: |- @@ -2215,6 +2223,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. @@ -2286,6 +2304,27 @@ spec: - message: templateRef is immutable; delete and recreate the GameServer to switch templates rule: self == oldSelf + templateRollout: + description: |- + TemplateRollout controls adoption of changes to the shared GameTemplate. + Omitted preserves Automatic adoption. Per-server overrides remain live. + properties: + policy: + default: Automatic + description: Policy defaults to Automatic. OnRequest requires + the exact desired digest. + enum: + - Automatic + - OnRequest + type: string + requestedRevision: + description: |- + RequestedRevision authorizes this SHA256 only, never future revisions. + Read status.templateRevision.desired and copy it here to admit an update. + Initial provisioning always adopts the installed template. + pattern: ^[a-f0-9]{64}$ + type: string + type: object tolerations: items: description: |- @@ -2621,6 +2660,25 @@ spec: observed as Ready. format: date-time type: string + templateRevision: + description: |- + TemplateRevision is a read model; operator-owned annotations and immutable + ConfigMaps retain admission state across status resets and leader changes. + properties: + desired: + description: Desired is the installed template's SHA256; empty + if it is unavailable. + type: string + effective: + description: Effective is the immutable content used for all derived + resources. + type: string + running: + description: |- + Running comes from an owned Pod with a running game container. Empty + means no running revision is known, including pre-upgrade legacy Pods. + type: string + type: object tunnelEndpoints: description: |- TunnelEndpoints are the per-port addresses reported by the tunnel pod. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index e5e3e1e3b..374847afd 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: @@ -2166,6 +2166,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 5ade28cee..dc939332c 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 6c5dcbf36..033d1e44b 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index cc113e7e8..291261445 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index b77cd5c25..52e9bb3b7 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index d902ea995..7d088cf5d 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: @@ -129,6 +129,75 @@ spec: status: description: BackupStatus is the observed state of a Backup. properties: + captureManifest: + description: |- + CaptureManifest is recorded once, before the first capture worker starts. + It describes persistent-volume coverage independently of copy success. + Missing on pre-upgrade backups; never reconstruct it from current layout. + properties: + layoutSource: + description: LayoutSource identifies the frozen workload used + for mount paths. + type: string + quiesceOutcome: + description: |- + QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + agent-unavailable, or unknown, independent of subsequent copy success. + type: string + recordedAt: + format: date-time + type: string + serverUID: + type: string + state: + description: |- + State is Recorded for new captures, LegacyUnknown for an existing worker + whose original layout was never recorded. + enum: + - Recorded + - LegacyUnknown + type: string + strategy: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + mountPaths: + items: + type: string + type: array + reason: + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - state + type: object completionTime: format: date-time type: string diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index d85406090..9aa20db87 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 4d6aadcbb..9e2410aca 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index 95178e56e..12108a345 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1040,10 +1040,18 @@ spec: description: |- Enabled: opt-in flag for packet-capture capability on this GameServer. If false or omitted, no capture sidecar is injected; the game container is unchanged. - If true, the operator injects the capture sidecar as an ephemeral container via + In the legacy profile, the operator injects the capture sidecar via pods/ephemeralcontainers, live, without restarting the game container. Disabling (true→false) does not remove the sidecar from a running pod — Kubernetes has no API to remove ephemeral containers — but stops accepting new captures. + Restricted workloads require NodeDiagnostic instead and never inject a sidecar. + type: boolean + nodeDiagnostic: + description: |- + NodeDiagnostic permits the separately deployed node capture worker for + this server. Application administrators must explicitly opt in. The + worker has host-level capabilities; it never weakens the game Pod. + Required for capture on Restricted workloads. Off by default. type: boolean retentionSeconds: description: |- @@ -2215,6 +2223,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. @@ -2286,6 +2304,27 @@ spec: - message: templateRef is immutable; delete and recreate the GameServer to switch templates rule: self == oldSelf + templateRollout: + description: |- + TemplateRollout controls adoption of changes to the shared GameTemplate. + Omitted preserves Automatic adoption. Per-server overrides remain live. + properties: + policy: + default: Automatic + description: Policy defaults to Automatic. OnRequest requires + the exact desired digest. + enum: + - Automatic + - OnRequest + type: string + requestedRevision: + description: |- + RequestedRevision authorizes this SHA256 only, never future revisions. + Read status.templateRevision.desired and copy it here to admit an update. + Initial provisioning always adopts the installed template. + pattern: ^[a-f0-9]{64}$ + type: string + type: object tolerations: items: description: |- @@ -2621,6 +2660,25 @@ spec: observed as Ready. format: date-time type: string + templateRevision: + description: |- + TemplateRevision is a read model; operator-owned annotations and immutable + ConfigMaps retain admission state across status resets and leader changes. + properties: + desired: + description: Desired is the installed template's SHA256; empty + if it is unavailable. + type: string + effective: + description: Effective is the immutable content used for all derived + resources. + type: string + running: + description: |- + Running comes from an owned Pod with a running game container. Empty + means no running revision is known, including pre-upgrade legacy Pods. + type: string + type: object tunnelEndpoints: description: |- TunnelEndpoints are the per-port addresses reported by the tunnel pod. diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index e5e3e1e3b..374847afd 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: @@ -2166,6 +2166,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 5ade28cee..dc939332c 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 6c5dcbf36..033d1e44b 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index cc113e7e8..291261445 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index b77cd5c25..52e9bb3b7 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: dc993913a7a930b00001ba1ee07e5c5e29988ca82915ea217b90ae2f98289f82 controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index bec3e1ffb..baf7eb564 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -135,3 +135,33 @@ true true {{- end -}} {{- end -}} + +{{- /* Shared PodSpec placement fields; absent keys preserve older reuse-values. */ -}} +{{- define "gameplane.placement" -}} +{{- range $key := list "nodeSelector" "tolerations" "affinity" "topologySpreadConstraints" "priorityClassName" -}} +{{- with index $ $key }} +{{ $key }}: {{ toYaml . | nindent 2 }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- /* Complete auxiliary overrides; empty objects must not become defaults. */ -}} +{{- define "gameplane.auxiliaryResourceArgs" -}} +{{- range $name := list "agent" "sentinel" "tunnel" "backup" -}} +{{- $key := printf "%sResources" $name -}} +{{- $value := index $ $key -}} +{{- if and (hasKey $ $key) (not (kindIs "invalid" $value)) -}} +{{- if not (kindIs "map" $value) -}} +{{- fail (printf "operator.%s must be a resources object or null" $key) -}} +{{- end }} +- {{ printf "--%s-resources=%s" $name (toJson $value) | quote }} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- /* Resolve new diagnostics defaults for upgrades using older reuse-values. +mergeOverwrite preserves explicit zero/empty values so validation can reject them. */ -}} +{{- define "gameplane.nodeDiagnostics" -}} +{{- $configured := (.Values.capture | default dict).nodeDiagnostics | default dict -}} +{{- mergeOverwrite (dict "enabled" false "namespace" "gameplane-diagnostics" "maxCaptures" 10 "storageQuota" "10Gi" "storageClass" "") $configured | toJson -}} +{{- end -}} diff --git a/charts/gameplane/templates/api.yaml b/charts/gameplane/templates/api.yaml index a562d7c7b..f04fa99ec 100644 --- a/charts/gameplane/templates/api.yaml +++ b/charts/gameplane/templates/api.yaml @@ -258,6 +258,7 @@ spec: checksum/mtls: {{ include "gameplane.mtlsChecksum" . }} {{- end }} spec: + {{- include "gameplane.placement" .Values.api | nindent 6 }} serviceAccountName: gameplane-api {{- if .Values.api.standalone }} automountServiceAccountToken: false diff --git a/charts/gameplane/templates/audit-syslog-bridge.yaml b/charts/gameplane/templates/audit-syslog-bridge.yaml index 11bb8b279..c030d0150 100644 --- a/charts/gameplane/templates/audit-syslog-bridge.yaml +++ b/charts/gameplane/templates/audit-syslog-bridge.yaml @@ -27,6 +27,7 @@ spec: app.kubernetes.io/name: gameplane-audit-syslog-bridge {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" $b | nindent 6 }} securityContext: runAsNonRoot: true runAsUser: 65532 diff --git a/charts/gameplane/templates/crd-apply-hook.yaml b/charts/gameplane/templates/crd-apply-hook.yaml index b767b4e28..4ad3925cb 100644 --- a/charts/gameplane/templates/crd-apply-hook.yaml +++ b/charts/gameplane/templates/crd-apply-hook.yaml @@ -194,6 +194,7 @@ spec: app.kubernetes.io/name: gameplane-crd-apply {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" .Values.crds.autoApply | nindent 6 }} serviceAccountName: {{ .Release.Name }}-crd-apply restartPolicy: Never securityContext: diff --git a/charts/gameplane/templates/diagnostics.yaml b/charts/gameplane/templates/diagnostics.yaml new file mode 100644 index 000000000..2f886951e --- /dev/null +++ b/charts/gameplane/templates/diagnostics.yaml @@ -0,0 +1,122 @@ +{{- $d := include "gameplane.nodeDiagnostics" . | fromJson }} +{{- if $d.enabled }} +# This namespace is an explicit host-privilege exception. Never share it with +# games or the control plane; the worker has no Kubernetes token or host mount. +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $d.namespace }} + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/enforce-version: {{ .Values.podSecurity.version | default "v1.34" | quote }} +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: gameplane-diagnostics + namespace: {{ $d.namespace }} +spec: + hard: + persistentvolumeclaims: {{ $d.maxCaptures | quote }} + requests.storage: {{ $d.storageQuota | quote }} + pods: {{ mul (int $d.maxCaptures) 2 | quote }} + count/jobs.batch: {{ $d.maxCaptures | quote }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-diagnostics-namespace +rules: + - apiGroups: [""] + resources: [namespaces] + resourceNames: [{{ $d.namespace | quote }}] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-diagnostics-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-diagnostics-namespace +subjects: + - kind: ServiceAccount + name: gameplane-operator + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gameplane-diagnostics + namespace: {{ $d.namespace }} +rules: + - apiGroups: [""] + resources: [pods, services, persistentvolumeclaims, configmaps, secrets] + verbs: [get, list, create, update, patch, delete] + - apiGroups: ["batch"] + resources: [jobs] + verbs: [get, list, create, update, patch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gameplane-diagnostics + namespace: {{ $d.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gameplane-diagnostics +subjects: + - kind: ServiceAccount + name: gameplane-operator + namespace: {{ .Release.Namespace }} +--- +# Files remain behind mTLS and UID-bound routes. Workers need no network +# egress: their AF_PACKET socket receives traffic in the selected game netns. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gameplane-diagnostics + namespace: {{ $d.namespace }} +spec: + podSelector: {} + policyTypes: [Ingress, Egress] + egress: [] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + podSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: [gameplane-operator, gameplane-api, gameplane-gateway] + ports: + - { protocol: TCP, port: 9091 } +{{- if (.Values.gateway | default dict).enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gameplane-gateway-diagnostics + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gameplane-gateway + app.kubernetes.io/instance: {{ .Release.Name }} + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ $d.namespace }} + podSelector: + matchLabels: + gameplane.local/capture-component: artifacts + ports: + - { protocol: TCP, port: 9091 } +{{- end }} +{{- end }} diff --git a/charts/gameplane/templates/gateway.yaml b/charts/gameplane/templates/gateway.yaml index 82c46613e..aeab6bafc 100644 --- a/charts/gameplane/templates/gateway.yaml +++ b/charts/gameplane/templates/gateway.yaml @@ -113,6 +113,7 @@ spec: app.kubernetes.io/name: gameplane-gateway {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" $gateway | nindent 6 }} serviceAccountName: gameplane-gateway securityContext: runAsNonRoot: true diff --git a/charts/gameplane/templates/mcp-server.yaml b/charts/gameplane/templates/mcp-server.yaml index 3858878a3..40051e993 100644 --- a/charts/gameplane/templates/mcp-server.yaml +++ b/charts/gameplane/templates/mcp-server.yaml @@ -66,6 +66,7 @@ spec: app.kubernetes.io/name: gameplane-mcp-server {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" .Values.mcpServer | nindent 6 }} serviceAccountName: gameplane-mcp-server securityContext: runAsNonRoot: true diff --git a/charts/gameplane/templates/namespaces.yaml b/charts/gameplane/templates/namespaces.yaml index 8642b33dd..b73cce0bd 100644 --- a/charts/gameplane/templates/namespaces.yaml +++ b/charts/gameplane/templates/namespaces.yaml @@ -7,7 +7,7 @@ metadata: {{- include "gameplane.labels" . | nindent 4 }} {{- if .Values.podSecurity.enforceRestricted }} pod-security.kubernetes.io/enforce: restricted - pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/enforce-version: {{ .Values.podSecurity.version | default "v1.34" | quote }} {{- end }} annotations: # F-212: exclude the games Namespace from Helm's ownership so diff --git a/charts/gameplane/templates/operator-strategy-migrate.yaml b/charts/gameplane/templates/operator-strategy-migrate.yaml new file mode 100644 index 000000000..e376fca4b --- /dev/null +++ b/charts/gameplane/templates/operator-strategy-migrate.yaml @@ -0,0 +1,85 @@ +{{- /* SSA cannot remove a server-defaulted rollingUpdate field. Patch it before +Helm applies Recreate, including installations that manage CRDs externally. */ -}} +{{- if and .Release.IsUpgrade (eq (include "gameplane.operatorEnabled" .) "true") (not .Values.operator.leaderElect) }} +{{- $existing := lookup "apps/v1" "Deployment" .Release.Namespace "gameplane-operator" }} +{{- if $existing }} +{{- $name := printf "%s-op-roll" .Release.Name }} +{{- $annotations := dict "helm.sh/hook" "pre-upgrade" "helm.sh/hook-delete-policy" "before-hook-creation,hook-succeeded" }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gameplane.labels" . | nindent 4 }} + annotations: {{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $annotations) | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gameplane.labels" . | nindent 4 }} + annotations: {{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $annotations) | nindent 4 }} +rules: + - apiGroups: [apps] + resources: [deployments] + resourceNames: [gameplane-operator] + verbs: [get, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gameplane.labels" . | nindent 4 }} + annotations: {{- toYaml (merge (dict "helm.sh/hook-weight" "-5") $annotations) | nindent 4 }} +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: {{ $name }} } +subjects: + - kind: ServiceAccount + name: {{ $name }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $name }} + namespace: {{ .Release.Namespace }} + labels: {{- include "gameplane.labels" . | nindent 4 }} + annotations: {{- toYaml (merge (dict "helm.sh/hook-weight" "-4") $annotations) | nindent 4 }} +spec: + backoffLimit: 2 + ttlSecondsAfterFinished: 300 + template: + spec: + {{- include "gameplane.placement" .Values.crds.autoApply | nindent 6 }} + serviceAccountName: {{ $name }} + restartPolicy: Never + securityContext: + runAsNonRoot: true + runAsUser: 65532 + fsGroup: 65532 + seccompProfile: { type: RuntimeDefault } + containers: + - name: kubectl + image: {{ .Values.crds.autoApply.image | quote }} + imagePullPolicy: {{ .Values.crds.autoApply.pullPolicy }} + command: [kubectl, patch, deployment, gameplane-operator, --type=merge, -p, '{"spec":{"strategy":{"type":"Recreate","rollingUpdate":null}}}'] + env: + - { name: HOME, value: /tmp } + resources: + requests: { cpu: 50m, memory: 64Mi } + limits: { cpu: 500m, memory: 256Mi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: [ALL] } + volumeMounts: + - { name: tmp, mountPath: /tmp } + volumes: + - name: tmp + emptyDir: {} + {{- with .Values.image.pullSecrets }} + imagePullSecrets: {{ toYaml . | nindent 8 }} + {{- end }} +{{- end }} +{{- end }} diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 772e0b10e..66cecce24 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -137,7 +137,7 @@ rules: resources: [gameservers/status, backups/status, backupschedules/status, restores/status, networkcaptures/status] verbs: [get, update, patch] - apiGroups: ["gameplane.local"] - resources: [gameservers/finalizers, backups/finalizers, backupschedules/finalizers, restores/finalizers] + resources: [gameservers/finalizers, backups/finalizers, backupschedules/finalizers, restores/finalizers, networkcaptures/finalizers] verbs: [update] - apiGroups: ["apps"] resources: [statefulsets, deployments] @@ -239,6 +239,17 @@ metadata: labels: {{- include "gameplane.labels" . | nindent 4 }} spec: replicas: {{ .Values.operator.replicas }} + {{- if not .Values.operator.leaderElect }} + # Without leader election, prevent a rolling update from overlapping writers. + strategy: + type: Recreate + rollingUpdate: null + {{- else }} + # Explicitly restore rolling updates when election is re-enabled after a + # singleton install; the migration hook may own the previous Recreate field. + strategy: + type: RollingUpdate + {{- end }} selector: matchLabels: app.kubernetes.io/name: gameplane-operator @@ -250,6 +261,7 @@ spec: annotations: checksum/mtls: {{ include "gameplane.mtlsChecksum" . }} spec: + {{- include "gameplane.placement" .Values.operator | nindent 6 }} serviceAccountName: gameplane-operator securityContext: runAsNonRoot: true @@ -261,6 +273,7 @@ spec: image: {{ include "gameplane.operatorImage" . }} imagePullPolicy: {{ .Values.image.pullPolicy }} args: + {{- include "gameplane.auxiliaryResourceArgs" .Values.operator | nindent 12 }} {{- if .Values.operator.leaderElect }} - --leader-elect {{- end }} @@ -314,9 +327,16 @@ spec: - --agent-ca-secret-name={{ .Values.api.agentMTLS.caSecretRef.name }} - --agent-ca-secret-namespace={{ .Release.Namespace }} - --module-namespace={{ .Release.Namespace }} + - --restricted-workloads={{ .Values.podSecurity.enforceRestricted }} + - --agent-token-expiration-seconds={{ .Values.operator.agentTokenExpirationSeconds | default 3600 }} {{- if and .Values.capture .Values.capture.enabled }} - --capture-enabled=true - --capture-sidecar-image={{ include "gameplane.captureImage" . }} + {{- $diagnostics := include "gameplane.nodeDiagnostics" . | fromJson }} + {{- if $diagnostics.enabled }} + - --diagnostics-namespace={{ $diagnostics.namespace }} + - --diagnostics-storage-class={{ $diagnostics.storageClass | default "" }} + {{- end }} - --capture-default-retention-seconds={{ .Values.capture.defaultRetentionSeconds | int64 }} - --capture-max-retention-seconds={{ .Values.capture.maxRetentionSeconds | int64 }} - --capture-default-max-duration-seconds={{ .Values.capture.defaultMaxDurationSeconds | int64 }} diff --git a/charts/gameplane/templates/poddisruptionbudgets.yaml b/charts/gameplane/templates/poddisruptionbudgets.yaml new file mode 100644 index 000000000..35667e143 --- /dev/null +++ b/charts/gameplane/templates/poddisruptionbudgets.yaml @@ -0,0 +1,47 @@ +{{- $apiEnabled := eq (include "gameplane.apiEnabled" .) "true" -}} +{{- $bridge := .Values.api.audit.webhook.syslogBridge -}} +{{- $receiver := .Values.api.telemetry.receiver -}} +{{- $gateway := .Values.gateway | default dict -}} +{{- $components := list + (dict "name" "operator" "config" .Values.operator "enabled" (eq (include "gameplane.operatorEnabled" .) "true")) + (dict "name" "api" "config" .Values.api "enabled" $apiEnabled) + (dict "name" "web" "config" .Values.web "enabled" (and $apiEnabled .Values.web.enabled)) + (dict "name" "mcp-server" "config" .Values.mcpServer "enabled" .Values.mcpServer.enabled) + (dict "name" "gateway" "config" $gateway "enabled" $gateway.enabled) + (dict "name" "audit-syslog-bridge" "config" $bridge "enabled" (and $apiEnabled $bridge.enabled (not .Values.api.audit.webhook.url))) + (dict "name" "telemetry-receiver" "config" $receiver "enabled" (and $apiEnabled $receiver.enabled (not .Values.api.telemetry.endpoint))) -}} +{{- range $components -}} +{{- $pdb := .config.pdb | default dict -}} +{{- if and .enabled $pdb.enabled -}} +{{- $minimum := not (kindIs "invalid" $pdb.minAvailable) -}} +{{- $maximum := not (kindIs "invalid" $pdb.maxUnavailable) -}} +{{- if eq .name "api" -}} +{{- if or $minimum (and $maximum (ne (toString $pdb.maxUnavailable) "1")) -}} +{{- fail "api.pdb must allow the singleton API to be evicted: use maxUnavailable: 1 without minAvailable" -}} +{{- end -}} +{{- else if and $minimum $maximum -}} +{{- fail (printf "%s.pdb must set only one of minAvailable or maxUnavailable" .name) -}} +{{- end }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: gameplane-{{ .name }} + namespace: {{ $.Release.Namespace }} + labels: {{- include "gameplane.labels" $ | nindent 4 }} +spec: + {{- if eq .name "api" }} + maxUnavailable: 1 + {{- else if $minimum }} + minAvailable: {{ toYaml $pdb.minAvailable }} + {{- else if $maximum }} + maxUnavailable: {{ toYaml $pdb.maxUnavailable }} + {{- else }} + maxUnavailable: 1 + {{- end }} + selector: + matchLabels: + app.kubernetes.io/name: gameplane-{{ .name }} + app.kubernetes.io/instance: {{ $.Release.Name }} +{{- end -}} +{{- end -}} diff --git a/charts/gameplane/templates/telemetry-receiver.yaml b/charts/gameplane/templates/telemetry-receiver.yaml index 4e8255fbc..18a084d32 100644 --- a/charts/gameplane/templates/telemetry-receiver.yaml +++ b/charts/gameplane/templates/telemetry-receiver.yaml @@ -88,6 +88,7 @@ spec: app.kubernetes.io/name: gameplane-telemetry-receiver {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" $r | nindent 6 }} securityContext: runAsNonRoot: true runAsUser: 65532 diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index 79b5e17b6..d29173bc8 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,3 +1,21 @@ +{{- $diagnostics := include "gameplane.nodeDiagnostics" . | fromJson }} +{{- if $diagnostics.enabled }} +{{- if or (not .Values.capture.enabled) (ne (include "gameplane.operatorEnabled" .) "true") }} +{{- fail "capture.nodeDiagnostics requires capture.enabled and the operator" }} +{{- end }} +{{- if or (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $diagnostics.namespace)) (gt (len $diagnostics.namespace) 63) (eq $diagnostics.namespace .Values.gamesNamespace) (eq $diagnostics.namespace .Release.Namespace) }} +{{- fail "capture.nodeDiagnostics.namespace must be a separate explicit DNS-label namespace" }} +{{- end }} +{{- if lt (int $diagnostics.maxCaptures) 1 }} +{{- fail "capture.nodeDiagnostics.maxCaptures must be positive" }} +{{- end }} +{{- end }} +{{- if not (regexMatch "^v1\\.[0-9]+$" (.Values.podSecurity.version | default "v1.34")) }} +{{- fail "podSecurity.version must pin a Kubernetes minor such as v1.34" }} +{{- end }} +{{- if and (eq (include "gameplane.operatorEnabled" .) "true") (gt (int .Values.operator.replicas) 1) (not .Values.operator.leaderElect) }} +{{- fail "operator.leaderElect must be enabled with more than one replica" }} +{{- end }} {{- if .Values.api.standalone }} {{- if ne (include "gameplane.apiEnabled" .) "true" }} {{- fail "api.standalone requires api.enabled=true" }} diff --git a/charts/gameplane/templates/web.yaml b/charts/gameplane/templates/web.yaml index 4429fc5e4..736fd7136 100644 --- a/charts/gameplane/templates/web.yaml +++ b/charts/gameplane/templates/web.yaml @@ -25,6 +25,7 @@ spec: app.kubernetes.io/name: gameplane-web {{- include "gameplane.labels" . | nindent 8 }} spec: + {{- include "gameplane.placement" .Values.web | nindent 6 }} automountServiceAccountToken: false securityContext: runAsNonRoot: true diff --git a/charts/gameplane/values-production.example.yaml b/charts/gameplane/values-production.example.yaml new file mode 100644 index 000000000..229448ac3 --- /dev/null +++ b/charts/gameplane/values-production.example.yaml @@ -0,0 +1,36 @@ +# Example policy, not a capacity recommendation. Label/taint worker nodes before +# applying this profile; see docs/production-placement.md. Never remove or +# tolerate control-plane taints just to satisfy these selectors. +operator: + replicas: 2 + leaderElect: true + nodeSelector: &managementNodes { gameplane.local/pool: management } + tolerations: &managementTolerations + - { key: gameplane.local/pool, operator: Equal, value: management, effect: NoSchedule } + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: { app.kubernetes.io/name: gameplane-operator } + pdb: { enabled: true, maxUnavailable: 1 } +api: + replicas: 1 + nodeSelector: *managementNodes + tolerations: *managementTolerations + pdb: { enabled: true, maxUnavailable: 1 } +web: + replicas: 2 + nodeSelector: *managementNodes + tolerations: *managementTolerations + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: { app.kubernetes.io/name: gameplane-web } + pdb: { enabled: true, maxUnavailable: 1 } +crds: + autoApply: + nodeSelector: *managementNodes + tolerations: *managementTolerations diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 66d95af0b..ff3cc1751 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -39,6 +39,12 @@ image: crds: autoApply: enabled: true + # The upgrade hook must be schedulable before new Deployments roll out. + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" # kubectl image the apply Job runs. Any image with kubectl on PATH works; # retag to a private mirror for air-gapped installs (needed at upgrade # time, and also on an install over CRDs an earlier, uninstalled release @@ -50,20 +56,33 @@ crds: pullPolicy: IfNotPresent operator: + # Bound, rotating agent-only token; 600 supports short rotation drills. + agentTokenExpirationSeconds: 3600 # Disable the operator and bundled native CRDs for a standalone central panel. enabled: true replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } # optional minAvailable OR maxUnavailable; defaults to maxUnavailable: 1 logLevel: info # zap log level: debug|info|error (warn not supported by zap's --zap-log-level) agentLogLevel: "" # injected into every agent sidecar as GAMEPLANE_LOG_LEVEL (debug|info|warn|error); empty injects nothing, avoiding StatefulSet rollouts on upgrade - # Leader election guards against two operators reconciling at once. With a - # single replica it's unnecessary, and on a resource-starved node a missed - # lease renewal makes controller-runtime exit (the operator then crash-loops - # and stops reconciling). Leave on for HA installs; turn off for single-replica - # or constrained clusters (the e2e install sets it false). + # Required with multiple replicas and recommended during rolling upgrades. + # Disabling it with one replica selects Recreate to prevent overlapping writers. leaderElect: true resources: requests: { cpu: 100m, memory: 64Mi } limits: { cpu: 500m, memory: 256Mi } + # Complete auxiliary ResourceRequirements overrides. Null/missing uses binary + # defaults, including on --reuse-values upgrades. {} (or requests: {}, limits: + # {}) clears defaults. No new CPU/memory limits are set by default. + agentResources: null # requests 25m CPU / 64Mi memory; also config-init + sentinelResources: null # requests 25m CPU / 32Mi memory + tunnelResources: null # requests 25m CPU / 64Mi memory (every provider) + backupResources: null # backup/init/restore requests 100m CPU / 128Mi memory; + # cleanup retains 50m / 128Mi, memory limit 512Mi agentImage: "" # defaults to {image.registry}/agent:{image.tag} sentinelImage: "" # defaults to {image.registry}/sentinel:{image.tag} # Which load-balancer address manager runs in this cluster, so the operator @@ -133,6 +152,14 @@ api: # Standalone mode also skips the CRD apply hook and its RBAC. standalone: false replicas: 1 # Keep at 1. SQLite is single-writer on a RWO volume; with db.driver=postgres the user-management lock and audit hash chain are still per-process + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + # The API is a singleton: eviction causes downtime. A blocking budget would + # hang voluntary node drains, so only maxUnavailable: 1 is supported here. + pdb: { enabled: false, maxUnavailable: 1 } logLevel: info # API log verbosity (debug|info|warn|error), passed as GAMEPLANE_LOG_LEVEL # Port of the API's Prometheus metrics listener. It is separate from the # public API port (the one the ingress and the web front end route to), so @@ -241,6 +268,12 @@ api: facility: local0 # RFC 5424 facility: kern/user/daemon/auth/.../local0..local7 severity: info # emerg/alert/crit/err/warning/notice/info/debug replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: {} # Native S3-compatible audit sink. When endpoint and bucket are set, the API # batches audit events as NDJSON and uploads them to S3 (or MinIO, Backblaze, @@ -314,6 +347,12 @@ api: # Must stay 1 while persistence is enabled: the SQLite file on the PVC # has a single writer, so the template fails the render otherwise. replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: {} # Aggregate storage (SQLite on a PVC mounted at /data). With # persistence disabled an emptyDir is used instead and every @@ -422,6 +461,12 @@ api: gateway: enabled: false replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } # Must match the Cluster resource name in the central Gameplane installation. clusterID: "" # Exact URI SAN trusted in the central API client certificate. @@ -465,6 +510,12 @@ gateway: web: enabled: true replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: requests: { cpu: 10m, memory: 16Mi } limits: { cpu: 200m, memory: 128Mi } @@ -602,6 +653,12 @@ clusterOps: mcpServer: enabled: false replicas: 1 + nodeSelector: {} + tolerations: [] + affinity: {} + topologySpreadConstraints: [] + priorityClassName: "" + pdb: { enabled: false } resources: requests: { cpu: 50m, memory: 32Mi } limits: { cpu: 200m, memory: 128Mi } @@ -654,7 +711,8 @@ grafanaDashboards: grafana_dashboard: "1" podSecurity: - enforceRestricted: false # label gamesNamespace with pod-security.kubernetes.io/enforce=restricted + enforceRestricted: false # require compatible nonroot templates and enforce Restricted game pods + version: v1.34 # pinned admission policy; change after testing workload compatibility # Default ModuleSource shipped with the chart. Disable when you manage # ModuleSource resources via GitOps. The operator indexes this source on a @@ -742,6 +800,14 @@ uploadModuleSource: # opt in has no capture sidecar attached. Off by default. capture: enabled: false + # Separate host-capable Linux/containerd worker; each server also needs an + # audited administrator opt-in. Never enables privileges in game Pods. + nodeDiagnostics: + enabled: false + namespace: gameplane-diagnostics + storageClass: "" + maxCaptures: 10 + storageQuota: 10Gi # Default retention window for completed captures (seconds). Per-server overrides # are clamped to the cluster maximum below; this is the default when a server does # not specify a retention policy. diff --git a/charts/tests/check_workload_security.py b/charts/tests/check_workload_security.py new file mode 100644 index 000000000..5718f7ae7 --- /dev/null +++ b/charts/tests/check_workload_security.py @@ -0,0 +1,65 @@ +"""CI-only chart checks for the opt-in Restricted workload and node exception.""" +from pathlib import Path +import shutil +import tempfile + +import yaml + +from check_gateway import CHART, one, render, settings + + +def check_reuse_values(): + with tempfile.TemporaryDirectory() as directory: + chart = Path(directory) / "chart" + shutil.copytree(CHART, chart) + values_file = chart / "values.yaml" + old_values = yaml.safe_load(values_file.read_text()) + old_values["capture"].pop("nodeDiagnostics", None) + values_file.write_text(yaml.safe_dump(old_values)) + enabled = {"capture": {"enabled": True, "nodeDiagnostics": {"enabled": True}}} + objects = render(enabled, upgrade=True, chart=chart) + one(objects, "Namespace", "gameplane-diagnostics") + quota = one(objects, "ResourceQuota", "gameplane-diagnostics")["spec"]["hard"] + assert quota == {"persistentvolumeclaims": "10", "requests.storage": "10Gi", "pods": "20", "count/jobs.batch": "10"} + args = one(objects, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert "--diagnostics-namespace=gameplane-diagnostics" in args + enabled["capture"]["nodeDiagnostics"]["maxCaptures"] = 0 + render(enabled, upgrade=True, chart=chart, failure="maxCaptures must be positive") + enabled["capture"]["nodeDiagnostics"].pop("maxCaptures") + enabled["capture"]["nodeDiagnostics"]["namespace"] = "" + render(enabled, upgrade=True, chart=chart, failure="separate explicit") + + +def main(): + check_reuse_values() + defaults = render() + assert not any(o["metadata"]["name"] == "gameplane-diagnostics" for o in defaults) + objects = render({"podSecurity": {"enforceRestricted": True}}) + games = one(objects, "Namespace", "gameplane-games") + assert games["metadata"]["labels"]["pod-security.kubernetes.io/enforce"] == "restricted" + assert games["metadata"]["labels"]["pod-security.kubernetes.io/enforce-version"] == "v1.34" + args = one(objects, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert "--restricted-workloads=true" in args # even when capture is disabled + assert not any(a.startswith("--diagnostics-namespace=") for a in args) + values = {"podSecurity": {"enforceRestricted": True, "version": "v1.35"}, + "capture": {"enabled": True, "nodeDiagnostics": {"enabled": True}}, + "gateway": settings()} + objects = render(values) + assert one(objects, "Namespace", "gameplane-games")["metadata"]["labels"]["pod-security.kubernetes.io/enforce-version"] == "v1.35" + assert one(objects, "Namespace", "gameplane-diagnostics")["metadata"]["labels"]["pod-security.kubernetes.io/enforce"] == "privileged" + quota = one(objects, "ResourceQuota", "gameplane-diagnostics")["spec"]["hard"] + assert quota["persistentvolumeclaims"] == "10" and quota["requests.storage"] == "10Gi" + role = one(objects, "Role", "gameplane-diagnostics") + assert role["metadata"]["namespace"] == "gameplane-diagnostics" + assert all("*" not in r["resources"] for r in role["rules"]) + policy = one(objects, "NetworkPolicy", "gameplane-diagnostics")["spec"] + assert policy["egress"] == [] and policy["ingress"][0]["ports"] == [{"protocol": "TCP", "port": 9091}] + one(objects, "NetworkPolicy", "gameplane-gateway-diagnostics") + for namespace in ("gameplane-games", "gameplane-system", "../host"): + render({"capture": {"enabled": True, "nodeDiagnostics": {"enabled": True, "namespace": namespace}}}, failure="separate explicit") + render({"capture": {"nodeDiagnostics": {"enabled": True}}}, failure="requires capture.enabled") + render({"podSecurity": {"version": "latest"}}, failure="must pin") + + +if __name__ == "__main__": + main() diff --git a/charts/tests/check_workloads.py b/charts/tests/check_workloads.py new file mode 100644 index 000000000..a4c433528 --- /dev/null +++ b/charts/tests/check_workloads.py @@ -0,0 +1,112 @@ +"""CI-only resource override, management placement and disruption checks.""" +import json +from pathlib import Path +import shutil +import tempfile + +import yaml + +from check_gateway import CHART, one, render, settings + + +def check_operator_migration(): + """Exercise live-lookup hook output with the longest valid Helm name.""" + with tempfile.TemporaryDirectory() as directory: + chart = Path(directory) / "chart" + shutil.copytree(CHART, chart) + template = chart / "templates/operator-strategy-migrate.yaml" + source = template.read_text() + lookup = 'lookup "apps/v1" "Deployment" .Release.Namespace "gameplane-operator"' + assert lookup in source + # helm template has no API server. Supply only the existing-object + # lookup response; leave the actual hook gates and resources intact. + template.write_text(source.replace(lookup, 'dict "metadata" (dict "name" "gameplane-operator")')) + release = "r" * 53 + name = release + "-op-roll" + values = {"api": {"enabled": False}, "operator": {"leaderElect": False}, + "crds": {"autoApply": {"enabled": False, "nodeSelector": {"pool": "management"}}}} + objects = render(values, upgrade=True, release=release, chart=chart) + job = one(objects, "Job", name) + assert len(job["metadata"]["name"]) <= 63 + pod = job["spec"]["template"]["spec"] + assert pod["serviceAccountName"] == name + assert pod["nodeSelector"] == {"pool": "management"} + assert one(objects, "Role", name)["rules"] == [{"apiGroups": ["apps"], "resources": ["deployments"], + "resourceNames": ["gameplane-operator"], "verbs": ["get", "patch"]}] + binding = one(objects, "RoleBinding", name) + assert binding["roleRef"]["name"] == name and binding["subjects"][0]["name"] == name + one(objects, "ServiceAccount", name) + assert not any(obj["kind"] == "Job" for obj in render(values, release=release, chart=chart)) + values["operator"]["leaderElect"] = True + assert not any(obj["kind"] == "Job" for obj in render(values, upgrade=True, release=release, chart=chart)) + + +def main(): + check_operator_migration() + defaults = render() + assert one(defaults, "Deployment", "gameplane-operator")["spec"]["strategy"]["type"] == "RollingUpdate" + assert not any(obj["kind"] == "PodDisruptionBudget" for obj in defaults) + args = one(defaults, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert not any(arg.startswith("--tunnel-resources=") for arg in args) + placement = { + "nodeSelector": {"gameplane.local/pool": "management"}, + "tolerations": [{"key": "gameplane.local/pool", "operator": "Equal", "value": "management", "effect": "NoSchedule"}], + "affinity": {"podAntiAffinity": {"preferredDuringSchedulingIgnoredDuringExecution": [{"weight": 100, "podAffinityTerm": {"topologyKey": "kubernetes.io/hostname", "labelSelector": {"matchLabels": {"app.kubernetes.io/instance": "gameplane"}}}}]}}, + "topologySpreadConstraints": [{"maxSkew": 1, "topologyKey": "kubernetes.io/hostname", "whenUnsatisfiable": "DoNotSchedule", "labelSelector": {"matchLabels": {"app.kubernetes.io/instance": "gameplane"}}}], + "priorityClassName": "management", + } + component = dict(placement, pdb={"enabled": True, "maxUnavailable": 1}) + gateway = dict(settings(), **component) + values = { + "operator": dict(component, replicas=2, tunnelResources={"requests": {}, "limits": {}}, agentResources={"requests": {"cpu": "80m"}}), + "api": dict(component, audit={"webhook": {"syslogBridge": dict(component, enabled=True, syslog={"addr": "logs:514"})}}, telemetry={"receiver": dict(component, enabled=True)}), + "web": component, "mcpServer": dict(component, enabled=True), "gateway": gateway, + "crds": {"autoApply": placement}, + } + production = render(values, upgrade=True) + for name in ["operator", "api", "web", "mcp-server", "gateway", "audit-syslog-bridge", "telemetry-receiver"]: + deployment = one(production, "Deployment", "gameplane-" + name) + spec = deployment["spec"]["template"]["spec"] + for key, expected in placement.items(): + assert spec[key] == expected, (name, key) + pdb = one(production, "PodDisruptionBudget", "gameplane-" + name)["spec"] + assert pdb["maxUnavailable"] == 1 and "minAvailable" not in pdb + assert pdb["selector"]["matchLabels"].items() <= deployment["spec"]["template"]["metadata"]["labels"].items() + hook = one(production, "Job", "gameplane-crd-apply")["spec"]["template"]["spec"] + for key, expected in placement.items(): + assert hook[key] == expected + args = one(production, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + overrides = {arg.split("=", 1)[0]: json.loads(arg.split("=", 1)[1]) for arg in args if arg.startswith(("--agent-resources=", "--tunnel-resources="))} + assert overrides["--tunnel-resources"] == {"requests": {}, "limits": {}} + assert overrides["--agent-resources"] == {"requests": {"cpu": "80m"}} + render({"api": {"pdb": {"enabled": True, "minAvailable": 1}}}, failure="api.pdb must allow the singleton API to be evicted") + render({"api": {"pdb": {"enabled": True, "maxUnavailable": 0}}}, failure="api.pdb must allow the singleton API to be evicted") + render({"operator": {"replicas": 2, "leaderElect": False}}, failure="operator.leaderElect must be enabled") + single = render({"operator": {"leaderElect": False}}) + assert one(single, "Deployment", "gameplane-operator")["spec"]["strategy"] == {"type": "Recreate", "rollingUpdate": None} + removed = render({"operator": {"agentResources": None, "tunnelResources": {}}}) + args = one(removed, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert not any(arg.startswith("--agent-resources=") for arg in args) + assert "--tunnel-resources={}" in args + disabled = render({"api": {"enabled": False, "pdb": {"enabled": True}}, "web": {"pdb": {"enabled": True}}}) + assert not any(obj["kind"] == "PodDisruptionBudget" for obj in disabled) + profile = yaml.safe_load((CHART / "values-production.example.yaml").read_text()) + example = render(profile, upgrade=True) + assert one(example, "Deployment", "gameplane-operator")["spec"]["replicas"] == 2 + assert one(example, "PodDisruptionBudget", "gameplane-api")["spec"]["maxUnavailable"] == 1 + # Helm --reuse-values may not contain new keys. Omission must retain binary + # resource defaults and empty placement without indexing a missing map. + with tempfile.TemporaryDirectory() as directory: + old_chart = Path(directory) / "chart" + shutil.copytree(CHART, old_chart) + old_values = yaml.safe_load((old_chart / "values.yaml").read_text()) + for key in ["agentResources", "sentinelResources", "tunnelResources", "backupResources", "pdb", *placement]: + old_values["operator"].pop(key, None) + (old_chart / "values.yaml").write_text(yaml.safe_dump(old_values)) + old = render(chart=old_chart) + args = one(old, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert not any(arg.startswith("--tunnel-resources=") for arg in args) + + +if __name__ == "__main__": + main() diff --git a/deploy/kind/e2e.sh b/deploy/kind/e2e.sh index 3c2f7b12f..8f5df5211 100755 --- a/deploy/kind/e2e.sh +++ b/deploy/kind/e2e.sh @@ -6,7 +6,7 @@ # deploy/kind/e2e.sh down [cluster-name] # # Diffs from dev-up (deploy/kind/up.sh): -# - Single-node cluster (faster boot, sufficient for E2E coverage). +# - Single-node by default; management profile adds two scheduling workers. # - Skips ingress-nginx (the dashboard isn't exercised here). # - Loads pre-built gameplane-test/{operator,api,agent,sentinel,capture-sidecar,telemetry-receiver,fakeoidc}: # images (and gameplane-test/gameprobe: when present). @@ -28,9 +28,9 @@ CLUSTER="${2:-gameplane-e2e}" TAG="${3:-${GAMEPLANE_E2E_TAG:-e2e}}" PROFILE="${4:-default}" case "${PROFILE}" in -default | postgres) ;; +default | postgres | management) ;; *) - echo "unknown profile '${PROFILE}' (want default or postgres)" >&2 + echo "unknown profile '${PROFILE}' (want default, postgres or management)" >&2 exit 2 ;; esac @@ -188,12 +188,17 @@ up) echo "removing pre-existing cluster ${CLUSTER} for a clean slate" kind delete cluster --name "${CLUSTER}" || true fi - echo "creating kind cluster ${CLUSTER} (single-node)" + echo "creating kind cluster ${CLUSTER} (${PROFILE})" + extra_nodes="" + if [[ "${PROFILE}" == "management" ]]; then + extra_nodes=$' - role: worker\n - role: worker' + fi kind create cluster --name "${CLUSTER}" --wait 90s --config - </dev/null diff --git a/docs/install.md b/docs/install.md index b4da5a2a3..64efea6ee 100644 --- a/docs/install.md +++ b/docs/install.md @@ -198,7 +198,10 @@ Top-level knobs (see `values.yaml` for the full list): - `mcpServer.enabled` — optional strictly read-only MCP (Model Context Protocol) server [optional] for AI assistants to read cluster state and propose fixes (default off); see [mcp-server/README.md](../mcp-server/README.md) - `mcpServer.replicas` — MCP server replicas (default 1) - `updates.channel` — informational release-channel label (e.g., `stable`, `edge`) shown read-only in the dashboard's Admin Settings → Updates section; purely informational (Gameplane upgrades via Helm, not auto-update) -- `podSecurity.enforceRestricted` — label games namespace for Pod Security Standards +- `podSecurity.enforceRestricted` — enable the Restricted game/config-init profile and namespace admission. Templates require positive `security.runAsUser` and `security.fsGroup`; verify image compatibility and migrate existing file ownership before upgrading. Enabling it rolls existing Pods once. + - `podSecurity.version` — pinned PSA minor, default `v1.34`; administrator override for a tested cluster version. + - `operator.agentTokenExpirationSeconds` — projected agent-only token lifetime, 600–3600 seconds (default 3600); rotation is handled by client-go. + - `capture.nodeDiagnostics.enabled` — off by default. With `capture.enabled`, creates a separate privileged namespace and bounded diagnostic storage. An administrator must additionally opt each server in through `:capture-node-enable`. Linux/containerd only; no capture-related game restart. Read the [host-privilege and cleanup requirements](security.md#network-capture-security-exception) before deployment. - `defaultModuleSource.*` — the official game catalog shipped with the chart (enabled by default) - `defaultModuleSource.enabled` — whether to create the default `ModuleSource` (default `true`; disable when managing sources via GitOps) - `defaultModuleSource.name` — name of the `ModuleSource` resource (default `default`) diff --git a/docs/module-authoring.md b/docs/module-authoring.md index 1141965bb..a0553c207 100644 --- a/docs/module-authoring.md +++ b/docs/module-authoring.md @@ -375,6 +375,13 @@ re-pinning `spec.version`. ### Verifying and pinning bundles +Pin production `Module.spec.version` to avoid following catalog updates across +the fleet. Servers default to automatic template adoption; opt into +`GameServer.spec.templateRollout.policy: OnRequest` before installing a new +version to admit each server separately. See [template adoption and rollback](template-rollouts.md) +for baseline upgrade behavior, exact revision requests, retained PVCs and the +difference between reverting a runtime and restoring application data. + Two independent, opt-in controls harden the supply chain for OCI modules: **Digest pin** (`Module.spec.digest`) refuses to install unless the resolved @@ -1636,7 +1643,35 @@ useful, so readiness falls back to TCP on the game port. `spec.storage.mountPath` is where the PVC is mounted inside the game container. Pick the path the game writes its world/config files to — -backups snapshot this whole directory. +backups copy the primary claim mounted here. Separate PVCs mounted beneath +that directory are not included. + +Extra persistent volumes can declare their recovery role: + +```yaml +spec: + storage: + extra: + - name: serverfiles + mountPath: /serverfiles + size: 10Gi + backupRole: rebuildable # or data +``` + +`backupRole: data` identifies recovery data; `rebuildable` declares that the +contents can be downloaded or reconstructed. Neither includes the extra claim +in today's single-volume backup. Omission remains valid and appears as +unclassified/not covered. Do not mark an installation volume rebuildable if it +also contains unique configuration or mods that cannot be reconstructed. +The GameServer's existing `spec.storage.extra` override can supply the same field. + +The Backup pins its volume list, claim UIDs, observed game-container mount paths +and these declarations before copying. Template changes do not rewrite that +record. Nested loader/mod claims and retained server-owned claims remain +explicitly uncovered. See [backup coverage](operator-operations.md#backup-consistency-and-volume-coverage). +The module editor schema is derived from the GameTemplate CRD by +`make module-schema`; publish its regenerated copy in the module repository +when opting modules into this field. ## Testing a new module locally diff --git a/docs/operator-operations.md b/docs/operator-operations.md new file mode 100644 index 000000000..7c60b4fd9 --- /dev/null +++ b/docs/operator-operations.md @@ -0,0 +1,86 @@ +# Backup, restore and wipe ownership + +## Backup consistency and volume coverage + +A successful copy is not necessarily a complete server recovery point. +`status.conditions[type=Consistency]` reports the achieved result separately +from `Completed`: Quiesced means the agent confirmed the module's quiesce +commands before the completed copy; it does not prove those commands flush all +application state. UnsupportedRawCopy, RawCopy and AgentUnavailableRawCopy +describe restic copies without confirmed quiesce. CSI without confirmed +quiesce reports CrashConsistentSnapshot. StoppedSource means the source was +already stopped and its writers drained before capture; the backup did not stop +it. Failed or unfinished copies, including completed Jobs still awaiting a +snapshot ID, cannot report achieved quiesce or stopped-source consistency. + +`status.captureManifest` is recorded once before the first copy worker is +created, after lifecycle ownership and quiesce preparation. It lists the primary +`-data` claim as included and every other observed persistent claim as +uncovered, including extra volumes, nested mod/loader mounts and retained +server-owned claims. It records claim UIDs, observed mount paths, source layout +and optional `data`/`rebuildable` declarations. Included describes copy scope; +check the Backup's phase and snapshot ID for copy success. A declaration that +an uncovered claim is rebuildable is not a recovery guarantee. + +The API returns this status and the backup details drawer displays consistency +and coverage. Capture metadata is immutable through ordinary reconciliation: +editing a template later never changes the historical list. This release does +not add multi-volume restic, automatic game stopping or a stricter consistency +request policy. + +Pre-upgrade captures have missing metadata or `state: LegacyUnknown`; their +Consistency condition is Unknown. The operator never reconstructs their history +from the current template. Before recovery, inspect the original restic snapshot +or CSI source claim, recover the original module/server configuration and arrange +separate recovery of extra/mod volumes. Use an isolated recovery server to +verify those prerequisites before replacing a live world. + +## Shared operation ownership + +Gameplane serializes these operations per GameServer. All backups participate, +including backups with `quiesce: false` and CSI VolumeSnapshots. A held operation +also defers idle sleep/wake, restart, and pod-template rollout. Requests remain +pending and reconciliation resumes after the operation's cleanup completes. + +The GameServer annotation `operations.gameplane.local/owner` identifies the +operation kind, name, UID, admitted generation and target UID. A wipe is identified +by the GameServer UID and its request token. The existing backup, restore and wipe +owner annotations remain compatibility markers. A `recovery` array identifies +older owners whose workers or quiesce obligations must be cleaned up first. + +Deleting an operation requests cleanup. Restic Jobs are canceled with foreground +deletion; live and terminating worker pods must drain before the owner is released. +Quiesced backups also owe a successful save-on call. The existing bounded Backup +deletion retry may release its finalizer, but an unconfirmed save-on leaves the +durable owner behind for recovery by the next operation. Time alone never releases +ownership. A pending CSI capture retains the current capture-drain behavior; a +separate CSI timeout change is tracked in issue #719. + +If an operation waits unexpectedly, inspect the referenced Backup/Restore, its +conditions, Jobs and worker pods. Resolve terminating workers, repository failures +or agent connectivity before retrying. Removing owner annotations or force-removing +finalizers cannot establish that a worker has stopped. A failed or canceled restore +leaves its target suspended for inspection. + +Upgrades recognize existing owner annotations and import backup recovery lineage. +Already-running legacy quiesced copies continue being observed until their workers +finish and cleanup succeeds. Legacy raw workers also block new admission. Conflicting +legacy guards fail closed until their existing operations finish cleanup. Keep a +single active operator version during upgrade; older binaries do not honor the +shared owner. No new CRD fields or Kubernetes permissions are required. + +CSI restores create a new target and reserve it for the Restore. Initial startup +is allowed under that reservation so storage using WaitForFirstConsumer can bind. +Other operations wait until the restore completes. Existing data-PVC restores +continue to require an observed stopped workload and drained writers. + +If initial startup fails or the Restore is deleted, cleanup first retires its +startup permission and suspends the target. The operation remains held until the +StatefulSet has observed zero replicas and every live or terminating game/PVC +writer has drained. Recovery of a force-deleted Restore applies the same check +before a successor can proceed, preserving the existing pod template while +requesting zero replicas. A successful Restore leaves its initial workload running. + +Stopped-source preparation (#716), Restic preparation and quiesce deadlines (#717), +CSI capture deadlines (#719), and staged template adoption (#731) are separate +changes building on this ownership boundary. diff --git a/docs/production-placement.md b/docs/production-placement.md new file mode 100644 index 000000000..175d23f3a --- /dev/null +++ b/docs/production-placement.md @@ -0,0 +1,147 @@ +# Resource reservations and management placement + +New installs and upgrades reserve resources for supporting containers. These +requests are scheduling floors, not measured capacity targets: + +| Container | CPU request | Memory request | +| --- | --- | --- | +| Agent and config initializer | 25m | 64Mi | +| Wake sentinel | 25m | 32Mi | +| Tunnel, every provider | 25m | 64Mi | +| Restic backup, repository initializer and restore | 100m | 128Mi | + +These defaults add no CPU or memory limits. CPU limits can throttle bursts; +memory limits can terminate the process when its working set exceeds them. +Repository forget/prune retains its existing 50m/128Mi requests and 512Mi memory +limit unless overridden. Init containers receive requests too; Kubernetes +accounts for their peak requirements when scheduling the Pod. + +## Overrides and upgrades + +The new chart keys `operator.agentResources`, `operator.sentinelResources`, +`operator.tunnelResources` and `operator.backupResources` each accept a complete +Kubernetes ResourceRequirements object. Backup resources also apply to restore +and forget/prune workers; agent resources also apply to the config initializer. +The matching operator flags are `--agent-resources`, `--sentinel-resources`, +`--tunnel-resources`, and `--backup-resources`, with JSON values. + +Null or a missing chart key uses the binary defaults, including older +`helm upgrade --reuse-values` configurations. A supplied object replaces defaults +wholesale. Clear reservations explicitly: + +```yaml +operator: + agentResources: { requests: {}, limits: {} } + sentinelResources: { requests: {}, limits: {} } + tunnelResources: { requests: {}, limits: {} } + backupResources: { requests: {}, limits: {} } +``` + +An empty object `{}` has the same effect. Setting a key back to `null` restores +the defaults. No chart-side default map is merged into the supplied object. +For example `agentResources: {requests: {memory: 96Mi}}` reserves only memory. + +Helm still merges user-supplied maps. When replacing a previously populated +override with empty maps, export the release's values, edit that complete file, +and upgrade with `--reset-values -f `. An empty nested map +combined with `--reuse-values` can retain the release's old resource entries. +Keep all unrelated install values in the edited file; resetting to chart defaults +alone is not a resource-only change. + +Existing `GameTemplate.spec.agent.resources` takes precedence when it declares +requests or limits, replacing the install agent object wholesale. An image-only +agent override keeps install defaults. Kubernetes may omit empty maps in a +template, so use explicit zero CPU/memory requests for a per-template zero +reservation, or the install-level empty object to clear all defaults. Game +container resources continue to use the existing template defaults and complete +`GameServer.spec.resources` override. Game placement fields are unchanged. + +Changing resources changes generated Pod templates. Existing game StatefulSets, +sentinels and tunnels roll according to their normal reconciliation policy; +new Restic Jobs use the new configuration. Schedule this upgrade when game pod +restarts are acceptable. Insufficient capacity can leave replacement pods or +Jobs Pending. Check requested resources and concurrent backups before upgrading. + +## Management workers + +The chart now exposes `nodeSelector`, `tolerations`, `affinity`, +`topologySpreadConstraints`, and `priorityClassName` beneath: + +- `operator`, `api`, `web`, `gateway`, and `mcpServer`. +- `api.audit.webhook.syslogBridge` and `api.telemetry.receiver`. +- `crds.autoApply` for the upgrade hook Job. + +Each Deployment also supports `pdb.enabled` and either `pdb.minAvailable` or +`pdb.maxUnavailable`; enabled budgets default to `maxUnavailable: 1`. PDBs select +both the component name and Helm release. Disabled components get no PDB. Hook +Jobs have placement settings but no disruption budget. + +The API remains one process for both SQLite and PostgreSQL. Its optional PDB +only permits `maxUnavailable: 1`, so a voluntary node drain can evict it and will +cause API downtime. A PDB is not an availability guarantee and does not prevent +node failures. The persistent telemetry receiver is also a singleton; avoid a +zero-unavailable budget unless intentionally accepting a blocked drain. + +Keep operator leader election enabled with redundant replicas and rolling +updates. The chart rejects multiple replicas with leader election disabled; +with one replica and election disabled it uses Recreate to avoid overlapping +reconcilers during upgrades. API replica/process restrictions remain unchanged. + +When switching an existing operator to Recreate, a pre-upgrade hook clears the +server-defaulted RollingUpdate fields before Helm applies the Deployment. The +hook uses `crds.autoApply.image` and its placement settings, including when CRDs +are managed externally. It can patch only `gameplane-operator` in the release +namespace and is skipped when that Deployment does not exist. Upgrades need +access to the configured kubectl image. + +The [example profile](../charts/gameplane/values-production.example.yaml) places +management services on labeled, tainted workers and spreads redundant operator +and dashboard pods. First label and taint the selected **worker** nodes: + +```sh +kubectl label node management-a management-b gameplane.local/pool=management +kubectl taint node management-a management-b gameplane.local/pool=management:NoSchedule +kubectl label node game-a game-b gameplane.local/pool=games +kubectl taint node game-a game-b gameplane.local/pool=games:NoSchedule +``` + +Use existing GameServer placement fields for game workers: + +```yaml +spec: + nodeSelector: { gameplane.local/pool: games } + tolerations: + - { key: gameplane.local/pool, operator: Equal, value: games, effect: NoSchedule } +``` + +Preserve control-plane taints. Avoid broad `operator: Exists` tolerations. +Optional management components need their corresponding placement values too. +Per-game sentinel and tunnel Deployments are operator-generated and are not +covered by these chart management placement keys. A priorityClassName must name +an existing PriorityClass. Unsatisfied hard selectors, taints or topology +constraints correctly leave pods Pending; do not silently relax them. + +Local or node-bound volumes cannot move just because a pod has a new selector. +Check API/receiver PVC node affinity before selecting management workers, and +game PVC topology before backups or moving game pods. Restic scheduling and +source preparation are separate from management placement. + +## Sizing and controlled-drain checks + +Collect per-container CPU/memory, pod requests, Pending events, game tick/query +latency, storage throughput and backup duration during busy periods and actual +backup concurrency. Use `kubectl top pods --containers`, scheduler events and +the available Prometheus metrics. Include headroom for one failed management +node, game startup, repository maintenance and replacement pods. No fixed server +count follows from these request values. + +In a disposable multi-worker environment, apply the example and verify every +management pod's node labels. Temporarily set a nonexistent selector and verify +Pending/FailedScheduling, then restore it. Cordon one management worker, evict +its redundant operator/web pods through the eviction API, and verify their PDB +allows only one unavailable replica while the remaining replica stays Ready. +Uncordon after checking replacements. A full `kubectl drain` also evicts the +singleton API and can interrupt access; account for local PVCs and validate +recovery before doing this in production. CI chart checks verify selectors, +PDB gates, resource clearing and the production profile; cluster behavior needs +the multi-node integration scenario, not just a successful render. diff --git a/docs/release-notes.md b/docs/release-notes.md new file mode 100644 index 000000000..f3da01f11 --- /dev/null +++ b/docs/release-notes.md @@ -0,0 +1,21 @@ +# Unreleased operational changes + +## Auxiliary resource requests and management placement (#724) + +Agent/config-init (25m CPU, 64Mi), sentinel (25m, 32Mi), tunnel (25m, 64Mi), and +Restic backup/init/restore (100m, 128Mi) now request resources on every install. +These are initial scheduling reservations, not workload capacity estimates. +No new CPU or memory limits are imposed. Existing forget/prune limits remain. + +Upgrades change generated Pod templates and can restart game/auxiliary pods or +leave them Pending when capacity is insufficient. Review capacity and schedule +the upgrade accordingly. The four `operator.*Resources` chart values provide +complete overrides; explicit empty objects clear defaults, while null/missing +keys use the defaults even with `--reuse-values`. + +Chart-managed pods now accept placement, topology-spread and priority settings, +with optional PDBs for Deployments. API remains a singleton and its PDB uses +`maxUnavailable: 1`; node drains can cause API downtime. Operator replicas greater +than one require leader election. Disabling election on a singleton uses Recreate. +See [production placement](production-placement.md) for upgrade controls and a +worker-pool example. diff --git a/docs/security.md b/docs/security.md index b50a6d01b..d6456ad81 100644 --- a/docs/security.md +++ b/docs/security.md @@ -367,13 +367,32 @@ the image's own default user, which may be root. The operator sets no container runtime's defaults. This keeps arbitrary third-party game images working unchanged. -Game pods are shaped per-template. For a hostile game module, enable -Pod Security Standards `restricted` on the games namespace via -`podSecurity.enforceRestricted=true`. +`podSecurity.enforceRestricted=true` enables a different workload profile as +well as Restricted admission. Game and config-init containers use a positive +template UID/GID, `runAsNonRoot`, no privilege escalation, `drop: [ALL]`, and +`RuntimeDefault` seccomp. The operator rejects templates without positive +`security.runAsUser` and `security.fsGroup` (and a positive `runAsGroup` when +supplied) and Hostport exposure before creating workloads; use NodePort or +LoadBalancer with Restricted admission. Administrators must choose a compatible +image and migrate existing data ownership; numeric identities cannot prove +image compatibility. Enabling the profile rolls existing game Pods once. PSA +is pinned to `v1.34`; `podSecurity.version` accepts another tested minor. +Config bootstrap uses the declared fsGroup as its primary group even on +volumes that do not apply fsGroup ownership automatically. Repeated starts +preserve already-correct group-write permissions on root-owned config files. +This does not repair arbitrary pre-existing data ownership after root access +has been removed; migrate that data before enabling the profile. + +In both profiles, automatic ServiceAccount mounts are disabled. Only the agent +gets a projected, rotating token, CA and namespace at the standard client-go +path. Game and config-init containers have no token mount. Shared PID remains +enabled for metrics: processes in one Pod still share a trust boundary. +`/proc`, matching UIDs, shared files or a compromised agent can undermine mount +isolation. Restricted is not a safe sandbox for hostile modules. ### Network Capture Security Exception -The optional network capture feature [optional] (see +The legacy optional network capture feature [optional] (see [`docs/roadmap.md`](roadmap.md)) adds an ephemeral sidecar container to game pods when capture is enabled. This sidecar requires **`allowPrivilegeEscalation: true`** in its securityContext, which violates the Pod Security Standards `restricted` @@ -404,35 +423,47 @@ file capability to function. The game container does not share this exception: the operator never sets `allowPrivilegeEscalation: true` on it or adds `NET_RAW` to it, and the capture -sidecar is the only container Gameplane grants `CAP_NET_RAW` for capture. The +worker receives explicit `CAP_NET_RAW` for capture. The game container keeps the posture described in [Pod security](#pod-security): the template's uid/gid or the image's default user, and the container runtime's default capability set. Which capabilities the game container holds therefore depends on that runtime default and on the user the image runs as, not on the capture feature. -**Trade-off with PodSecurity `restricted`**: A cluster enforcing the `restricted` -Pod Security Standards profile on the games namespace will reject any pod with -`allowPrivilegeEscalation: true`. If you require `restricted` admission on your -games namespace, you have three options: - -1. **Disable capture** — leave the cluster's capture feature disabled via Helm - value `capture.enabled: false` (default is false). Captures are not required - for normal operation; this is the safest option if you cannot or prefer not to - relax the `restricted` profile. -2. **Exempt the games namespace** — remove or relax the Pod Security Standards - `restricted` enforcement for the games namespace, using `baseline` or - `privileged` instead. The games namespace remains an untrusted environment - (game code can run arbitrary containers), but the admission level permits the - capture sidecar to be injected when needed. -3. **Leave the games-namespace label off** — the chart only adds the - `pod-security.kubernetes.io/enforce: restricted` label to the games namespace - when `podSecurity.enforceRestricted=true`; that value defaults to `false`. - A default install therefore already leaves the label off, and captures work - without any change. If you previously set `podSecurity.enforceRestricted=true` - and accept the operational trade-off, set it back to `false` in the Helm - values to drop the label. This setting only affects the games `Namespace` - object; it has no cluster-wide effect and there is no per-pod opt-in. +**Capture with Restricted games** uses a separate node diagnostic worker. +Enable both `capture.enabled` and `capture.nodeDiagnostics.enabled`, then an +administrator calls `POST /servers/{name}:capture-node-enable` for each allowed +server. The API synchronously audits the host-privilege opt-in before changing +`spec.capture.nodeDiagnostic`; ordinary edits cannot grant it and clones do not +inherit it. `:capture-node-disable` revokes it and drains active workers. Direct +Kubernetes administrators can manage the field through their own audited +GitOps process. The game Pod receives no capture container or restart. + +The chart creates a separate `gameplane-diagnostics` namespace (configurable), +explicitly labeled `privileged`. Deployment policy must permit hostPID, root, +`SYS_ADMIN`, `SYS_PTRACE`, `NET_RAW` and unconfined seccomp there. This is +host-level trust: a compromised worker can affect its node despite per-server +application authorization. It has no Kubernetes token, runtime socket or +hostPath. It accepts live Kubernetes identities, never user-supplied PIDs or +paths; checks Pod/container cgroups immediately before and after entering the +pinned network namespace; and stops when the target changes. Support is +limited to Linux/containerd, cgroupfs or systemd Pod cgroups, and target `eth0`. +Host-network and other runtime targets fail closed. Windows, rootless or user +namespaced runtimes, and nonstandard CNI interfaces are unsupported. +SELinux/AppArmor may deny these capabilities; capture fails rather than +broadening privileges. + +Each capture is bounded to one hour and 900MiB with a 1Gi PVC. The chart defaults +to ten captures and a 10Gi storage quota, including retained files. Privileged +Jobs and Pods are removed on completion, cancellation, revocation or target +identity change. A separate nonroot Pod serves only UID-bound files and status +over mTLS. A capture finalizer deletes its same-namespace anchor and waits for +garbage collection of Pods, PVC, Secret and Service on expiry, capture deletion +or GameServer deletion. Do not force-remove the finalizer without cleaning its +labeled diagnostic resources. Keep the operator and diagnostic namespace +installed until cleanup finishes. Disabling the deployment option deletes the +diagnostics namespace and retained files. No games-namespace or RuntimeClass +exemption is used. **Data sensitivity**: Captures contain binary game protocols, player IP addresses, and may include sensitive data like in-game chat or credentials. An admin with diff --git a/docs/template-rollouts.md b/docs/template-rollouts.md new file mode 100644 index 000000000..67e537590 --- /dev/null +++ b/docs/template-rollouts.md @@ -0,0 +1,108 @@ +# Adopting template updates + +Installing a Module updates its shared GameTemplate. Each GameServer now retains +an immutable copy of the template content it has admitted. The default +`Automatic` policy adopts installed changes as before, with lifecycle exclusion +and a graceful stop before replacement. To choose the time for a server, set: + +```yaml +spec: + templateRollout: + policy: OnRequest +``` + +Enable this before updating the Module. A new server always provisions from the +currently installed template. Existing servers without a recorded snapshot get +a baseline snapshot without a workload restart. The operator preserves their +existing Pod template, including API defaults and admission additions, until +workload inputs change or the workload has stopped and drained. Their `running` +revision stays unknown until a replacement Pod actually carries a revision. + +This baseline exception does not defer actual security changes: installing +agent-only credential projection or enabling the Restricted profile still +updates the workload. New template revisions must satisfy the enabled security +profile before admission and before the old workload is drained. + +The baseline records currently installed content; it cannot recover historical +template contents from an old Pod or prove that the old Pod used that content. +Pin `Module.spec.version` before upgrading and enable staging before a later +catalog change. A pre-upgrade backup, restore or wipe already in progress finishes +before baseline initialization. A new server gets its pin before first startup. + +Inspect the three digests: + +```sh +kubectl get gameserver my-server -n gameplane-games \ + -o jsonpath='{.status.templateRevision}' +``` + +- `desired` identifies the installed template contents. It is empty when the + installed template is unavailable. +- `effective` identifies the immutable template used for images, versions, + config files and init containers, storage, ports, probes, agent configuration + and other derived resources. During baseline initialization, the existing + unknown Pod template is retained until a real workload change or stopped drain. +- `running` is observed on an owned Pod with a running game container. It is + empty while stopped, while the game container is not running, or when the + pre-upgrade Pod has no revision annotation. It is not a health assertion. + +`TemplateUpdatePending=True` explains whether a revision is available, waiting +for another operation, or being adopted. `Unknown/TemplateUnavailable` preserves +an existing effective snapshot if the installed template disappears. A missing, +mutable, corrupt or foreign effective snapshot fails closed and requires repair; +it never silently switches the server to the latest catalog content. + +To admit an update, copy the exact 64-character `desired` digest into the request: + +```sh +revision=$(kubectl get gameserver my-server -n gameplane-games \ + -o jsonpath='{.status.templateRevision.desired}') +kubectl patch gameserver my-server -n gameplane-games --type=merge \ + -p "{\"spec\":{\"templateRollout\":{\"policy\":\"OnRequest\",\"requestedRevision\":\"$revision\"}}}" +``` + +An old request never approves a newer digest. Requests can be made one server at +a time or by an administrator's batch process. There is no built-in canary, +maintenance-window or fleet concurrency scheduler. Once admitted, an update +finishes using that exact snapshot even if another catalog update arrives or +the policy changes during shutdown. + +Admission uses the same durable owner as backups, restores and wipes. It waits +for their cleanup, runs the previous effective template's graceful stop contract, +waits for the old Pod to disappear, then switches the effective pin and creates +the replacement. Ownership is released when the new Pod is observed, or when a +suspended workload is confirmed stopped with the new template. A bad image may +still fail startup; check ordinary readiness and health conditions separately. +Suspend and idle intent remain in effect. User changes to per-server overrides, +including `spec.version`, `spec.image` and `spec.config`, remain immediate and are +not controlled by the shared-template adoption policy. + +Snapshots are immutable, GameServer-owned ConfigMaps, bounded to 900 KiB each. +The current and admitted revisions are retained during adoption; obsolete copies +are removed afterwards. Do not edit or delete them or the operator-owned revision +and operation annotations. A snapshot pins template contents, including image +references; use immutable image digests to prevent a mutable image tag changing +its registry content independently. + +## Pinning and rollback + +Set `Module.spec.version` to a known version to prevent tracking the catalog's +latest entry, including on installations that do not yet support staged adoption. +Use `Module.spec.digest` as well when the source supports OCI digest verification; +version tags alone may move. See [module installation](module-authoring.md#installing-a-module). + +Adoption retains existing primary, extra and mod PVC identities and their data; +it does not recreate them to apply new template storage defaults. Removed extra +or mod volumes remain retained. Deleting the GameServer is different: its owned +claims can be garbage-collected. Keep an independently verified backup before +changing a game's runtime or save format. + +Reinstalling an older Module version produces a desired template that can be +admitted with the same process. Reverting a runtime does not reverse save-format +changes. Restoring data may be necessary, and compatibility is game-specific. +Backups remain data backups, not automatic runtime rollback bundles. Record the +effective digest, immutable image reference and server configuration with your +backup procedure. Restic restores use the target's current effective layout; CSI +restores copy the source server's effective template at restore planning time, +not necessarily its template at backup time. Select a compatible Module version +and `OnRequest` policy before recovery if automatic adoption would be unsafe. diff --git a/hack/ci_scope.py b/hack/ci_scope.py index 91da39131..dfdb4709a 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -29,7 +29,8 @@ FORCE_ALL_PREFIXES = (".github/actions/",) # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml. E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components"] + "telemetry", "optional-components", "restricted-workloads", "management-placement"] +E2E_AMD64_ONLY = {"management-placement"} SUITES_FILE = "test/e2e/suite-components.json" # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. NO_E2E_FILES = { @@ -936,7 +937,8 @@ def e2e_row(suite, gate_on, picked, key, feed): for bucket in E2E_BUCKETS: name = "e2e api-auth + ratelimit tail" if bucket == "api-auth" else f"e2e {bucket}" picked = bucket in e2e_run or (bucket == "api-auth" and "ratelimit" in e2e_run) - e2e_row(f"{name} (amd64, arm64)", gate("e2e"), picked, bucket, e2e_feed()) + arches = "amd64" if bucket in E2E_AMD64_ONLY else "amd64, arm64" + e2e_row(f"{name} ({arches})", gate("e2e"), picked, bucket, e2e_feed()) e2e_row("e2e multicluster (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-multicluster"), "multicluster", e2e_feed()) e2e_row("e2e upgrade (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-upgrade"), "upgrade", diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index c8dd10dc1..3436a0208 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -9,7 +9,7 @@ from unittest.mock import patch from subprocess import CompletedProcess -from ci_scope import (E2E_BUCKETS, FORCE_ALL_FILES, ROOT, affected_modules, bot_matrix, +from ci_scope import (E2E_AMD64_ONLY, E2E_BUCKETS, FORCE_ALL_FILES, ROOT, affected_modules, bot_matrix, bucket_lists, changed_paths, e2e_outputs, full_scope_reason, image_inputs, is_unmapped, load_suites, main, publish_paths, select_images, summarize, verify_suites, go_packages, web_tests, CI_FILE, ci_blocks, ci_jobs, @@ -347,10 +347,10 @@ def test_every_e2e_bucket_and_bot_test_has_a_row(self): """Check every e2e bucket and bot test has a selection row.""" rows, _ = self.web_only() suites = {row["suite"] for row in rows} - self.assertEqual(len(E2E_BUCKETS), 8) for bucket in E2E_BUCKETS: name = "e2e api-auth + ratelimit tail" if bucket == "api-auth" else f"e2e {bucket}" - self.assertIn(f"{name} (amd64, arm64)", suites) + arches = "amd64" if bucket in E2E_AMD64_ONLY else "amd64, arm64" + self.assertIn(f"{name} ({arches})", suites) for test in SUMMARY_BOTS: self.assertIn(f"e2e game bot / {test}", suites) @@ -408,6 +408,32 @@ def select(self, *paths): """Return the e2e outputs for a partial-scope change to paths.""" return e2e_outputs(ROOT, list(paths), None) + def test_audit_restricted_workloads_has_a_run_pattern(self): + """The node capture worker and all drill components retain CI coverage.""" + for path in ("operator/internal/controller/workload_security.go", + "api/internal/handlers/capture_node.go", "agent/cmd/main.go", + "capture-sidecar/internal/nodecapture/worker.go", + "test/e2e/restricted_workloads_e2e_test.go"): + with self.subTest(path=path): + self.assertIn("TestRestrictedWorkloads_NodeCaptureAndTokenRotation", + self.select(path)["e2e-run"].get("restricted-workloads", "")) + + def test_audit_management_placement_has_a_run_pattern(self): + """Affected components and the owning test file select the audit bucket.""" + for path in ("operator/cmd/main.go", "api/cmd/main.go", "agent/cmd/main.go", + "test/e2e/management_placement_e2e_test.go"): + with self.subTest(path=path): + self.assertIn("TestManagementPlacementAndDisruption", + self.select(path)["e2e-run"].get("management-placement", "")) + + def test_audit_staged_rollout_keeps_operator_coverage(self): + """Template adoption and its owning fixture select the rollout scenario.""" + for path in ("operator/internal/controller/gameserver_template_revision.go", + "test/e2e/template_rollout_e2e_test.go"): + with self.subTest(path=path): + self.assertIn("TestGameServer_TemplateOnRequestRetainsData", + self.select(path)["e2e-run"].get("operator", "")) + def test_agent_only_change_selects_agent_tests_and_skips_auth_and_telemetry(self): values = self.select("agent/internal/console/console.go") self.assertEqual(values["reason"], "") @@ -504,6 +530,20 @@ def test_repository_suite_map_is_valid(self): class E2ERunTests(unittest.TestCase): """The e2e-run regexes that replace buckets.sh regex in the e2e jobs (T016).""" + def test_bucket_registry_matches_workflow_matrices(self): + """Every matrix bucket emits a run pattern and reports its real architectures.""" + text = (ROOT / CI_FILE).read_text(encoding="utf-8") + owner, _ = ci_blocks(text) + matrices = {} + for number, line in enumerate(text.splitlines(), 1): + job = owner[number] + if job in {"e2e-go", "e2e-go-arm64"} and line.strip().startswith("bucket: ["): + self.assertNotIn(job, matrices) + matrices[job] = {name.strip() for name in line.split("[", 1)[1].split("]", 1)[0].split(",")} + self.assertEqual(set(matrices), {"e2e-go", "e2e-go-arm64"}) + self.assertEqual(set(E2E_BUCKETS), matrices["e2e-go"] | matrices["e2e-go-arm64"]) + self.assertEqual(E2E_AMD64_ONLY, matrices["e2e-go"] - matrices["e2e-go-arm64"]) + def test_full_scope_regex_equals_buckets_sh_regex(self): values = e2e_outputs(ROOT, None, "master push") self.assertEqual(values["reason"], "master push") diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index 059470fd2..05925af51 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -175,6 +175,42 @@ "defaultMessage": "The backup or restore completed successfully.", "description": "Completed condition on a Backup or Restore that finished." }, + "condition.consistency.agentunavailablerawcopy": { + "defaultMessage": "No agent quiesce capability was configured. The primary volume was copied without pausing game writes.", + "description": "Backup achieved consistency: AgentUnavailableRawCopy." + }, + "condition.consistency.capturefailed": { + "defaultMessage": "The capture failed; no usable consistency guarantee was achieved.", + "description": "Backup achieved consistency: CaptureFailed." + }, + "condition.consistency.capturepending": { + "defaultMessage": "The capture has not completed; its achieved consistency is not yet known.", + "description": "Backup achieved consistency: CapturePending." + }, + "condition.consistency.crashconsistentsnapshot": { + "defaultMessage": "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed.", + "description": "Backup achieved consistency: CrashConsistentSnapshot." + }, + "condition.consistency.legacymetadataunknown": { + "defaultMessage": "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template.", + "description": "Backup achieved consistency: LegacyMetadataUnknown." + }, + "condition.consistency.quiesced": { + "defaultMessage": "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup.", + "description": "Backup achieved consistency: Quiesced." + }, + "condition.consistency.rawcopy": { + "defaultMessage": "The primary volume was copied without confirmed application quiesce.", + "description": "Backup achieved consistency: RawCopy." + }, + "condition.consistency.stoppedsource": { + "defaultMessage": "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup.", + "description": "Backup achieved consistency: StoppedSource." + }, + "condition.consistency.unsupportedrawcopy": { + "defaultMessage": "The game does not support quiesce. The primary volume was copied while it could still change.", + "description": "Backup achieved consistency: UnsupportedRawCopy." + }, "condition.datawipe.jobfailed": { "defaultMessage": "The data wipe job did not complete. The volume may not be fully cleared.", "description": "DataWipe condition when the wipe job fails." @@ -559,6 +595,26 @@ "defaultMessage": "Indexing the module source failed.", "description": "Synced condition after a failed index." }, + "condition.templateupdatepending.adoptingrevision": { + "defaultMessage": "The admitted template revision is being adopted.", + "description": "Template rollout owns the server lifecycle while draining and replacing its workload." + }, + "condition.templateupdatepending.operationinprogress": { + "defaultMessage": "Template adoption waits for the active lifecycle operation to finish.", + "description": "A backup, restore or wipe prevents admitting an available template revision." + }, + "condition.templateupdatepending.revisionavailable": { + "defaultMessage": "An installed template revision is waiting for admission.", + "description": "An installed template differs from the server's immutable effective revision." + }, + "condition.templateupdatepending.revisioncurrent": { + "defaultMessage": "The installed template revision is effective.", + "description": "No newer template revision is waiting for adoption. Running revision is observed separately." + }, + "condition.templateupdatepending.templateunavailable": { + "defaultMessage": "The installed template is unavailable; the effective snapshot is retained.", + "description": "Template availability is unknown but the server retains its immutable effective content." + }, "condition.tunneladdressinvalid.validationfailed": { "defaultMessage": "Some tunnel endpoints were rejected: {details}", "description": "Condition when endpoints reported by the playit tunnel pod fail validation. details is the English validation text." @@ -599,6 +655,10 @@ "defaultMessage": "The server could not be resumed after the backup. Its world may still have auto-save turned off.", "description": "Unquiesced condition when the unfreeze failed." }, + "condition.unquiesced.sourceprocessgone": { + "defaultMessage": "The original game process has exited. No resume command was needed.", + "description": "Unquiesced condition when the original source pod no longer runs." + }, "condition.unquiesced.unquiesced": { "defaultMessage": "The server was resumed after the backup.", "description": "Unquiesced condition when the world was unfrozen." @@ -1131,10 +1191,34 @@ "defaultMessage": "Backup details", "description": "Backup details drawer: heading at the top of the side panel for one backup; the backup name is shown below it" }, + "web.components.backups.backup_detail_drawer.classification": { + "defaultMessage": "{role, select, data {Data} rebuildable {Rebuildable} other {Unclassified}}", + "description": "Backup volume: optional declaration from the module or server at capture time" + }, "web.components.backups.backup_detail_drawer.completed": { "defaultMessage": "Completed", "description": "Backup details drawer: field label above the time the backup completed, shown in relative time" }, + "web.components.backups.backup_detail_drawer.consistency": { + "defaultMessage": "Achieved consistency", + "description": "Backup details: actual consistency reported by the controller, independent of successful copying" + }, + "web.components.backups.backup_detail_drawer.consistency_unknown": { + "defaultMessage": "Consistency has not been reported.", + "description": "Backup details: no achieved-consistency condition is available" + }, + "web.components.backups.backup_detail_drawer.coverage": { + "defaultMessage": "Volume coverage", + "description": "Backup details: persistent claims recorded before the backup copy began" + }, + "web.components.backups.backup_detail_drawer.coverage_recorded": { + "defaultMessage": "Recorded: {time}", + "description": "Backup details: timestamp of the immutable capture manifest" + }, + "web.components.backups.backup_detail_drawer.coverage_scope": { + "defaultMessage": "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", + "description": "Backup details: explains the current single-volume backup scope and rebuildable declarations" + }, "web.components.backups.backup_detail_drawer.delete": { "defaultMessage": "Delete", "description": "Backup detail drawer: delete button label" @@ -1143,6 +1227,18 @@ "defaultMessage": "Deleting…", "description": "Backup detail drawer: delete button label while the delete request is pending" }, + "web.components.backups.backup_detail_drawer.included": { + "defaultMessage": "Included", + "description": "Backup volume: this claim is selected for copying; success is reported separately" + }, + "web.components.backups.backup_detail_drawer.legacy_coverage": { + "defaultMessage": "Capture-time volume metadata is unavailable. Inspect the original backup and module configuration before recovery.", + "description": "Backup details: older backups have no reliable volume manifest, so the current template is not used to invent coverage" + }, + "web.components.backups.backup_detail_drawer.not_covered": { + "defaultMessage": "Not covered", + "description": "Backup volume: contents of this claim are not included" + }, "web.components.backups.backup_detail_drawer.phase": { "defaultMessage": "Phase", "description": "Backup details drawer: field label above the backup's current phase, shown as a status chip" @@ -1155,6 +1251,10 @@ "defaultMessage": "Restore backup", "description": "Backup details drawer: screen-reader name of the restore button in the drawer header, read instead of its visible word Restore; not visible on screen" }, + "web.components.backups.backup_detail_drawer.retained_unmounted": { + "defaultMessage": "Retained claim; not mounted in the captured workload.", + "description": "Backup volume: an older server-owned claim was retained but was not part of the observed pod layout" + }, "web.components.backups.backup_detail_drawer.server": { "defaultMessage": "Server", "description": "Backup details drawer: field label above the name of the server the backup belongs to" diff --git a/operator/api/v1alpha1/backup_types.go b/operator/api/v1alpha1/backup_types.go index d093916f4..c2ec70c00 100644 --- a/operator/api/v1alpha1/backup_types.go +++ b/operator/api/v1alpha1/backup_types.go @@ -17,11 +17,10 @@ const ( BackupPhaseFailed BackupPhase = "Failed" ) -// BackupFinalizer blocks a quiesced Backup's deletion until the operator has -// sent the matching unquiesce. Without it, deleting a Backup while -// quiesce-attempted=true drops the unquiesce entirely and the game is left -// with auto-save off indefinitely (F-048). Added only for Backups with -// spec.quiesce=true; see backup_forget.go's finalizeDelete. +// BackupFinalizer blocks deletion until copy workers have drained and any +// matching unquiesce has completed. Every Backup carries it, including raw +// copies and CSI snapshots, so deletion cannot release shared lifecycle +// ownership while a copy is still using the target. See finalizeDelete. const BackupFinalizer = "gameplane.local/backup-finalizer" // BackupSnapshotFinalizer blocks the deletion of a restic-strategy Backup until @@ -89,6 +88,12 @@ type LocalObjectRef struct { // BackupStatus is the observed state of a Backup. type BackupStatus struct { + // CaptureManifest is recorded once, before the first capture worker starts. + // It describes persistent-volume coverage independently of copy success. + // Missing on pre-upgrade backups; never reconstruct it from current layout. + // +optional + CaptureManifest *BackupCaptureManifest `json:"captureManifest,omitempty"` + // +optional Phase BackupPhase `json:"phase,omitempty"` @@ -129,6 +134,47 @@ type BackupStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } +// BackupCaptureManifest pins the layout and quiesce result at capture admission. +// Recorded coverage does not imply a successful or complete server backup. +type BackupCaptureManifest struct { + // State is Recorded for new captures, LegacyUnknown for an existing worker + // whose original layout was never recorded. + // +kubebuilder:validation:Enum=Recorded;LegacyUnknown + State string `json:"state"` + // +optional + RecordedAt *metav1.Time `json:"recordedAt,omitempty"` + // +optional + ServerUID string `json:"serverUID,omitempty"` + // +optional + Strategy string `json:"strategy,omitempty"` + // LayoutSource identifies the frozen workload used for mount paths. + // +optional + LayoutSource string `json:"layoutSource,omitempty"` + // QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + // agent-unavailable, or unknown, independent of subsequent copy success. + // +optional + QuiesceOutcome string `json:"quiesceOutcome,omitempty"` + // +optional + Volumes []BackupVolumeCoverage `json:"volumes,omitempty"` +} + +// BackupVolumeCoverage describes one persistent claim. Extra and loader/mod +// claims remain uncovered even when mounted beneath the primary data directory. +type BackupVolumeCoverage struct { + ClaimName string `json:"claimName"` + // ClaimUID is empty if the claim did not yet exist at admission. + // +optional + ClaimUID string `json:"claimUID,omitempty"` + // +optional + MountPaths []string `json:"mountPaths,omitempty"` + Included bool `json:"included"` + // Classification records the module/server declaration at admission; + // it does not prove that the volume can actually be reconstructed. + // +kubebuilder:validation:Enum=data;rebuildable;unclassified + Classification string `json:"classification"` + CoverageReason string `json:"reason"` +} + // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=bk // +kubebuilder:printcolumn:name="Server",type=string,JSONPath=`.spec.serverRef.name` diff --git a/operator/api/v1alpha1/gameserver_types.go b/operator/api/v1alpha1/gameserver_types.go index 654cba6ed..625398c70 100644 --- a/operator/api/v1alpha1/gameserver_types.go +++ b/operator/api/v1alpha1/gameserver_types.go @@ -49,6 +49,11 @@ type GameServerSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="templateRef is immutable; delete and recreate the GameServer to switch templates" TemplateRef GameTemplateRef `json:"templateRef"` + // TemplateRollout controls adoption of changes to the shared GameTemplate. + // Omitted preserves Automatic adoption. Per-server overrides remain live. + // +optional + TemplateRollout *TemplateRolloutSpec `json:"templateRollout,omitempty"` + // Suspend, when true, scales the underlying StatefulSet to zero. // Data is preserved. Transitions the GameServer to Suspended. // +kubebuilder:default=false @@ -224,15 +229,23 @@ type IdleSpec struct { type CaptureConfiguration struct { // Enabled: opt-in flag for packet-capture capability on this GameServer. // If false or omitted, no capture sidecar is injected; the game container is unchanged. - // If true, the operator injects the capture sidecar as an ephemeral container via + // In the legacy profile, the operator injects the capture sidecar via // pods/ephemeralcontainers, live, without restarting the game container. // Disabling (true→false) does not remove the sidecar from a running pod — Kubernetes // has no API to remove ephemeral containers — but stops accepting new captures. + // Restricted workloads require NodeDiagnostic instead and never inject a sidecar. // +kubebuilder:validation:Optional // +kubebuilder:validation:Type=boolean // +optional Enabled bool `json:"enabled,omitempty"` + // NodeDiagnostic permits the separately deployed node capture worker for + // this server. Application administrators must explicitly opt in. The + // worker has host-level capabilities; it never weakens the game Pod. + // Required for capture on Restricted workloads. Off by default. + // +optional + NodeDiagnostic bool `json:"nodeDiagnostic,omitempty"` + // RetentionSeconds: optional per-GameServer override of the cluster-default retention window // after a capture completes. Unit: seconds. If omitted, the cluster default // (Helm value capture.defaultRetentionSeconds) is used. Must not exceed the cluster @@ -508,8 +521,41 @@ type InlineBackupPolicy struct { Suspend bool `json:"suspend,omitempty"` } +// TemplateRolloutSpec controls when installed template content is adopted. +type TemplateRolloutSpec struct { + // Policy defaults to Automatic. OnRequest requires the exact desired digest. + // +kubebuilder:validation:Enum=Automatic;OnRequest + // +kubebuilder:default=Automatic + // +optional + Policy string `json:"policy,omitempty"` + // RequestedRevision authorizes this SHA256 only, never future revisions. + // Read status.templateRevision.desired and copy it here to admit an update. + // Initial provisioning always adopts the installed template. + // +kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` + // +optional + RequestedRevision string `json:"requestedRevision,omitempty"` +} + +// TemplateRevisionStatus distinguishes installed, admitted and observed content. +type TemplateRevisionStatus struct { + // Desired is the installed template's SHA256; empty if it is unavailable. + // +optional + Desired string `json:"desired,omitempty"` + // Effective is the immutable content used for all derived resources. + // +optional + Effective string `json:"effective,omitempty"` + // Running comes from an owned Pod with a running game container. Empty + // means no running revision is known, including pre-upgrade legacy Pods. + // +optional + Running string `json:"running,omitempty"` +} + // GameServerStatus is the observed state of a GameServer. type GameServerStatus struct { + // TemplateRevision is a read model; operator-owned annotations and immutable + // ConfigMaps retain admission state across status resets and leader changes. + // +optional + TemplateRevision *TemplateRevisionStatus `json:"templateRevision,omitempty"` // +optional Phase GameServerPhase `json:"phase,omitempty"` diff --git a/operator/api/v1alpha1/gametemplate_types.go b/operator/api/v1alpha1/gametemplate_types.go index bea234aea..8001a2b27 100644 --- a/operator/api/v1alpha1/gametemplate_types.go +++ b/operator/api/v1alpha1/gametemplate_types.go @@ -946,6 +946,14 @@ type GameStorageSpec struct { // primary Storage.MountPath, since these exist precisely for directories // that share no safe common parent with it. type ExtraVolumeSpec struct { + // BackupRole describes recovery requirements without adding this volume + // to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + // Unset remains unclassified and not covered. Both backup strategies still + // capture only the primary data claim. + // +kubebuilder:validation:Enum=data;rebuildable + // +optional + BackupRole string `json:"backupRole,omitempty"` + // Name identifies the volume and forms part of its PVC name. Must be a // DNS-1123 label. // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 05926fd58..85b8d51a5 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -178,6 +178,32 @@ func (in *Backup) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupCaptureManifest) DeepCopyInto(out *BackupCaptureManifest) { + *out = *in + if in.RecordedAt != nil { + in, out := &in.RecordedAt, &out.RecordedAt + *out = (*in).DeepCopy() + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]BackupVolumeCoverage, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupCaptureManifest. +func (in *BackupCaptureManifest) DeepCopy() *BackupCaptureManifest { + if in == nil { + return nil + } + out := new(BackupCaptureManifest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupList) DeepCopyInto(out *BackupList) { *out = *in @@ -388,6 +414,11 @@ func (in *BackupSpec) DeepCopy() *BackupSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { *out = *in + if in.CaptureManifest != nil { + in, out := &in.CaptureManifest, &out.CaptureManifest + *out = new(BackupCaptureManifest) + (*in).DeepCopyInto(*out) + } if in.StartTime != nil { in, out := &in.StartTime, &out.StartTime *out = (*in).DeepCopy() @@ -420,6 +451,26 @@ func (in *BackupStatus) DeepCopy() *BackupStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupVolumeCoverage) DeepCopyInto(out *BackupVolumeCoverage) { + *out = *in + if in.MountPaths != nil { + in, out := &in.MountPaths, &out.MountPaths + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupVolumeCoverage. +func (in *BackupVolumeCoverage) DeepCopy() *BackupVolumeCoverage { + if in == nil { + return nil + } + out := new(BackupVolumeCoverage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BanListSpec) DeepCopyInto(out *BanListSpec) { *out = *in @@ -953,6 +1004,11 @@ func (in *GameServerNetworking) DeepCopy() *GameServerNetworking { func (in *GameServerSpec) DeepCopyInto(out *GameServerSpec) { *out = *in out.TemplateRef = in.TemplateRef + if in.TemplateRollout != nil { + in, out := &in.TemplateRollout, &out.TemplateRollout + *out = new(TemplateRolloutSpec) + **out = **in + } if in.StopGracePeriodSeconds != nil { in, out := &in.StopGracePeriodSeconds, &out.StopGracePeriodSeconds *out = new(int32) @@ -1042,6 +1098,11 @@ func (in *GameServerSpec) DeepCopy() *GameServerSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GameServerStatus) DeepCopyInto(out *GameServerStatus) { *out = *in + if in.TemplateRevision != nil { + in, out := &in.TemplateRevision, &out.TemplateRevision + *out = new(TemplateRevisionStatus) + **out = **in + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) @@ -2573,6 +2634,36 @@ func (in *TailscaleTunnelSpec) DeepCopy() *TailscaleTunnelSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemplateRevisionStatus) DeepCopyInto(out *TemplateRevisionStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemplateRevisionStatus. +func (in *TemplateRevisionStatus) DeepCopy() *TemplateRevisionStatus { + if in == nil { + return nil + } + out := new(TemplateRevisionStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemplateRolloutSpec) DeepCopyInto(out *TemplateRolloutSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemplateRolloutSpec. +func (in *TemplateRolloutSpec) DeepCopy() *TemplateRolloutSpec { + if in == nil { + return nil + } + out := new(TemplateRolloutSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VerifySpec) DeepCopyInto(out *VerifySpec) { *out = *in diff --git a/operator/cmd/main.go b/operator/cmd/main.go index 85263e901..57bbe5688 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -143,6 +143,10 @@ func main() { gameIngressFromCIDR cidrListFlag gameDataStorageClass string captureEnabled bool + restrictedWorkloads bool + agentTokenExpirationSeconds int64 + diagnosticsNamespace string + diagnosticsStorageClass string captureDefaultRetention int64 captureMaxRetention int64 captureSidecarImage string @@ -150,10 +154,18 @@ func main() { captureDefaultMaxSizeBytes int64 backupJobBackoffLimit int64 backupJobActiveDeadlineSeconds int64 + agentResourcesJSON string + sentinelResourcesJSON string + tunnelResourcesJSON string + backupResourcesJSON string ) flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "Address the metrics endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "Address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election.") + flag.StringVar(&agentResourcesJSON, "agent-resources", "", "JSON ResourceRequirements for agents and config init containers; default requests 25m/64Mi, {} clears defaults.") + flag.StringVar(&sentinelResourcesJSON, "sentinel-resources", "", "JSON ResourceRequirements for sentinels; default requests 25m/32Mi, {} clears defaults.") + flag.StringVar(&tunnelResourcesJSON, "tunnel-resources", "", "JSON ResourceRequirements for tunnel relays; default requests 25m/64Mi, {} clears defaults.") + flag.StringVar(&backupResourcesJSON, "backup-resources", "", "JSON ResourceRequirements for restic backup/init, restore and cleanup workers; default backup/restore requests 100m/128Mi, {} clears defaults.") flag.StringVar(&agentImage, "agent-image", "ghcr.io/gameplanepanel/gameplane/agent:dev", "Image to use for the Gameplane agent sidecar injected into game pods.") flag.StringVar(&agentImagePullPolicy, "agent-image-pull-policy", "", @@ -185,6 +197,10 @@ func main() { flag.StringVar(&tunnelPlayitImage, "tunnel-playit-image", controller.DefaultTunnelPlayitImage, "Image for the Playit tunnel relay pod that routes players through a Playit tunnel. "+ "Point at a private registry mirror for air-gapped installs.") + flag.BoolVar(&restrictedWorkloads, "restricted-workloads", false, "Require compatible nonroot templates and render Restricted game workloads") + flag.Int64Var(&agentTokenExpirationSeconds, "agent-token-expiration-seconds", 3600, "Projected agent token lifetime (600-3600 seconds)") + flag.StringVar(&diagnosticsNamespace, "diagnostics-namespace", "", "Opt in to node diagnostics in this dedicated privileged namespace") + flag.StringVar(&diagnosticsStorageClass, "diagnostics-storage-class", "", "StorageClass for bounded node capture artifact PVCs") flag.BoolVar(&captureEnabled, "capture-enabled", false, "Enable the network capture feature cluster-wide. When false (the default), "+ "the capture capability is disabled and cannot be enabled per-GameServer.") @@ -259,10 +275,38 @@ func main() { opts := zap.Options{Development: true} opts.BindFlags(flag.CommandLine) flag.Parse() + if agentTokenExpirationSeconds < 600 || agentTokenExpirationSeconds > 3600 { + fmt.Fprintln(os.Stderr, "agent-token-expiration-seconds must be between 600 and 3600") + os.Exit(1) + } + if diagnosticsNamespace != "" && diagnosticsNamespace == controlPlaneNamespace { + fmt.Fprintln(os.Stderr, "diagnostics-namespace must be separate from the control-plane namespace") + os.Exit(1) + } ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) setupLog := ctrl.Log.WithName("setup") + agentResources, err := controller.ParseAuxiliaryResources(agentResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --agent-resources") + os.Exit(1) + } + sentinelResources, err := controller.ParseAuxiliaryResources(sentinelResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --sentinel-resources") + os.Exit(1) + } + tunnelResources, err := controller.ParseAuxiliaryResources(tunnelResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --tunnel-resources") + os.Exit(1) + } + backupResources, err := controller.ParseAuxiliaryResources(backupResourcesJSON) + if err != nil { + setupLog.Error(err, "invalid --backup-resources") + os.Exit(1) + } backupJobBackoff32, err := boundedInt32(backupJobBackoffLimit) // rejects <0 and >MaxInt32 if err != nil { setupLog.Error(err, "invalid --backup-job-backoff-limit value (must be >= 0)", "value", backupJobBackoffLimit) @@ -380,6 +424,9 @@ func main() { APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), AgentImage: agentImage, + AgentResources: agentResources, + SentinelResources: sentinelResources, + TunnelResources: tunnelResources, AgentImagePullPolicy: agentImagePullPolicy, ConfigInitImage: configInitImage, SentinelImage: sentinelImage, @@ -390,17 +437,21 @@ func main() { AgentCASecretName: agentCASecretName, AgentCASecretNamespace: agentCASecretNamespace, AgentClient: agentClient, + OperationQuiescer: agentClient, PodAttacher: &controller.StopAttachClient{ Config: mgr.GetConfig(), Clientset: kubernetes.NewForConfigOrDie(mgr.GetConfig()), }, - AddressManager: addressManager, - MetalLBNamespace: metalLBNamespace, - GameIngressPolicyEnabled: gameIngressPolicy, - GameIngressFromCIDRs: gameIngressFromCIDR, - CaptureEnabled: captureEnabled, - CaptureSidecarImage: captureSidecarImage, - DefaultStorageClassName: gameDataStorageClass, + AddressManager: addressManager, + MetalLBNamespace: metalLBNamespace, + GameIngressPolicyEnabled: gameIngressPolicy, + GameIngressFromCIDRs: gameIngressFromCIDR, + CaptureEnabled: captureEnabled, + RestrictedWorkloads: restrictedWorkloads, + AgentTokenExpirationSeconds: agentTokenExpirationSeconds, + DiagnosticsNamespace: diagnosticsNamespace, + CaptureSidecarImage: captureSidecarImage, + DefaultStorageClassName: gameDataStorageClass, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to set up controller", "controller", "GameServer") os.Exit(1) @@ -419,6 +470,7 @@ func main() { Clientset: kubernetes.NewForConfigOrDie(mgr.GetConfig()), AgentClient: agentClient, ResticImage: resticImage, + BackupResources: backupResources, JobBackoffLimit: &backupJobBackoff32, JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds, EventRecorder: mgr.GetEventRecorder("backup-controller"), @@ -437,7 +489,9 @@ func main() { Client: mgr.GetClient(), APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), + OperationQuiescer: agentClient, ResticImage: resticImage, + BackupResources: backupResources, JobBackoffLimit: &backupJobBackoff32, JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds, }).SetupWithManager(mgr); err != nil { @@ -451,6 +505,12 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), SidecarClient: captureClient, + APIReader: mgr.GetAPIReader(), + RestrictedWorkloads: restrictedWorkloads, + DiagnosticsNamespace: diagnosticsNamespace, + DiagnosticsStorageClass: diagnosticsStorageClass, + AgentCASecretName: agentCASecretName, + AgentCASecretNamespace: agentCASecretNamespace, CaptureEnabled: captureEnabled, CaptureSidecarImage: captureSidecarImage, CaptureDefaultMaxDurationSeconds: captureDefaultMaxDurationSeconds, diff --git a/operator/config/crd/gameplane.local_backups.yaml b/operator/config/crd/gameplane.local_backups.yaml index f96731731..649a29a0d 100644 --- a/operator/config/crd/gameplane.local_backups.yaml +++ b/operator/config/crd/gameplane.local_backups.yaml @@ -128,6 +128,75 @@ spec: status: description: BackupStatus is the observed state of a Backup. properties: + captureManifest: + description: |- + CaptureManifest is recorded once, before the first capture worker starts. + It describes persistent-volume coverage independently of copy success. + Missing on pre-upgrade backups; never reconstruct it from current layout. + properties: + layoutSource: + description: LayoutSource identifies the frozen workload used + for mount paths. + type: string + quiesceOutcome: + description: |- + QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + agent-unavailable, or unknown, independent of subsequent copy success. + type: string + recordedAt: + format: date-time + type: string + serverUID: + type: string + state: + description: |- + State is Recorded for new captures, LegacyUnknown for an existing worker + whose original layout was never recorded. + enum: + - Recorded + - LegacyUnknown + type: string + strategy: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + mountPaths: + items: + type: string + type: array + reason: + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - state + type: object completionTime: format: date-time type: string diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index 4be670ebc..b9a5b2682 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -1039,10 +1039,18 @@ spec: description: |- Enabled: opt-in flag for packet-capture capability on this GameServer. If false or omitted, no capture sidecar is injected; the game container is unchanged. - If true, the operator injects the capture sidecar as an ephemeral container via + In the legacy profile, the operator injects the capture sidecar via pods/ephemeralcontainers, live, without restarting the game container. Disabling (true→false) does not remove the sidecar from a running pod — Kubernetes has no API to remove ephemeral containers — but stops accepting new captures. + Restricted workloads require NodeDiagnostic instead and never inject a sidecar. + type: boolean + nodeDiagnostic: + description: |- + NodeDiagnostic permits the separately deployed node capture worker for + this server. Application administrators must explicitly opt in. The + worker has host-level capabilities; it never weakens the game Pod. + Required for capture on Restricted workloads. Off by default. type: boolean retentionSeconds: description: |- @@ -2214,6 +2222,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. @@ -2285,6 +2303,27 @@ spec: - message: templateRef is immutable; delete and recreate the GameServer to switch templates rule: self == oldSelf + templateRollout: + description: |- + TemplateRollout controls adoption of changes to the shared GameTemplate. + Omitted preserves Automatic adoption. Per-server overrides remain live. + properties: + policy: + default: Automatic + description: Policy defaults to Automatic. OnRequest requires + the exact desired digest. + enum: + - Automatic + - OnRequest + type: string + requestedRevision: + description: |- + RequestedRevision authorizes this SHA256 only, never future revisions. + Read status.templateRevision.desired and copy it here to admit an update. + Initial provisioning always adopts the installed template. + pattern: ^[a-f0-9]{64}$ + type: string + type: object tolerations: items: description: |- @@ -2620,6 +2659,25 @@ spec: observed as Ready. format: date-time type: string + templateRevision: + description: |- + TemplateRevision is a read model; operator-owned annotations and immutable + ConfigMaps retain admission state across status resets and leader changes. + properties: + desired: + description: Desired is the installed template's SHA256; empty + if it is unavailable. + type: string + effective: + description: Effective is the immutable content used for all derived + resources. + type: string + running: + description: |- + Running comes from an owned Pod with a running game container. Empty + means no running revision is known, including pre-upgrade legacy Pods. + type: string + type: object tunnelEndpoints: description: |- TunnelEndpoints are the per-port addresses reported by the tunnel pod. diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index 13d8a1fa5..ce25ffca9 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -2165,6 +2165,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/operator/config/rbac/role_namespace.yaml b/operator/config/rbac/role_namespace.yaml index cdbefd6bd..9e4e3e29b 100644 --- a/operator/config/rbac/role_namespace.yaml +++ b/operator/config/rbac/role_namespace.yaml @@ -40,6 +40,7 @@ rules: - backups/finalizers - backupschedules/finalizers - restores/finalizers + - networkcaptures/finalizers verbs: ["update"] # Deployments: the sentinel and tunnel-client Deployments live beside # their GameServer. diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index e644cf041..4d7a8cbfc 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -46,6 +46,7 @@ type row struct { // guardCalls maps a helper that takes a condition reason as a string literal // to the index of that argument. var guardCalls = map[string]int{ + "setTemplateCondition": 2, "markUnhealthy": 2, "markFailed": 2, "markPending": 2, @@ -67,6 +68,11 @@ var guardSkipFiles = map[string]bool{ // rows is the English reference for every condition reason. var rows = []row{ + {types: []string{"TemplateUpdatePending"}, reason: "AdoptingRevision", english: "The admitted template revision is being adopted.", description: "Template rollout owns the server lifecycle while draining and replacing its workload."}, + {types: []string{"TemplateUpdatePending"}, reason: "TemplateUnavailable", english: "The installed template is unavailable; the effective snapshot is retained.", description: "Template availability is unknown but the server retains its immutable effective content."}, + {types: []string{"TemplateUpdatePending"}, reason: "RevisionCurrent", english: "The installed template revision is effective.", description: "No newer template revision is waiting for adoption. Running revision is observed separately."}, + {types: []string{"TemplateUpdatePending"}, reason: "OperationInProgress", english: "Template adoption waits for the active lifecycle operation to finish.", description: "A backup, restore or wipe prevents admitting an available template revision."}, + {types: []string{"TemplateUpdatePending"}, reason: "RevisionAvailable", english: "An installed template revision is waiting for admission.", description: "An installed template differs from the server's immutable effective revision."}, // GameServer: Ready {types: []string{"Ready"}, reason: "Running", english: "The server is running and the agent is reporting heartbeats.", description: "Ready condition while the game server is up."}, {types: []string{"Ready"}, reason: "Pending", english: "The server is waiting to start.", description: "Ready condition while the game server is being created."}, @@ -140,10 +146,22 @@ var rows = []row{ {types: []string{"Completed"}, reason: "Pending", english: "The backup is waiting to start.", description: "Completed condition on a Backup that has not started (dynamic: reason is the phase).", dynamic: true}, {types: []string{"Completed"}, reason: "Running", english: "The backup is running.", description: "Completed condition on a Backup in progress (dynamic: reason is the phase).", dynamic: true}, {types: []string{"Unquiesced"}, reason: "Unquiesced", english: "The server was resumed after the backup.", description: "Unquiesced condition when the world was unfrozen."}, + {types: []string{"Unquiesced"}, reason: "SourceProcessGone", english: "The original game process has exited. No resume command was needed.", description: "Unquiesced condition when the original source pod no longer runs."}, {types: []string{"Unquiesced"}, reason: "Failed", english: "The server could not be resumed after the backup. Its world may still have auto-save turned off.", description: "Unquiesced condition when the unfreeze failed."}, {types: []string{"SnapshotForgotten"}, reason: "Forgotten", english: "The snapshot was removed from the backup repository.", description: "SnapshotForgotten condition after a deleted Backup's snapshot was forgotten."}, {types: []string{"SnapshotForgotten"}, reason: "ForgetAbandoned", english: "The snapshot was not removed from the backup repository when the backup was deleted.", description: "SnapshotForgotten condition when the forget step was abandoned."}, + // Consistency is separate from copy completion and volume coverage. + {types: []string{"Consistency"}, reason: "CaptureFailed", english: "The capture failed; no usable consistency guarantee was achieved.", description: "Backup achieved consistency: CaptureFailed."}, + {types: []string{"Consistency"}, reason: "LegacyMetadataUnknown", english: "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template.", description: "Backup achieved consistency: LegacyMetadataUnknown."}, + {types: []string{"Consistency"}, reason: "CapturePending", english: "The capture has not completed; its achieved consistency is not yet known.", description: "Backup achieved consistency: CapturePending."}, + {types: []string{"Consistency"}, reason: "Quiesced", english: "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup.", description: "Backup achieved consistency: Quiesced."}, + {types: []string{"Consistency"}, reason: "StoppedSource", english: "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup.", description: "Backup achieved consistency: StoppedSource."}, + {types: []string{"Consistency"}, reason: "CrashConsistentSnapshot", english: "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed.", description: "Backup achieved consistency: CrashConsistentSnapshot."}, + {types: []string{"Consistency"}, reason: "UnsupportedRawCopy", english: "The game does not support quiesce. The primary volume was copied while it could still change.", description: "Backup achieved consistency: UnsupportedRawCopy."}, + {types: []string{"Consistency"}, reason: "AgentUnavailableRawCopy", english: "No agent quiesce capability was configured. The primary volume was copied without pausing game writes.", description: "Backup achieved consistency: AgentUnavailableRawCopy."}, + {types: []string{"Consistency"}, reason: "RawCopy", english: "The primary volume was copied without confirmed application quiesce.", description: "Backup achieved consistency: RawCopy."}, + // BackupSchedule {types: []string{"ScheduleValid"}, reason: "CronParseError", english: "The schedule is not a valid cron expression.", description: "ScheduleValid condition for an unparseable schedule."}, {types: []string{"ScheduleValid"}, reason: "CronValid", english: "The schedule is a valid cron expression.", description: "ScheduleValid condition for a good schedule."}, diff --git a/operator/internal/agent/client.go b/operator/internal/agent/client.go index d606c3c75..2b1c06923 100644 --- a/operator/internal/agent/client.go +++ b/operator/internal/agent/client.go @@ -14,6 +14,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "time" ) @@ -119,6 +120,47 @@ func (c *Client) Unquiesce(ctx context.Context, namespace, server string) error return c.callQuiesce(ctx, namespace, server, "/unquiesce", false) } +// QuiesceEnabled reports whether this client can perform quiesce operations. +func (c *Client) QuiesceEnabled() bool { return c != nil && !c.Disabled } + +// CheckPod verifies that the responding agent belongs to the selected Pod. +// Older agents return 404; callers must not fall back to unbound mutations. +func (c *Client) CheckPod(ctx context.Context, namespace, server, podUID string) error { + if c.Disabled || podUID == "" { + return fmt.Errorf("agent: pod identity check requires an enabled client and pod UID") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, agentURL(namespace, server, "/v1/pods/"+url.PathEscape(podUID)+"/identity"), nil) + if err != nil { + return err + } + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("agent: pod identity check: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("agent: pod identity check: status %d", resp.StatusCode) + } + return nil +} + +// QuiescePod disables saving only on the selected Pod's agent. +func (c *Client) QuiescePod(ctx context.Context, namespace, server, podUID string) error { + return c.callPodQuiesce(ctx, namespace, server, podUID, "/quiesce", true) +} + +// UnquiescePod restores saving only on the selected Pod's agent. +func (c *Client) UnquiescePod(ctx context.Context, namespace, server, podUID string) error { + return c.callPodQuiesce(ctx, namespace, server, podUID, "/unquiesce", false) +} + +func (c *Client) callPodQuiesce(ctx context.Context, namespace, server, podUID, path string, expectQuiesced bool) error { + if c.Disabled || podUID == "" { + return fmt.Errorf("agent: pod-bound quiesce requires an enabled client and pod UID") + } + return c.callQuiesce(ctx, namespace, server, "/v1/pods/"+url.PathEscape(podUID)+path, expectQuiesced) +} + // expectQuiesced is what the response should report on a successful // completion: true after /quiesce, false after /unquiesce. A mismatch // is a protocol bug worth surfacing. diff --git a/operator/internal/agent/client_test.go b/operator/internal/agent/client_test.go index c3866488e..7d04dc9d1 100644 --- a/operator/internal/agent/client_test.go +++ b/operator/internal/agent/client_test.go @@ -122,6 +122,46 @@ func TestAgentURL(t *testing.T) { } } +func TestPodBoundQuiesceNeverFallsBackToLegacyRoute(t *testing.T) { + for _, status := range []int{http.StatusOK, http.StatusNotFound} { + t.Run(http.StatusText(status), func(t *testing.T) { + paths := []string{} + c, closeServer := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"quiesced":true}`)) + }) + defer closeServer() + err := c.QuiescePod(t.Context(), "ns", "server", "original-pod") + if (err == nil) != (status == http.StatusOK) { + t.Fatalf("status=%d error=%v", status, err) + } + if len(paths) != 1 || paths[0] != "/v1/pods/original-pod/quiesce" { + t.Fatalf("unbound mutation/fallback: %v", paths) + } + }) + } +} + +func TestPodBoundUnquiesceAndDisabledClient(t *testing.T) { + c, closeServer := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/pods/original-pod/unquiesce" { + t.Errorf("wrong cleanup target: %s", r.URL.Path) + } + _, _ = w.Write([]byte(`{"quiesced":false}`)) + }) + defer closeServer() + if err := c.UnquiescePod(t.Context(), "ns", "server", "original-pod"); err != nil { + t.Fatal(err) + } + if err := c.QuiescePod(t.Context(), "ns", "server", ""); err == nil { + t.Fatal("empty pod UID admitted") + } + if err := (&Client{Disabled: true}).QuiescePod(t.Context(), "ns", "server", "pod"); err == nil { + t.Fatal("disabled client reported identity-bound quiesce") + } +} + func TestQuiesce_DisabledNoOp(t *testing.T) { c := &Client{Disabled: true} if err := c.Quiesce(context.Background(), "ns", "srv"); err != nil { @@ -257,6 +297,32 @@ func TestUnquiesce_HappyPath(t *testing.T) { } } +func TestCheckPodRequiresIdentityEndpoint(t *testing.T) { + for _, status := range []int{http.StatusNoContent, http.StatusNotFound, http.StatusOK, http.StatusServiceUnavailable} { + t.Run(http.StatusText(status), func(t *testing.T) { + calls := 0 + c, cleanup := newMockClient(t, func(w http.ResponseWriter, req *http.Request) { + calls++ + if req.Method != http.MethodGet || req.URL.Path != "/v1/pods/original/identity" { + t.Errorf("unexpected preflight request: %s %s", req.Method, req.URL.Path) + } + w.WriteHeader(status) + }) + defer cleanup() + err := c.CheckPod(t.Context(), "ns", "server", "original") + if (err == nil) != (status == http.StatusNoContent) || calls != 1 { + t.Fatalf("identity result = %v, calls = %d", err, calls) + } + }) + } + if err := (&Client{Disabled: true}).CheckPod(t.Context(), "ns", "server", "original"); err == nil { + t.Fatal("disabled client confirmed identity") + } + if err := (&Client{}).CheckPod(t.Context(), "ns", "server", ""); err == nil { + t.Fatal("empty pod UID confirmed identity") + } +} + func TestQuiesce_TransportError(t *testing.T) { // Use a base transport that always errors. c := &Client{ diff --git a/operator/internal/controller/auxiliary_resources.go b/operator/internal/controller/auxiliary_resources.go new file mode 100644 index 000000000..8f7bc7901 --- /dev/null +++ b/operator/internal/controller/auxiliary_resources.go @@ -0,0 +1,65 @@ +package controller + +import ( + "encoding/json" + "fmt" + "io" + "strings" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/util/validation" +) + +// ParseAuxiliaryResources parses an install-level resource override. An absent +// flag returns nil (use defaults); an explicit object, including {}, replaces +// defaults wholesale. Reject typos before the operator starts creating pods. +func ParseAuxiliaryResources(value string) (*corev1.ResourceRequirements, error) { + if value == "" { + return nil, nil + } + var configured *corev1.ResourceRequirements + decoder := json.NewDecoder(strings.NewReader(value)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&configured); err != nil { + return nil, fmt.Errorf("decode resources: %w", err) + } + if configured == nil { + return nil, fmt.Errorf("resources must be a JSON object; use {} to clear defaults") + } + if err := decoder.Decode(new(any)); err != io.EOF { + return nil, fmt.Errorf("resources must contain exactly one JSON object") + } + if len(configured.Claims) != 0 { + return nil, fmt.Errorf("resource claims are not supported for auxiliary containers") + } + for _, values := range []corev1.ResourceList{configured.Requests, configured.Limits} { + for name, quantity := range values { + if len(validation.IsQualifiedName(string(name))) != 0 || quantity.Sign() < 0 { + return nil, fmt.Errorf("invalid resource %q or negative quantity", name) + } + } + } + for name, request := range configured.Requests { + if limit, exists := configured.Limits[name]; exists && request.Cmp(limit) > 0 { + return nil, fmt.Errorf("request for %s exceeds its limit", name) + } + } + return configured, nil +} + +func auxiliaryResources(configured *corev1.ResourceRequirements, cpu, memory string) corev1.ResourceRequirements { + if configured != nil { + return *configured.DeepCopy() + } + return corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(cpu), corev1.ResourceMemory: resource.MustParse(memory), + }} +} + +func optionalResources(values []*corev1.ResourceRequirements) *corev1.ResourceRequirements { + if len(values) == 0 { + return nil + } + return values[0] +} diff --git a/operator/internal/controller/auxiliary_resources_test.go b/operator/internal/controller/auxiliary_resources_test.go new file mode 100644 index 000000000..f63cef5f8 --- /dev/null +++ b/operator/internal/controller/auxiliary_resources_test.go @@ -0,0 +1,105 @@ +package controller + +import ( + "reflect" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestAuxiliaryResourceConfiguration(t *testing.T) { + for _, bad := range []string{"null", `{"request":{"cpu":"1"}}`, `{"requests":{"cpu":"-1"}}`, `{"requests":{"memory":"128Mi"},"limits":{"memory":"64Mi"}}`, `{} {}`} { + if _, err := ParseAuxiliaryResources(bad); err == nil { + t.Errorf("accepted invalid resources %s", bad) + } + } + unset, err := ParseAuxiliaryResources("") + if err != nil || unset != nil { + t.Fatalf("absent configuration: %v %v", unset, err) + } + cleared, err := ParseAuxiliaryResources(`{"requests":{},"limits":{}}`) + if err != nil || cleared == nil { + t.Fatalf("explicit clear: %v %v", cleared, err) + } + if got := auxiliaryResources(cleared, "25m", "64Mi"); len(got.Requests) != 0 || len(got.Limits) != 0 { + t.Fatal("clear restored default reservations") + } + defaults := auxiliaryResources(nil, "25m", "64Mi") + if defaults.Requests.Cpu().String() != "25m" || defaults.Requests.Memory().String() != "64Mi" || len(defaults.Limits) != 0 { + t.Fatalf("unexpected scheduling defaults: %+v", defaults) + } +} + +func TestAuxiliaryResourcesReachAgentAndResticContainers(t *testing.T) { + configured := &corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("73m")}} + gs := &gameplanev1alpha1.GameServer{} + tmpl := &gameplanev1alpha1.GameTemplate{} + agent := buildAgentContainer(gs, tmpl, nil, "agent", "", "", configured) + if !reflect.DeepEqual(agent.Resources, *configured) { + t.Fatalf("agent lost install resources: %+v", agent.Resources) + } + tmpl.Spec.Agent = &gameplanev1alpha1.AgentSpec{Image: "custom-agent"} + agent = buildAgentContainer(gs, tmpl, nil, "agent", "", "", configured) + if !reflect.DeepEqual(agent.Resources, *configured) { + t.Fatal("image-only template override removed resource defaults") + } + tmpl.Spec.Agent.Resources = corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("91Mi")}} + agent = buildAgentContainer(gs, tmpl, nil, "agent", "", "", configured) + if !reflect.DeepEqual(agent.Resources, tmpl.Spec.Agent.Resources) { + t.Fatal("template agent resources did not replace the install defaults") + } + b := &gameplanev1alpha1.Backup{Spec: gameplanev1alpha1.BackupSpec{RepoRef: &gameplanev1alpha1.SecretKeySelector{Name: "repo"}}} + br := &BackupReconciler{BackupResources: configured} + backup := br.buildBackupPodSpec(b, tmpl) + forget := br.buildForgetPodSpec(b) + rr := &RestoreReconciler{BackupResources: configured} + restore := rr.buildRestorePodSpec(&gameplanev1alpha1.Restore{}, b, tmpl) + for _, container := range append(append(append(backup.Containers, backup.InitContainers...), forget.Containers...), restore.Containers...) { + if !reflect.DeepEqual(container.Resources, *configured) { + t.Errorf("container %s lost resource configuration: %+v", container.Name, container.Resources) + } + } + if init := buildConfigInitContainer("init", tmpl, configured); !reflect.DeepEqual(init.Resources, *configured) { + t.Fatal("config init lost install resource configuration") + } + // Builders must not return the shared maps: one container's customization + // cannot mutate the defaults for another server or operation. + agent.Resources.Requests[corev1.ResourceMemory] = resource.MustParse("1Mi") + if tmpl.Spec.Agent.Resources.Requests.Memory().String() != "91Mi" { + t.Fatal("agent resources alias the template") + } +} + +func TestAuxiliaryDeploymentResourceOverrideAndRemoval(t *testing.T) { + s := wipeScheme(t) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "games", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "frp", Frp: &gameplanev1alpha1.FrpTunnelSpec{ServerAddr: "relay.example"}}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + custom := &corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("81m")}} + for _, configured := range []*corev1.ResourceRequirements{nil, custom, {}} { + r.SentinelResources, r.TunnelResources = configured, configured + if err := r.reconcileSentinel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + if err := r.reconcileTunnel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + for _, component := range []struct{ suffix, memory string }{{"waker", "32Mi"}, {"tunnel", "64Mi"}} { + var dep appsv1.Deployment + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-" + component.suffix}, &dep); err != nil { + t.Fatal(err) + } + want := auxiliaryResources(configured, "25m", component.memory) + if !reflect.DeepEqual(dep.Spec.Template.Spec.Containers[0].Resources, want) { + t.Fatalf("%s did not apply/clear resources", component.suffix) + } + } + } +} diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index b350a5448..1c7d97965 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -119,6 +119,9 @@ type BackupReconciler struct { // operator flag so air-gapped installs can point it at a private // registry mirror. Empty falls back to DefaultResticImage. ResticImage string + // BackupResources replaces resources for backup/init and repository cleanup + // workers. Nil uses defaults; an explicit empty object clears them. + BackupResources *corev1.ResourceRequirements // JobBackoffLimit / JobActiveDeadlineSeconds bound the restic backup Job. // Set from operator flags; nil / zero fall back to // DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds. @@ -247,14 +250,12 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr if !b.DeletionTimestamp.IsZero() { return r.finalizeDelete(ctx, &b) } - // Only Backups that might quiesce the world need to block on releasing - // it before deletion; everything else deletes immediately. - if b.Spec.Quiesce && !controllerutil.ContainsFinalizer(&b, gameplanev1alpha1.BackupFinalizer) { + // Every copy owns the source until its workers drain, even without quiesce. + if !controllerutil.ContainsFinalizer(&b, gameplanev1alpha1.BackupFinalizer) { controllerutil.AddFinalizer(&b, gameplanev1alpha1.BackupFinalizer) if err := r.Update(ctx, &b); err != nil { return ctrl.Result{}, err } - return ctrl.Result{}, nil } // Restic Backups own a snapshot in the repository, so hold their deletion @@ -275,6 +276,11 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr // Re-entering the full pipeline would re-validate the GameServer and repo // Secret (flipping an already-Succeeded Backup to Failed if either was since // deleted) and would re-create a cleaned-up Job, running a second backup. + if b.Status.Phase == gameplanev1alpha1.BackupPhaseSucceeded || b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + if err := r.syncBackupConsistency(ctx, &b); err != nil { + return ctrl.Result{}, err + } + } if b.Status.Phase == gameplanev1alpha1.BackupPhaseSucceeded && b.Status.SnapshotID == "" { return r.retrySnapshotScrape(ctx, &b) } @@ -288,6 +294,19 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { return r.runUnquiesce(ctx, &b) } + // Observe pre-upgrade workers even when multiple legacy quiesced copies + // already overlap. They must finish and unquiesce before new admission; + // trying to acquire exclusive ownership first would deadlock their cleanup. + if legacyBackupOwesUnquiesce(&b) && b.Spec.Strategy != "volume-snapshot" { + var existing batchv1.Job + err := r.backupAPIReader().Get(ctx, client.ObjectKeyFromObject(&b), &existing) + if err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil && metav1.IsControlledBy(&existing, &b) { + return r.mirrorJobStatus(ctx, &b, &existing) + } + } // Resolve the target before building the Job: a Backup against a // GameServer that doesn't exist would produce a pod waiting forever @@ -303,13 +322,6 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, err } - // A missing template is not fatal: it only refines the FSGroup, and - // buildBackupPodSecurityContext falls back to 65532 for a zero template. - var tmpl gameplanev1alpha1.GameTemplate - if err := r.Get(ctx, types.NamespacedName{Name: gs.Spec.TemplateRef.Name}, &tmpl); err != nil && !apierrors.IsNotFound(err) { - return ctrl.Result{}, err - } - // Volume-snapshot backups take a CSI snapshot of the data PVC rather // than running restic, so they branch off here — before the restic // repo Secret checks, which don't apply to them. @@ -348,6 +360,19 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr if res, err := r.prepareBackupQuiesce(ctx, &b); err != nil || res.RequeueAfter > 0 { return res, err } + // Admission refreshes the target. Resolve the pin only after owning the + // lifecycle fence, so a concurrent rollout cannot change the copy layout. + bound, err := r.ownedBackupTarget(ctx, &b) + if err != nil { + return ctrl.Result{}, err + } + if bound == nil { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + tmpl, err := effectiveGameTemplate(ctx, r.backupAPIReader(), bound) + if err != nil && (bound.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { + return ctrl.Result{}, err + } backoff, deadline := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace}} @@ -356,7 +381,7 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr job.Spec.BackoffLimit = &backoff job.Spec.ActiveDeadlineSeconds = &deadline job.Spec.Template.Labels = map[string]string{backupRestoreJobLabel: backupRestoreJobValue, backupUIDLabel: string(b.UID)} - job.Spec.Template.Spec = r.buildBackupPodSpec(&b, &tmpl) + job.Spec.Template.Spec = r.buildBackupPodSpec(&b, tmpl) job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever } return controllerutil.SetControllerReference(&b, job, r.Scheme) @@ -372,23 +397,62 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr // requeues; ErrUnsupported is recorded as "attempted but unsupported" // so the matching unquiesce step is skipped on terminal phase. func (r *BackupReconciler) maybeQuiesce(ctx context.Context, b *gameplanev1alpha1.Backup) error { + preparation, err := parseBackupPreparation(b) + if err != nil { + return err + } + if preparation != nil && preparation.Outcome == "already-stopped" { + return nil + } if !b.Spec.Quiesce { return nil } - if r.AgentClient == nil { + if !agentQuiesceAvailable(r.AgentClient) { + if preparation != nil && preparation.Outcome == "live" && b.Annotations[annoQuiesceAttempted] != "true" && b.Annotations[annoQuiesceAttempted] != "unsupported" { + err := fmt.Errorf("agent quiesce client was disabled after live preparation") + if _, ferr := r.fail(ctx, b, err.Error()); ferr != nil { + return ferr + } + return err + } return nil } if state := b.Annotations[annoQuiesceAttempted]; state != "" && state != "pending" { return nil } + // Verify the versioned endpoint before creating the first cleanup debt. + // Old agents cannot mutate through it and must not pin an owner that + // would prevent their own upgrade. The POST still checks the UID again. + if preparation != nil && b.Annotations[annoQuiesceAttempted] == "" { + bound, ok := r.AgentClient.(podAgentQuiescer) + if !ok { + err = fmt.Errorf("agent client does not support pod-bound quiesce") + } else { + err = bound.CheckPod(ctx, b.Namespace, b.Spec.ServerRef.Name, string(preparation.PodUID)) + } + if err != nil { + _, ferr := r.fail(ctx, b, fmt.Sprintf("quiesce identity check failed: %v", err)) + if ferr != nil { + return ferr + } + return err + } + } // Persist cleanup intent BEFORE save-off: a crash or a partial agent failure // must still owe save-on even when quiesced-at was never recorded. + if err := r.setLifecycleUnquiesceIntent(ctx, b, true); err != nil { + return err + } patchBackupAnnotations(b, map[string]string{annoQuiesceAttempted: "pending"}) if err := r.Update(ctx, b); err != nil { return err } - err := r.AgentClient.Quiesce(ctx, b.Namespace, b.Spec.ServerRef.Name) + var podUID types.UID + if preparation != nil { + podUID = preparation.PodUID + } + err = callBackupQuiesce(ctx, r.AgentClient, b.Namespace, b.Spec.ServerRef.Name, podUID, false) switch { case err == nil: now := time.Now().UTC().Format(time.RFC3339) @@ -402,6 +466,9 @@ func (r *BackupReconciler) maybeQuiesce(ctx context.Context, b *gameplanev1alpha patchBackupAnnotations(b, map[string]string{ annoQuiesceAttempted: "unsupported", }) + if err := r.setLifecycleUnquiesceIntent(ctx, b, false); err != nil { + return err + } default: // Hard failure (network, RCON down). Surface as Failed: a // backup taken against an unflushed Minecraft world can be @@ -520,6 +587,7 @@ func (r *BackupReconciler) mirrorJobStatus( cond.Reason = string(phase) } newConds := upsertCondition(b.Status.Conditions, cond) + newConds = upsertCondition(newConds, backupConsistencyCondition(b)) if !sameConditions(b.Status.Conditions, newConds) { b.Status.Conditions = newConds dirty = true @@ -609,8 +677,8 @@ func (r *BackupReconciler) runUnquiesce(ctx context.Context, b *gameplanev1alpha } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } - // When an unquiesce was actually performed, record success so the - // "world might be frozen" signal clears. + // Confirmed save-on or confirmed original-process exit clears the + // "world might be frozen" signal, with distinct condition reasons. if _, ok := b.Annotations[annoUnquiescedAt]; ok { if cerr := r.setUnquiescedCondition(ctx, b, true, ""); cerr != nil { return ctrl.Result{}, cerr @@ -629,6 +697,10 @@ func (r *BackupReconciler) setUnquiescedCondition( if ok { cond.Status = metav1.ConditionTrue cond.Reason = "Unquiesced" + if b.Annotations[backupCleanupOutcomeAnnotation] == "source-gone" { + cond.Reason = "SourceProcessGone" + cond.Message = "original process no longer exists; no unquiesce RPC was sent" + } } else { cond.Status = metav1.ConditionFalse cond.Reason = "Failed" @@ -720,49 +792,29 @@ func (r *BackupReconciler) finalizeUnquiesce(ctx context.Context, b *gameplanev1 return ctrl.Result{}, nil } -// unquiesceTargetGone reports whether there is nothing left to unquiesce: -// the target GameServer no longer exists or is itself being deleted, or its -// pod no longer exists. A GameServer can be gone-together-with-its-Backup -// under an auto BackupSchedule's owner chain, and a pod alone can be gone if -// it was deleted independently of the GameServer; either way a fresh pod -// starts with auto-save on, so retrying the unquiesce would never succeed -// and would only block this Backup's deletion. +// unquiesceTargetGone requires evidence that the original process has exited. +// A deleted/deleting GameServer alone is insufficient: its pod can still run. func (r *BackupReconciler) unquiesceTargetGone(ctx context.Context, b *gameplanev1alpha1.Backup) (bool, error) { targetName := b.Spec.ServerRef.Name - var targetUID types.UID if value := b.Annotations[backupTargetAnnotation]; value != "" { identity, err := parseBackupIdentity(value) if err != nil { return false, err } - targetName, targetUID = identity.Name, identity.UID + targetName = identity.Name } if targetName == "" { return true, nil } - gs := &gameplanev1alpha1.GameServer{} - key := types.NamespacedName{Namespace: b.Namespace, Name: targetName} - switch err := r.Get(ctx, key, gs); { - case apierrors.IsNotFound(err): - return true, nil - case err != nil: - return false, fmt.Errorf("get gameserver %s to check unquiesce target: %w", key.Name, err) - case gs.DeletionTimestamp != nil: - return true, nil - } - if targetUID != "" && gs.UID != targetUID { - return true, nil + preparation, err := parseBackupPreparation(b) + if err != nil { + return false, err } - - var pod corev1.Pod - podKey := types.NamespacedName{Namespace: b.Namespace, Name: gs.Name + "-0"} - switch err := r.Get(ctx, podKey, &pod); { - case apierrors.IsNotFound(err): - return true, nil - case err != nil: - return false, fmt.Errorf("get pod %s to check unquiesce target: %w", podKey.Name, err) + var podUID types.UID + if preparation != nil { + podUID = preparation.PodUID } - return false, nil + return backupQuiesceProcessGone(ctx, r.backupAPIReader(), b.Namespace, targetName, podUID) } // unquiesceRetryExpired records (on first call) when finalizeDelete's @@ -850,10 +902,35 @@ func (r *BackupReconciler) maybeUnquiesce(ctx context.Context, b *gameplanev1alp return fmt.Errorf("legacy quiesce cleanup is blocked by a protected backup") } } - if r.AgentClient == nil { + if targetName != "" { + gone, err := r.unquiesceTargetGone(ctx, b) + if err != nil { + return err + } + if gone { + patchBackupAnnotations(b, map[string]string{backupCleanupOutcomeAnnotation: "source-gone"}) + return r.recordBackupUnquiesced(ctx, b) + } + } + if !agentQuiesceAvailable(r.AgentClient) { return fmt.Errorf("cannot release quiesced world without an agent client") } - if err := r.AgentClient.Unquiesce(ctx, b.Namespace, targetName); err != nil { + preparation, err := parseBackupPreparation(b) + if err != nil { + return err + } + var podUID types.UID + if preparation != nil { + podUID = preparation.PodUID + } + if err := callBackupQuiesce(ctx, r.AgentClient, b.Namespace, targetName, podUID, true); err != nil { + return err + } + return r.recordBackupUnquiesced(ctx, b) +} + +func (r *BackupReconciler) recordBackupUnquiesced(ctx context.Context, b *gameplanev1alpha1.Backup) error { + if err := r.setLifecycleUnquiesceIntent(ctx, b, false); err != nil { return err } patchBackupAnnotations(b, map[string]string{ @@ -901,6 +978,7 @@ func (r *BackupReconciler) fail(ctx context.Context, b *gameplanev1alpha1.Backup Message: msg, ObservedGeneration: b.Generation, }) + b.Status.Conditions = upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) if err := r.Status().Update(ctx, b); err != nil { return ctrl.Result{}, err } @@ -965,9 +1043,10 @@ func (r *BackupReconciler) buildBackupPodSpec(b *gameplanev1alpha1.Backup, tmpl return corev1.PodSpec{ SecurityContext: buildBackupPodSecurityContext(tmpl), InitContainers: []corev1.Container{{ - Name: "restic-init", - Image: resticImageOrDefault(r.ResticImage), - Command: []string{"/bin/sh", "-c"}, + Name: "restic-init", + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), + Command: []string{"/bin/sh", "-c"}, // `cat config` is the canonical "does the repo exist" probe; // `init` is idempotent against an empty target. Args: []string{"restic cat config --no-lock >/dev/null 2>&1 || restic init"}, @@ -978,10 +1057,11 @@ func (r *BackupReconciler) buildBackupPodSpec(b *gameplanev1alpha1.Backup, tmpl SecurityContext: containerSec, }}, Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), - Args: args, - Env: env, + Name: "restic", + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), + Args: args, + Env: env, VolumeMounts: []corev1.VolumeMount{ {Name: "data", MountPath: "/data", ReadOnly: true}, {Name: "cache", MountPath: "/tmp"}, diff --git a/operator/internal/controller/backup_envtest_test.go b/operator/internal/controller/backup_envtest_test.go index 87ce6cc7e..f577563a6 100644 --- a/operator/internal/controller/backup_envtest_test.go +++ b/operator/internal/controller/backup_envtest_test.go @@ -8,9 +8,11 @@ import ( "testing" "time" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -379,6 +381,20 @@ func TestBackup_QuiesceAnnotationSetBeforeJob(t *testing.T) { fa := &fakeAgent{} startMgr(t, ns, withBackupReconciler(backupReconcilerOpts{agent: fa})) seedGameServer(t, ns, "smp") + // envtest has no StatefulSet controller or kubelet: provide the observed + // running source whose immutable identity is bound before the agent RPC. + var workload appsv1.StatefulSet + if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "smp"}, &workload); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "smp-0", Namespace: ns, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: workload.Name, UID: workload.UID, Controller: ownerBoolPtr(true)}}}, Spec: workload.Spec.Template.Spec} + if err := k8sClient.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + pod.Status.Phase = corev1.PodRunning + if err := k8sClient.Status().Update(context.Background(), pod); err != nil { + t.Fatal(err) + } if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo")); err != nil { t.Fatalf("create secret: %v", err) @@ -577,6 +593,7 @@ func TestBackup_UsesTemplateSecurityContext(t *testing.T) { if err := k8sClient.Create(context.Background(), gs); err != nil { t.Fatalf("create gameserver: %v", err) } + seedBackupWorkload(t, gs) if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo-secret")); err != nil { t.Fatalf("create secret: %v", err) } diff --git a/operator/internal/controller/backup_forget.go b/operator/internal/controller/backup_forget.go index 86f1c53cf..2d97d9410 100644 --- a/operator/internal/controller/backup_forget.go +++ b/operator/internal/controller/backup_forget.go @@ -420,6 +420,12 @@ func forgetJobName(backupName string) string { func (r *BackupReconciler) buildForgetPodSpec(b *gameplanev1alpha1.Backup) corev1.PodSpec { nonRoot := true uid := int64(65532) + resources := auxiliaryResources(r.BackupResources, "50m", "128Mi") + if r.BackupResources == nil { + // Preserve the existing bounded repository-cleanup default. Explicit + // install overrides replace it along with other restic worker resources. + resources.Limits = corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")} + } env := resticEnv(b.Spec.RepoRef.Name) env = append(env, corev1.EnvVar{Name: forgetSnapshotIDEnv, Value: b.Status.SnapshotID}) @@ -433,20 +439,12 @@ func (r *BackupReconciler) buildForgetPodSpec(b *gameplanev1alpha1.Backup) corev SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, }, Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), - Command: []string{"/bin/sh", "-c"}, - Args: []string{forgetScript}, - Env: env, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("50m"), - corev1.ResourceMemory: resource.MustParse("128Mi"), - }, - Limits: corev1.ResourceList{ - corev1.ResourceMemory: resource.MustParse("512Mi"), - }, - }, + Name: "restic", + Image: resticImageOrDefault(r.ResticImage), + Command: []string{"/bin/sh", "-c"}, + Args: []string{forgetScript}, + Env: env, + Resources: resources, VolumeMounts: []corev1.VolumeMount{ {Name: "cache", MountPath: "/tmp"}, }, diff --git a/operator/internal/controller/backup_forget_envtest_test.go b/operator/internal/controller/backup_forget_envtest_test.go index 3a6c642b4..9dbf6bf8e 100644 --- a/operator/internal/controller/backup_forget_envtest_test.go +++ b/operator/internal/controller/backup_forget_envtest_test.go @@ -16,6 +16,7 @@ import ( "k8s.io/client-go/tools/events" "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/manager" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" @@ -149,17 +150,17 @@ func TestBackupForget_FinalizerOnlyOnResticBackups(t *testing.T) { return false, "restic backup has no snapshot finalizer yet" }) - // Once the volume-snapshot Backup is Running the reconciler has been through - // the point where it would have added a finalizer. + // Every strategy gets an operation cleanup finalizer before admission. Do + // not wait for Running: the restic backup may hold the shared server owner. eventually(t, func() (bool, string) { b := getBackup(t, ns, "smp-vs") - if b.Status.Phase != gameplanev1alpha1.BackupPhaseRunning { + if !controllerutil.ContainsFinalizer(b, gameplanev1alpha1.BackupFinalizer) { return false, describeBackupStatus(b) } return true, "" }) - if got := getBackup(t, ns, "smp-vs").Finalizers; len(got) != 0 { - t.Errorf("volume-snapshot backup finalizers = %v, want none", got) + if got := getBackup(t, ns, "smp-vs"); controllerutil.ContainsFinalizer(got, gameplanev1alpha1.BackupSnapshotFinalizer) { + t.Errorf("volume-snapshot backup finalizers = %v, want no restic snapshot finalizer", got.Finalizers) } } diff --git a/operator/internal/controller/backup_forget_test.go b/operator/internal/controller/backup_forget_test.go index 12dd1aa51..ef917dd7f 100644 --- a/operator/internal/controller/backup_forget_test.go +++ b/operator/internal/controller/backup_forget_test.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -180,7 +181,7 @@ func TestReconcile_AddsSnapshotFinalizerForResticBackup(t *testing.T) { } // Volume-snapshot and Failed Backups never write a restic snapshot, so they -// must not be held on deletion. +// need operation cleanup but must not be held for restic snapshot deletion. func TestReconcile_NoSnapshotFinalizerWhenNothingToForget(t *testing.T) { volume := forgetBackup() volume.DeletionTimestamp = nil @@ -205,8 +206,11 @@ func TestReconcile_NoSnapshotFinalizerWhenNothingToForget(t *testing.T) { if !ok { t.Fatal("backup vanished") } - if len(got.Finalizers) != 0 { - t.Errorf("finalizers = %v, want none", got.Finalizers) + if !controllerutil.ContainsFinalizer(got, gameplanev1alpha1.BackupFinalizer) { + t.Errorf("finalizers = %v, want operation cleanup finalizer", got.Finalizers) + } + if controllerutil.ContainsFinalizer(got, gameplanev1alpha1.BackupSnapshotFinalizer) { + t.Errorf("finalizers = %v, want no restic snapshot finalizer", got.Finalizers) } }) } diff --git a/operator/internal/controller/backup_guard.go b/operator/internal/controller/backup_guard.go index 4cc09e2f9..ea0f8d84b 100644 --- a/operator/internal/controller/backup_guard.go +++ b/operator/internal/controller/backup_guard.go @@ -3,6 +3,7 @@ package controller import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -61,9 +62,6 @@ func (r *BackupReconciler) backupAPIReader() client.Reader { // the target before claiming it so a crash or a same-name replacement cannot // redirect cleanup to a different game. func (r *BackupReconciler) claimBackupTarget(ctx context.Context, b *gameplanev1alpha1.Backup) (bool, error) { - if !b.Spec.Quiesce { - return true, nil - } legacy, err := r.legacyQuiescedBackups(ctx, b) if err != nil { return false, err @@ -72,10 +70,9 @@ func (r *BackupReconciler) claimBackupTarget(ctx context.Context, b *gameplanev1 // Already-running pre-upgrade copies must be allowed to finish, but // nobody may start a new protected copy until all of their cleanup is // confirmed. Each legacy cleanup also waits for all legacy workers. - if legacyBackupOwesUnquiesce(b) { - return true, nil + if !legacyBackupOwesUnquiesce(b) { + return false, nil } - return false, nil } var gs gameplanev1alpha1.GameServer if err := r.backupAPIReader().Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: b.Spec.ServerRef.Name}, &gs); err != nil { @@ -98,55 +95,11 @@ func (r *BackupReconciler) claimBackupTarget(ctx context.Context, b *gameplanev1 return false, err } } - owner := backupIdentityJSON(b.Name, b.UID) - guard := gs.Annotations[backupGuardAnnotation] - if guard == owner { - return r.recoverBackupTarget(ctx, b) - } - if guard != "" { - id, err := parseBackupIdentity(guard) - if err != nil { - return false, err - } // Malformed guards fail closed. - var holder gameplanev1alpha1.Backup - err = r.backupAPIReader().Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: id.Name}, &holder) - if err == nil && holder.UID == id.UID { - return false, nil - } - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - // Persist recovery intent, then transfer ownership with CAS BEFORE - // touching the agent. Concurrent contenders cannot both call save-on - // while one of them has already started a new protected copy. - patchBackupAnnotations(b, map[string]string{backupRecoveryAnnotation: guard, annoQuiesceAttempted: "pending"}) - if err := r.Update(ctx, b); err != nil { - return false, err - } - // Preserve every predecessor on the surviving GameServer, atomically - // with the owner transfer. A recovering Backup can itself disappear - // before an earlier owner's copy worker has stopped. - lineage, err := parseBackupRecoveryLineage(gs.Annotations[backupRecoveryLineageAnnotation]) - if err != nil { - return false, err - } - found := false - for _, predecessor := range lineage { - if predecessor == id { - found = true - } - } - if !found { - lineage = append(lineage, id) - } - data, _ := json.Marshal(lineage) - gs.Annotations[backupRecoveryLineageAnnotation] = string(data) - } - if gs.Annotations == nil { - gs.Annotations = make(map[string]string) - } - gs.Annotations[backupGuardAnnotation] = owner - if err := r.Update(ctx, &gs); err != nil { + state := b.Annotations[annoQuiesceAttempted] + owesUnquiesce := state != "" && state != "unsupported" && b.Annotations[annoUnquiescedAt] == "" + claimed, err := claimLifecycleOperation(ctx, r.Client, r.backupAPIReader(), &gs, + backupLifecycleIdentity(&gs, b, owesUnquiesce), r.AgentClient) + if err != nil || !claimed { return false, err } return r.recoverBackupTarget(ctx, b) @@ -166,11 +119,19 @@ func (r *BackupReconciler) recoverBackupTarget(ctx context.Context, b *gameplane return false, err } } - if r.AgentClient == nil { - return false, fmt.Errorf("cannot recover orphaned quiesce without an agent client") + // Legacy owners have no pod UID. Retire them only on positive absence or + // terminal-pod proof; never bind an old RPC to a newly observed identity. + gone, err := backupQuiesceProcessGone(ctx, r.backupAPIReader(), b.Namespace, b.Spec.ServerRef.Name, "") + if err != nil { + return false, err } - if err := r.AgentClient.Unquiesce(ctx, b.Namespace, b.Spec.ServerRef.Name); err != nil { - return false, fmt.Errorf("recover orphaned quiesce: %w", err) + if !gone { + if !agentQuiesceAvailable(r.AgentClient) { + return false, fmt.Errorf("cannot recover orphaned quiesce without an agent client") + } + if err := r.AgentClient.Unquiesce(ctx, b.Namespace, b.Spec.ServerRef.Name); err != nil { + return false, fmt.Errorf("recover orphaned quiesce: %w", err) + } } delete(b.Annotations, backupRecoveryAnnotation) delete(b.Annotations, annoQuiesceAttempted) @@ -227,8 +188,19 @@ func (r *BackupReconciler) backupRecoveryIdentities(ctx context.Context, b *game } func (r *BackupReconciler) backupCleanupWorkersLive(ctx context.Context, b *gameplanev1alpha1.Backup, cancel bool) (bool, error) { - if !b.Spec.Quiesce && b.Annotations[annoQuiesceAttempted] == "" { - return false, nil + gs, err := r.ownedBackupTarget(ctx, b) + if err != nil { + return false, err + } + if gs != nil && gs.Annotations[lifecycleOperationAnnotation] != "" { + op, err := parseLifecycleOperation(gs.Annotations[lifecycleOperationAnnotation]) + if err != nil { + return false, err + } + ready, err := recoverLifecycleOperation(ctx, r.Client, r.backupAPIReader(), gs, op, r.AgentClient) + if err != nil || !ready { + return !ready, err + } } busy, err := r.backupWorkersLive(ctx, b.Namespace, backupIdentity{Name: b.Name, UID: b.UID}, cancel) if err != nil || busy { @@ -283,12 +255,54 @@ func (r *BackupReconciler) legacyQuiescedBackups(ctx context.Context, b *gamepla func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { claimed, err := r.claimBackupTarget(ctx, b) if err != nil { + if errors.Is(err, errLifecycleGenerationChanged) { + _, err = r.fail(ctx, b, err.Error()) + return ctrl.Result{RequeueAfter: time.Nanosecond}, err + } return ctrl.Result{}, err } if !claimed { return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } - return ctrl.Result{}, r.maybeQuiesce(ctx, b) + gs, err := r.ownedBackupTarget(ctx, b) + if err != nil { + return ctrl.Result{}, err + } + if gs == nil { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + fenced, err := fenceBackupWorkload(ctx, r.Client, r.backupAPIReader(), gs) + if err != nil || !fenced { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + ready, err := r.prepareBackupSource(ctx, b, gs) + if errors.Is(err, errBackupSourceChanged) { + _, err = r.fail(ctx, b, err.Error()) + return ctrl.Result{RequeueAfter: time.Nanosecond}, err + } + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + if err := r.maybeQuiesce(ctx, b); err != nil { + return ctrl.Result{}, err + } + // The Service RPC may overlap a pod replacement. Never start a copy on a + // different process from the one pinned before the request. + ready, err = r.prepareBackupSource(ctx, b, gs) + if errors.Is(err, errBackupSourceChanged) { + _, err = r.fail(ctx, b, err.Error()) + return ctrl.Result{RequeueAfter: time.Nanosecond}, err + } + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + if err := r.recordBackupManifest(ctx, b, gs); err != nil { + // Metadata must be durable before capture. Do not leave saving paused + // indefinitely when a lookup/status write prevents the copy starting. + _, failErr := r.fail(ctx, b, "cannot record capture metadata: "+err.Error()) + return ctrl.Result{RequeueAfter: time.Second}, failErr + } + return ctrl.Result{}, nil } // Return only the originally bound GameServer. A foreign owner means this @@ -313,6 +327,15 @@ func (r *BackupReconciler) ownedBackupTarget(ctx context.Context, b *gameplanev1 if gs.UID != id.UID || gs.Annotations[backupGuardAnnotation] != backupIdentityJSON(b.Name, b.UID) { return nil, nil } + if value := gs.Annotations[lifecycleOperationAnnotation]; value != "" { + op, err := parseLifecycleOperation(value) + if err != nil { + return nil, err + } + if !sameLifecycleOwner(op.lifecycleIdentity, backupLifecycleIdentity(&gs, b, false)) { + return nil, nil + } + } return &gs, nil } @@ -321,6 +344,9 @@ func (r *BackupReconciler) releaseBackupTarget(ctx context.Context, b *gameplane if err != nil || gs == nil { return err } + if err := clearLifecycleOperation(ctx, r.Client, r.backupAPIReader(), gs, backupLifecycleIdentity(gs, b, false)); err != nil { + return err + } delete(gs.Annotations, backupGuardAnnotation) delete(gs.Annotations, backupRecoveryLineageAnnotation) return r.Update(ctx, gs) diff --git a/operator/internal/controller/backup_guard_test.go b/operator/internal/controller/backup_guard_test.go index ac0e2be97..52efc597e 100644 --- a/operator/internal/controller/backup_guard_test.go +++ b/operator/internal/controller/backup_guard_test.go @@ -8,6 +8,7 @@ import ( "time" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -27,13 +28,30 @@ import ( type guardQuiescer struct { quiesced, unquiesced int quiesceErr error + unquiesceErr error + checkErr error + afterQuiesce func() } func (q *guardQuiescer) Quiesce(context.Context, string, string) error { q.quiesced++ return q.quiesceErr } -func (q *guardQuiescer) Unquiesce(context.Context, string, string) error { q.unquiesced++; return nil } +func (q *guardQuiescer) Unquiesce(context.Context, string, string) error { + q.unquiesced++ + return q.unquiesceErr +} +func (q *guardQuiescer) CheckPod(context.Context, string, string, string) error { return q.checkErr } +func (q *guardQuiescer) QuiescePod(ctx context.Context, namespace, server, _ string) error { + err := q.Quiesce(ctx, namespace, server) + if q.afterQuiesce != nil { + q.afterQuiesce() + } + return err +} +func (q *guardQuiescer) UnquiescePod(ctx context.Context, namespace, server, _ string) error { + return q.Unquiesce(ctx, namespace, server) +} func guardBackup(name string) *gameplanev1alpha1.Backup { return &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns", UID: types.UID(name + "-uid"), Finalizers: []string{gameplanev1alpha1.BackupFinalizer, gameplanev1alpha1.BackupSnapshotFinalizer}}, Spec: gameplanev1alpha1.BackupSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: "server"}, RepoRef: &gameplanev1alpha1.SecretKeySelector{Name: "repo"}, Quiesce: true}} @@ -45,7 +63,13 @@ func guardFixture(t *testing.T, backups ...*gameplanev1alpha1.Backup) (*BackupRe if err := snapshotv1.AddToScheme(s); err != nil { t.Fatal(err) } + if err := appsv1.AddToScheme(s); err != nil { + t.Fatal(err) + } objects := []client.Object{&gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "repo", Namespace: "ns"}, Data: map[string][]byte{"repo": []byte("repo"), "password": []byte("password")}}} + one := int32(1) + objects = append(objects, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "workload-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "GameServer", Name: "server", UID: "server-uid", Controller: ownerBoolPtr(true)}}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}}) + objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "game-pod-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: "server", UID: "workload-uid", Controller: ownerBoolPtr(true)}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}) for _, b := range backups { objects = append(objects, b) } @@ -85,12 +109,23 @@ func TestBackupGuardSerializesAcrossBackupsAndRestart(t *testing.T) { if len(jobs.Items) != 1 || q.quiesced != 1 || q.unquiesced != 0 { t.Fatalf("overlapping protected copies: jobs=%d, agent=%+v", len(jobs.Items), q) } + first := guardGetBackup(t, r, "first") + if first.Status.CaptureManifest == nil || first.Status.CaptureManifest.QuiesceOutcome != "confirmed" { + t.Fatalf("worker started without confirmed capture-time metadata: %+v", first.Status) + } + if second := guardGetBackup(t, r, "second"); second.Status.CaptureManifest != nil { + t.Fatalf("queued copy recorded metadata before admission: %+v", second.Status.CaptureManifest) + } job := &jobs.Items[0] job.Status.Succeeded = 1 if err := r.Status().Update(context.Background(), job); err != nil { t.Fatal(err) } guardPass(t, r, "first") + first = guardGetBackup(t, r, "first") + if condition := meta.FindStatusCondition(first.Status.Conditions, "Consistency"); condition == nil || condition.Status != metav1.ConditionTrue || condition.Reason != "Quiesced" { + t.Fatalf("completed copy did not expose achieved quiesce: %+v", first.Status) + } guardPass(t, r, "second") if q.quiesced != 2 || q.unquiesced != 1 { t.Fatalf("second did not acquire after first cleanup: %+v", q) diff --git a/operator/internal/controller/backup_manifest.go b/operator/internal/controller/backup_manifest.go new file mode 100644 index 000000000..c2c2afb70 --- /dev/null +++ b/operator/internal/controller/backup_manifest.go @@ -0,0 +1,180 @@ +package controller + +import ( + "context" + "sort" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// Called after lifecycle ownership and the workload fence, before creating the +// copy worker. Requeues and later template edits never rewrite this manifest. +func (r *BackupReconciler) recordBackupManifest(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer) error { + if b.Status.CaptureManifest != nil { + return nil + } + reader := r.backupAPIReader() + var worker client.Object = &batchv1.Job{} + if b.Spec.Strategy == "volume-snapshot" { + worker = &snapshotv1.VolumeSnapshot{} + } + if err := reader.Get(ctx, client.ObjectKeyFromObject(b), worker); err == nil { + // Existing workers predate this metadata. Current layout cannot prove + // what they captured, even if their requested quiesce was successful. + b.Status.CaptureManifest = &gameplanev1alpha1.BackupCaptureManifest{State: "LegacyUnknown"} + return r.Status().Update(ctx, b) + } else if !apierrors.IsNotFound(err) { + return err + } + var ss appsv1.StatefulSet + err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + var layout corev1.PodSpec + source := "Unavailable" + if err == nil && metav1.IsControlledBy(&ss, gs) { + layout, source = ss.Spec.Template.Spec, "StatefulSet" + var pod corev1.Pod + err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil && metav1.IsControlledBy(&pod, &ss) { + layout, source = pod.Spec, "Pod" + } + } + tmpl, err := effectiveGameTemplate(ctx, reader, gs) + if err != nil && (gs.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { + return err + } + primary := gs.Name + "-data" + volumes := map[string]*gameplanev1alpha1.BackupVolumeCoverage{ + primary: {ClaimName: primary, Included: true, Classification: "data", CoverageReason: "PrimaryDataClaim"}, + } + podNames := map[string]string{} + for _, volume := range layout.Volumes { + if volume.PersistentVolumeClaim == nil { + continue + } + name := volume.PersistentVolumeClaim.ClaimName + podNames[volume.Name] = name + if volumes[name] == nil { + volumes[name] = &gameplanev1alpha1.BackupVolumeCoverage{ClaimName: name, Classification: "unclassified", CoverageReason: "NotCovered"} + } + } + for _, container := range layout.Containers { + if container.Name != "game" { + continue + } + for _, mount := range container.VolumeMounts { + if volume := volumes[podNames[mount.Name]]; volume != nil { + volume.MountPaths = append(volume.MountPaths, mount.MountPath) + } + } + } + var claims corev1.PersistentVolumeClaimList + if err := reader.List(ctx, &claims, client.InNamespace(gs.Namespace)); err != nil { + return err + } + for _, claim := range claims.Items { + volume := volumes[claim.Name] + if volume == nil && metav1.IsControlledBy(&claim, gs) { + volume = &gameplanev1alpha1.BackupVolumeCoverage{ClaimName: claim.Name, Classification: "unclassified", CoverageReason: "RetainedUnmountedClaim"} + volumes[claim.Name] = volume + } + if volume != nil { + volume.ClaimUID = string(claim.UID) + } + } + for _, extra := range effectiveExtraVolumes(gs, tmpl) { + volume := volumes[extraPVCName(gs, extra.Name)] + if volume == nil { + continue + } + // A newer declaration for a different mount must not classify the + // frozen workload's volume as safely rebuildable. + for _, mount := range volume.MountPaths { + if mount == extra.MountPath && extra.BackupRole != "" { + volume.Classification = extra.BackupRole + } + } + } + now := metav1.Now() + manifest := &gameplanev1alpha1.BackupCaptureManifest{State: "Recorded", RecordedAt: &now, + ServerUID: string(gs.UID), Strategy: b.Spec.Strategy, LayoutSource: source, QuiesceOutcome: backupQuiesceOutcome(b)} + if manifest.Strategy == "" { + manifest.Strategy = "restic-snapshot" + } + for _, volume := range volumes { + sort.Strings(volume.MountPaths) + manifest.Volumes = append(manifest.Volumes, *volume) + } + sort.Slice(manifest.Volumes, func(i, j int) bool { return manifest.Volumes[i].ClaimName < manifest.Volumes[j].ClaimName }) + b.Status.CaptureManifest = manifest + return r.Status().Update(ctx, b) +} + +func backupQuiesceOutcome(b *gameplanev1alpha1.Backup) string { + if preparation, err := parseBackupPreparation(b); err == nil && preparation != nil && preparation.Outcome == "already-stopped" { + return "already-stopped" + } + switch b.Annotations[annoQuiesceAttempted] { + case "true": + return "confirmed" + case "unsupported": + return "unsupported" + case "pending": + return "unknown" + default: + if !b.Spec.Quiesce { + return "not-requested" + } + return "agent-unavailable" + } +} + +// Consistency=True means the copy completed after confirmed quiesce or from an +// already-stopped source whose writers had drained before capture admission. +// False is an explicit weaker guarantee, not a synonym for copy failure. +func backupConsistencyCondition(b *gameplanev1alpha1.Backup) metav1.Condition { + condition := metav1.Condition{Type: "Consistency", ObservedGeneration: b.Generation, Status: metav1.ConditionUnknown} + switch { + case b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed: + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "CaptureFailed", "The capture failed; no usable consistency guarantee was achieved." + case b.Status.CaptureManifest == nil || b.Status.CaptureManifest.State != "Recorded": + condition.Reason, condition.Message = "LegacyMetadataUnknown", "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template." + case b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || b.Status.SnapshotID == "": + condition.Reason, condition.Message = "CapturePending", "The capture has not completed; its achieved consistency is not yet known." + case b.Status.CaptureManifest.QuiesceOutcome == "confirmed": + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "Quiesced", "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup." + case b.Status.CaptureManifest.QuiesceOutcome == "already-stopped": + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "StoppedSource", "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup." + case b.Status.CaptureManifest.Strategy == "volume-snapshot": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "CrashConsistentSnapshot", "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed ("+b.Status.CaptureManifest.QuiesceOutcome+")." + case b.Status.CaptureManifest.QuiesceOutcome == "unsupported": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "UnsupportedRawCopy", "The game does not support quiesce. The primary volume was copied while it could still change." + case b.Status.CaptureManifest.QuiesceOutcome == "agent-unavailable": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "AgentUnavailableRawCopy", "No agent quiesce capability was configured. The primary volume was copied without pausing game writes." + default: + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "RawCopy", "The primary volume was copied without confirmed application quiesce." + } + return condition +} + +func (r *BackupReconciler) syncBackupConsistency(ctx context.Context, b *gameplanev1alpha1.Backup) error { + conditions := upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) + if sameConditions(b.Status.Conditions, conditions) { + return nil + } + b.Status.Conditions = conditions + return r.Status().Update(ctx, b) +} diff --git a/operator/internal/controller/backup_manifest_test.go b/operator/internal/controller/backup_manifest_test.go new file mode 100644 index 000000000..89b6a00fd --- /dev/null +++ b/operator/internal/controller/backup_manifest_test.go @@ -0,0 +1,184 @@ +package controller + +import ( + "reflect" + "testing" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestBackupManifestPinsCoverageAndClassification(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + _ = batchv1.AddToScheme(scheme) + _ = gameplanev1alpha1.AddToScheme(scheme) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "game-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}} + tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template"}, Spec: gameplanev1alpha1.GameTemplateSpec{Storage: gameplanev1alpha1.GameStorageSpec{ + Extra: []gameplanev1alpha1.ExtraVolumeSpec{{Name: "install", MountPath: "/serverfiles", BackupRole: "rebuildable"}}, + }}} + b := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "copy", Namespace: "games", UID: "backup-uid", Annotations: map[string]string{annoQuiesceAttempted: "unsupported"}}, Spec: gameplanev1alpha1.BackupSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: gs.Name}, Quiesce: true}} + owner := *metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer")) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "game", VolumeMounts: []corev1.VolumeMount{{Name: "data", MountPath: "/world"}, {Name: "extra-install", MountPath: "/serverfiles"}, {Name: "mods", MountPath: "/world/mods"}}}}, + Volumes: []corev1.Volume{ + {Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-data"}}}, + {Name: "extra-install", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-extra-install"}}}, + {Name: "mods", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-mods-active"}}}, + }, + }}}, + } + objects := []client.Object{gs, tmpl, b, ss} + for _, name := range []string{"game-data", "game-extra-install", "game-mods-active", "game-mods-old"} { + objects = append(objects, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: gs.Namespace, UID: types.UID("uid-" + name), OwnerReferences: []metav1.OwnerReference{owner}}}) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(b).WithObjects(objects...).Build() + r := &BackupReconciler{Client: c, APIReader: c, Scheme: scheme} + if err := r.recordBackupManifest(t.Context(), b, gs); err != nil { + t.Fatal(err) + } + manifest := b.Status.CaptureManifest + if manifest == nil || manifest.State != "Recorded" || manifest.QuiesceOutcome != "unsupported" || len(manifest.Volumes) != 4 { + t.Fatalf("incomplete manifest: %+v", manifest) + } + byName := map[string]gameplanev1alpha1.BackupVolumeCoverage{} + for _, volume := range manifest.Volumes { + byName[volume.ClaimName] = volume + } + if !byName["game-data"].Included || byName["game-data"].ClaimUID != "uid-game-data" || !reflect.DeepEqual(byName["game-data"].MountPaths, []string{"/world"}) { + t.Fatalf("primary coverage: %+v", byName["game-data"]) + } + if byName["game-extra-install"].Included || byName["game-extra-install"].Classification != "rebuildable" || byName["game-mods-active"].Included || byName["game-mods-old"].Included { + t.Fatalf("extra or nested mod volume incorrectly covered: %+v", manifest.Volumes) + } + before := manifest.DeepCopy() + tmpl.Spec.Storage.Extra[0].BackupRole = "data" + if err := c.Update(t.Context(), tmpl); err != nil { + t.Fatal(err) + } + if err := r.recordBackupManifest(t.Context(), b, gs); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, b.Status.CaptureManifest) { + t.Fatal("template change rewrote capture manifest") + } +} + +func TestBackupManifestUsesEffectiveTemplateClassification(t *testing.T) { + r, gs, installed := revisionFixture(t) + installed.Spec.Storage.Extra = []gameplanev1alpha1.ExtraVolumeSpec{{Name: "world", MountPath: "/world-extra", BackupRole: "data"}} + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + pin, err := r.saveTemplateSnapshot(t.Context(), gs, installed) + if err != nil { + t.Fatal(err) + } + gs.Annotations[effectiveTemplateAnnotation] = pin + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + installed.Spec.Storage.Extra[0].BackupRole = "rebuildable" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + b := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "capture", Namespace: gs.Namespace}} + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "game", VolumeMounts: []corev1.VolumeMount{{Name: "world", MountPath: "/world-extra"}}}}, + Volumes: []corev1.Volume{{Name: "world", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-extra-world"}}}}, + }}}} + for _, obj := range []client.Object{b, ss} { + if err := r.Create(t.Context(), obj); err != nil { + t.Fatal(err) + } + } + backups := &BackupReconciler{Client: r.Client, APIReader: r.APIReader, Scheme: r.Scheme} + if err := backups.recordBackupManifest(t.Context(), b, gs); err != nil { + t.Fatal(err) + } + for _, volume := range b.Status.CaptureManifest.Volumes { + if volume.ClaimName == "game-extra-world" && volume.Classification == "data" { + return + } + } + t.Fatal("pending catalog revision changed effective volume classification") +} + +func TestBackupManifestDoesNotInventHistoryForExistingWorker(t *testing.T) { + for _, strategy := range []string{"restic-snapshot", "volume-snapshot"} { + t.Run(strategy, func(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = batchv1.AddToScheme(scheme) + _ = snapshotv1.AddToScheme(scheme) + _ = gameplanev1alpha1.AddToScheme(scheme) + b := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "legacy", Namespace: "games"}, Spec: gameplanev1alpha1.BackupSpec{Strategy: strategy}} + var worker client.Object = &batchv1.Job{ObjectMeta: b.ObjectMeta} + if strategy == "volume-snapshot" { + worker = &snapshotv1.VolumeSnapshot{ObjectMeta: b.ObjectMeta} + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(b).WithObjects(b, worker).Build() + r := &BackupReconciler{Client: c, APIReader: c, Scheme: scheme} + if err := r.recordBackupManifest(t.Context(), b, &gameplanev1alpha1.GameServer{}); err != nil { + t.Fatal(err) + } + if b.Status.CaptureManifest.State != "LegacyUnknown" || len(b.Status.CaptureManifest.Volumes) != 0 { + t.Fatalf("invented history: %+v", b.Status.CaptureManifest) + } + }) + } +} + +func TestBackupConsistencyDescribesAchievedResult(t *testing.T) { + for _, tc := range []struct { + strategy, outcome, reason string + status metav1.ConditionStatus + }{ + {"restic-snapshot", "confirmed", "Quiesced", metav1.ConditionTrue}, + {"restic-snapshot", "already-stopped", "StoppedSource", metav1.ConditionTrue}, + {"restic-snapshot", "unsupported", "UnsupportedRawCopy", metav1.ConditionFalse}, + {"restic-snapshot", "not-requested", "RawCopy", metav1.ConditionFalse}, + {"restic-snapshot", "agent-unavailable", "AgentUnavailableRawCopy", metav1.ConditionFalse}, + {"volume-snapshot", "unsupported", "CrashConsistentSnapshot", metav1.ConditionFalse}, + {"volume-snapshot", "confirmed", "Quiesced", metav1.ConditionTrue}, + {"volume-snapshot", "already-stopped", "StoppedSource", metav1.ConditionTrue}, + } { + t.Run(tc.strategy+"/"+tc.outcome, func(t *testing.T) { + b := &gameplanev1alpha1.Backup{Status: gameplanev1alpha1.BackupStatus{Phase: gameplanev1alpha1.BackupPhaseSucceeded, SnapshotID: "snapshot-id", + CaptureManifest: &gameplanev1alpha1.BackupCaptureManifest{State: "Recorded", Strategy: tc.strategy, QuiesceOutcome: tc.outcome}}} + condition := backupConsistencyCondition(b) + if condition.Reason != tc.reason || condition.Status != tc.status { + t.Fatalf("got %+v", condition) + } + b.Status.SnapshotID = "" + if condition := backupConsistencyCondition(b); condition.Reason != "CapturePending" || condition.Status != metav1.ConditionUnknown { + t.Fatalf("copy without restorable identity claims achieved consistency: %+v", condition) + } + b.Status.SnapshotID = "snapshot-id" + b.Status.Phase = gameplanev1alpha1.BackupPhaseRunning + if condition := backupConsistencyCondition(b); condition.Status != metav1.ConditionUnknown { + t.Fatalf("unfinished copy claims achieved consistency: %+v", condition) + } + b.Status.Phase = gameplanev1alpha1.BackupPhaseFailed + if condition := backupConsistencyCondition(b); condition.Reason != "CaptureFailed" || condition.Status != metav1.ConditionFalse { + t.Fatalf("failed copy claims usable result: %+v", condition) + } + }) + } + b := &gameplanev1alpha1.Backup{Status: gameplanev1alpha1.BackupStatus{Phase: gameplanev1alpha1.BackupPhaseSucceeded}} + if condition := backupConsistencyCondition(b); condition.Status != metav1.ConditionUnknown || condition.Reason != "LegacyMetadataUnknown" { + t.Fatalf("legacy backup guarantee invented: %+v", condition) + } +} diff --git a/operator/internal/controller/backup_preparation.go b/operator/internal/controller/backup_preparation.go new file mode 100644 index 000000000..4f682ab53 --- /dev/null +++ b/operator/internal/controller/backup_preparation.go @@ -0,0 +1,234 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + backupPreparationAnnotation = "backup.gameplane.local/preparation" + backupCleanupOutcomeAnnotation = "backup.gameplane.local/cleanup-outcome" +) + +var errBackupSourceChanged = errors.New("backup source changed after preparation") + +type podAgentQuiescer interface { + CheckPod(context.Context, string, string, string) error + QuiescePod(context.Context, string, string, string) error + UnquiescePod(context.Context, string, string, string) error +} + +func agentQuiesceAvailable(q AgentQuiescer) bool { + if q == nil { + return false + } + if configured, ok := q.(interface{ QuiesceEnabled() bool }); ok { + return configured.QuiesceEnabled() + } + return true +} + +func callBackupQuiesce(ctx context.Context, q AgentQuiescer, namespace, server string, podUID types.UID, undo bool) error { + if !agentQuiesceAvailable(q) { + return fmt.Errorf("agent quiesce client is not configured") + } + if podUID != "" { + bound, ok := q.(podAgentQuiescer) + if !ok { + return fmt.Errorf("agent client does not support pod-bound quiesce") + } + if undo { + return bound.UnquiescePod(ctx, namespace, server, string(podUID)) + } + return bound.QuiescePod(ctx, namespace, server, string(podUID)) + } + if undo { + return q.Unquiesce(ctx, namespace, server) + } + return q.Quiesce(ctx, namespace, server) +} + +// Source identity is immutable for one copy. RPC outcome remains in the +// quiesce annotations, so a stopped source is never described as quiesced. +type backupPreparation struct { + Outcome string `json:"outcome"` + TargetUID types.UID `json:"targetUID"` + WorkloadUID types.UID `json:"workloadUID"` + WorkloadGeneration int64 `json:"workloadGeneration"` + PodUID types.UID `json:"podUID,omitempty"` +} + +func parseBackupPreparation(b *gameplanev1alpha1.Backup) (*backupPreparation, error) { + value := b.Annotations[backupPreparationAnnotation] + if value == "" { + return nil, nil + } + var p backupPreparation + if err := json.Unmarshal([]byte(value), &p); err != nil { + return nil, fmt.Errorf("decode backup preparation: %w", err) + } + if p.TargetUID == "" || p.WorkloadUID == "" || (p.Outcome != "live" && p.Outcome != "already-stopped") || (p.Outcome == "live" && p.PodUID == "") || (p.Outcome == "already-stopped" && p.PodUID != "") { + return nil, fmt.Errorf("invalid backup preparation identity") + } + return &p, nil +} + +// Call only after the shared owner and StatefulSet fence are held. Readiness +// and desired suspend/idle state are not evidence that writers have exited. +func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer) (bool, error) { + if !b.Spec.Quiesce { + return true, nil + } + p, err := parseBackupPreparation(b) + if err != nil { + return false, err + } + if p == nil && !agentQuiesceAvailable(r.AgentClient) { + return true, nil + } + // Pre-upgrade attempts have no source identity. Do not invent a new one + // after their RPC; preserve their existing cleanup contract. + if p == nil && b.Annotations[annoQuiesceAttempted] != "" { + return true, nil + } + var ss appsv1.StatefulSet + if err := r.backupAPIReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + if apierrors.IsNotFound(err) && p != nil { + return false, fmt.Errorf("%w: workload absent", errBackupSourceChanged) + } + return false, client.IgnoreNotFound(err) + } + if p != nil && (p.TargetUID != gs.UID || p.WorkloadUID != ss.UID || p.WorkloadGeneration != ss.Generation) { + return false, fmt.Errorf("%w: workload identity", errBackupSourceChanged) + } + if !metav1.IsControlledBy(&ss, gs) { + return false, fmt.Errorf("backup source workload is not owned by the bound GameServer") + } + if ss.Spec.Replicas == nil || ss.Status.ObservedGeneration < ss.Generation { + return false, nil + } + if p != nil && ((p.Outcome == "already-stopped") != (*ss.Spec.Replicas == 0)) { + return false, fmt.Errorf("%w: desired replicas", errBackupSourceChanged) + } + var candidate backupPreparation + candidate.TargetUID, candidate.WorkloadUID, candidate.WorkloadGeneration = gs.UID, ss.UID, ss.Generation + if *ss.Spec.Replicas == 0 { + if ss.Status.Replicas != 0 { + if p != nil { + return false, fmt.Errorf("%w: stopped workload has replicas", errBackupSourceChanged) + } + return false, nil + } + drained, err := r.backupSourceDrained(ctx, b, gs, &ss) + if err != nil || !drained { + if err == nil && p != nil { + return false, fmt.Errorf("%w: stopped source acquired a writer", errBackupSourceChanged) + } + return false, err + } + candidate.Outcome = "already-stopped" + } else { + var pod corev1.Pod + err := r.backupAPIReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if apierrors.IsNotFound(err) { + if p != nil { + return false, fmt.Errorf("%w: original pod absent", errBackupSourceChanged) + } + return false, nil + } + if err != nil { + return false, err + } + if p != nil && (p.PodUID != pod.UID || !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning) { + return false, fmt.Errorf("%w: original process no longer running", errBackupSourceChanged) + } + if !metav1.IsControlledBy(&pod, &ss) || pod.UID == "" { + return false, fmt.Errorf("backup source pod is not owned by the bound workload") + } + if !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning { + return false, nil + } + candidate.Outcome, candidate.PodUID = "live", pod.UID + } + if p != nil { + if p.Outcome != candidate.Outcome || p.PodUID != candidate.PodUID { + return false, fmt.Errorf("%w: process identity", errBackupSourceChanged) + } + return true, nil + } + encoded, err := json.Marshal(candidate) + if err != nil { + return false, err + } + patchBackupAnnotations(b, map[string]string{backupPreparationAnnotation: string(encoded)}) + return true, r.Update(ctx, b) +} + +// Current-copy workers are readers and are already protected by the owner; +// ignore only those, so requeues can observe their completion. Any other PVC +// user or unscheduled writer Job prevents stopped-source admission. +func (r *BackupReconciler) backupSourceDrained(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer, ss *appsv1.StatefulSet) (bool, error) { + var jobs batchv1.JobList + if err := r.backupAPIReader().List(ctx, &jobs, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + copyJobs := map[types.UID]bool{} + for i := range jobs.Items { + job := &jobs.Items[i] + if job.Name == b.Name && metav1.IsControlledBy(job, b) { + copyJobs[job.UID] = true + continue + } + if ((job.Status.Succeeded == 0 && !jobPermanentlyFailed(job)) || job.Status.Active > 0 || !job.DeletionTimestamp.IsZero()) && podMountsRestoreTarget(&corev1.Pod{Spec: job.Spec.Template.Spec}, gs.Name) { + return false, nil + } + } + var pods corev1.PodList + if err := r.backupAPIReader().List(ctx, &pods, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + for i := range pods.Items { + pod := &pods.Items[i] + if !liveRestorePod(pod) { + continue + } + owner := metav1.GetControllerOf(pod) + if owner != nil && owner.Kind == "Job" && copyJobs[owner.UID] { + continue + } + if metav1.IsControlledBy(pod, ss) || metav1.IsControlledBy(pod, gs) || pod.Name == gs.Name+"-0" || podMountsRestoreTarget(pod, gs.Name) { + return false, nil + } + } + return true, nil +} + +// An absent/replaced original pod cannot retain save-off state. A timeout, +// unready pod, or terminating pod with the same UID is never proof of exit. +// Empty UID is legacy state: only absence or an observed terminal pod suffices. +func backupQuiesceProcessGone(ctx context.Context, reader client.Reader, namespace, server string, uid types.UID) (bool, error) { + var pod corev1.Pod + err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: server + "-0"}, &pod) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + if uid != "" && pod.UID != uid { + return true, nil + } + return pod.DeletionTimestamp.IsZero() && (pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed), nil +} diff --git a/operator/internal/controller/backup_stopped_test.go b/operator/internal/controller/backup_stopped_test.go new file mode 100644 index 000000000..eb07407b8 --- /dev/null +++ b/operator/internal/controller/backup_stopped_test.go @@ -0,0 +1,389 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + "github.com/GameplanePanel/gameplane/operator/internal/agent" +) + +func stoppedBackupFixture(t *testing.T, suspended bool) (*BackupReconciler, *guardQuiescer) { + t.Helper() + r, q := guardFixture(t, guardBackup("first"), guardBackup("second")) + ctx := t.Context() + var gs gameplanev1alpha1.GameServer + key := client.ObjectKey{Namespace: "ns", Name: "server"} + if err := r.Get(ctx, key, &gs); err != nil { + t.Fatal(err) + } + gs.Spec.Suspend = suspended + if !suspended { + gs.Annotations = map[string]string{IdleAsleepSinceAnnotation: "2026-10-21T00:00:00Z"} + } + if err := r.Update(ctx, &gs); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(ctx, key, &ss); err != nil { + t.Fatal(err) + } + zero := int32(0) + ss.Spec.Replicas = &zero + if err := r.Update(ctx, &ss); err != nil { + t.Fatal(err) + } + if err := r.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + return r, q +} + +// A regression to unconditional agent quiesce would fail the first admission; +// releasing before worker drain would let the second Job appear prematurely. +func TestBackupStoppedCopiesWithoutWakingAndReleasesForNextCopy(t *testing.T) { + for _, suspended := range []bool{false, true} { + t.Run(map[bool]string{false: "idle", true: "suspended"}[suspended], func(t *testing.T) { + r, q := stoppedBackupFixture(t, suspended) + q.quiesceErr = errors.New("absent agent") + guardPass(t, r, "first") + first := guardGetBackup(t, r, "first") + var prep backupPreparation + if err := json.Unmarshal([]byte(first.Annotations[backupPreparationAnnotation]), &prep); err != nil { + t.Fatal(err) + } + if prep.Outcome != "already-stopped" || prep.TargetUID != "server-uid" || prep.WorkloadUID != "workload-uid" || prep.PodUID != "" { + t.Fatalf("wrong source proof: %+v", prep) + } + if first.Status.CaptureManifest == nil || first.Status.CaptureManifest.QuiesceOutcome != "already-stopped" { + t.Fatalf("stopped source was not recorded at admission: %+v", first.Status.CaptureManifest) + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatal(err) + } + // A restart and concurrent wake request must preserve the zero-replica fence. + r = &BackupReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme, AgentClient: q, LogReader: r.LogReader} + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[IdleWakeRequestedAnnotation] = "wake" + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if err := gr.reconcileStatefulSet(t.Context(), &gs, singlePortTemplate(true, "minecraft"), nil, nil, 1); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 { + t.Fatal("wake crossed the backup fence") + } + guardPass(t, r, "second") + var second batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &second); !apierrors.IsNotFound(err) { + t.Fatalf("second copy admitted before drain: %v", err) + } + job.Status.Succeeded = 1 + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + guardPass(t, r, "first") + if got := guardGetBackup(t, r, "first"); got.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || got.Status.SnapshotID == "" { + t.Fatalf("not restorable: %+v", got.Status) + } + first = guardGetBackup(t, r, "first") + if condition := backupConsistencyCondition(first); condition.Status != metav1.ConditionTrue || condition.Reason != "StoppedSource" { + t.Fatalf("stopped copy claims wrong consistency: %+v", condition) + } + guardPass(t, r, "second") + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &second); err != nil { + t.Fatal(err) + } + if q.quiesced != 0 || q.unquiesced != 0 { + t.Fatalf("stopped copy contacted absent agent: %+v", q) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + if gs.Spec.Suspend != suspended || gs.Annotations[IdleWakeRequestedAnnotation] != "wake" || (!suspended && gs.Annotations[IdleAsleepSinceAnnotation] != "2026-10-21T00:00:00Z") { + t.Fatal("copy changed power intent") + } + }) + } +} + +func TestBackupStoppedRequiresLiveDrainProof(t *testing.T) { + for _, tc := range []string{"desired-one", "observed-replicas", "stale-generation", "terminating-game", "foreign-writer", "pending-writer-job", "read-error"} { + t.Run(tc, func(t *testing.T) { + r, q := stoppedBackupFixture(t, true) + base := r.Client + r.APIReader = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + if ss, ok := obj.(*appsv1.StatefulSet); ok { + switch tc { + case "desired-one": + one := int32(1) + ss.Spec.Replicas = &one + case "observed-replicas": + ss.Status.Replicas = 1 + case "stale-generation": + ss.Generation = 2 + ss.Status.ObservedGeneration = 1 + } + } + return nil + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := c.List(ctx, list, opts...); err != nil { + return err + } + if pods, ok := list.(*corev1.PodList); ok { + if tc == "read-error" { + return errors.New("live list failed") + } + if tc == "terminating-game" { + now := metav1.Now() + pods.Items = append(pods.Items, corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", DeletionTimestamp: &now}, Status: corev1.PodStatus{Phase: corev1.PodSucceeded}}) + } + if tc == "foreign-writer" { + pods.Items = append(pods.Items, corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "writer", Namespace: "ns"}, Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "server-data"}}}}}}) + } + } + if jobs, ok := list.(*batchv1.JobList); ok && tc == "pending-writer-job" { + jobs.Items = append(jobs.Items, batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "writer", Namespace: "ns"}, Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "server-data"}}}}}}}}) + } + return nil + }, + }) + _, _ = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "first"}}) + var job batchv1.Job + if err := base.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatalf("copy admitted without drain proof: %v", err) + } + if q.quiesced != 0 { + t.Fatal("no proven live process to quiesce") + } + }) + } +} + +func TestBackupLiveTimeoutRetainsOwnerUntilOriginalPodGone(t *testing.T) { + r, q := guardFixture(t, guardBackup("first"), guardBackup("second")) + q.quiesceErr = errors.New("live agent unreachable") + q.unquiesceErr = errors.New("live agent still unreachable") + _, _ = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "first"}}) + first := guardGetBackup(t, r, "first") + if first.Status.Phase != gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("timeout did not fail closed: %+v", first.Status) + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatal("live timeout created copy") + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + op, _, err := lifecycleOwner(&gs) + if err != nil || op.QuiescePodUID != "game-pod-uid" || !op.NeedsUnquiesce { + t.Fatalf("cleanup identity not durable: %+v %v", op, err) + } + guardPass(t, r, "first") + guardPass(t, r, "second") + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &job); !apierrors.IsNotFound(err) { + t.Fatal("timeout released the live process owner") + } + cleanupAttempts := q.unquiesced + // The replacement must not receive its predecessor's delayed save-on. + if err := r.Delete(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + if err := r.Create(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "replacement"}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}); err != nil { + t.Fatal(err) + } + guardPass(t, r, "first") + if q.unquiesced != cleanupAttempts { + t.Fatal("replacement inherited original process cleanup") + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" { + t.Fatal("absent original process kept owner forever") + } +} + +func TestBackupOldAgentFailsBeforeCleanupIntent(t *testing.T) { + r, q := guardFixture(t, guardBackup("first")) + q.checkErr = errors.New("identity endpoint: status 404") + _, _ = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "first"}}) + b := guardGetBackup(t, r, "first") + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Annotations[annoQuiesceAttempted] != "" || q.quiesced != 0 { + t.Fatalf("unsupported identity created cleanup debt: backup=%+v agent=%+v", b, q) + } + guardPass(t, r, "first") + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" { + t.Fatal("old agent failure blocked the workload upgrade") + } +} + +func TestBackupDisabledAgentPreservesRawCopyWithoutClaimingQuiesce(t *testing.T) { + r, _ := guardFixture(t, guardBackup("first")) + r.AgentClient = &agent.Client{Disabled: true} + guardPass(t, r, "first") + b := guardGetBackup(t, r, "first") + if b.Annotations[backupPreparationAnnotation] != "" || b.Annotations[annoQuiesceAttempted] != "" { + t.Fatal("disabled agent claimed preparation or quiesce") + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatalf("disabled client prevented raw copy: %v", err) + } + if err := callBackupQuiesce(t.Context(), r.AgentClient, "ns", "server", "", true); err == nil { + t.Fatal("disabled client cleared an existing cleanup obligation") + } +} + +func TestBackupDisabledAgentCannotBypassPendingLiveQuiesce(t *testing.T) { + r, _ := guardFixture(t, guardBackup("first")) + b := guardGetBackup(t, r, "first") + if ok, err := r.claimBackupTarget(t.Context(), b); err != nil || !ok { + t.Fatalf("claim: %t %v", ok, err) + } + gs, err := r.ownedBackupTarget(t.Context(), b) + if err != nil { + t.Fatal(err) + } + if ready, err := r.prepareBackupSource(t.Context(), b, gs); err != nil || !ready { + t.Fatalf("prepare: %t %v", ready, err) + } + patchBackupAnnotations(b, map[string]string{annoQuiesceAttempted: "pending"}) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + r.AgentClient = &agent.Client{Disabled: true} + if err := r.maybeQuiesce(t.Context(), b); err == nil || b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("disabled client bypassed pending quiesce: %v %+v", err, b.Status) + } +} + +func TestBackupSourceReplacementFailsBeforeCopyOrSuccess(t *testing.T) { + for _, duringRPC := range []bool{true, false} { + t.Run(map[bool]string{true: "during-quiesce", false: "during-copy"}[duringRPC], func(t *testing.T) { + r, q := guardFixture(t, guardBackup("first")) + replace := func() { + var old corev1.Pod + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server-0"}, &old); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), &old); err != nil { + t.Fatal(err) + } + old.UID, old.ResourceVersion = "replacement", "" + if err := r.Create(t.Context(), &old); err != nil { + t.Fatal(err) + } + } + if duringRPC { + q.afterQuiesce = replace + } else { + guardPass(t, r, "first") + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatal(err) + } + job.Status.Succeeded = 1 + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + replace() + } + guardPass(t, r, "first") + if got := guardGetBackup(t, r, "first"); got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || got.Status.SnapshotID != "" { + t.Fatalf("replacement allowed a restorable copy: %+v", got.Status) + } + if duringRPC { + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatal("replacement during RPC created a copy") + } + } + if q.unquiesced != 0 { + t.Fatal("replacement received the old process cleanup") + } + }) + } +} + +func TestBackupCleanupRequiresOriginalProcessExit(t *testing.T) { + for _, state := range []string{"absent", "replacement", "terminal", "terminating", "unready", "legacy-running", "read-error"} { + t.Run(state, func(t *testing.T) { + r, _ := guardFixture(t) + uid := types.UID("game-pod-uid") + var pod corev1.Pod + key := client.ObjectKey{Namespace: "ns", Name: "server-0"} + if err := r.Get(t.Context(), key, &pod); err != nil { + t.Fatal(err) + } + reader := interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if state == "read-error" { + return errors.New("apiserver unavailable") + } + if state == "absent" { + return apierrors.NewNotFound(corev1.Resource("pods"), key.Name) + } + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + p := obj.(*corev1.Pod) + switch state { + case "replacement": + p.UID = "successor" + case "terminal": + p.Status.Phase = corev1.PodFailed + case "terminating": + now := metav1.Now() + p.DeletionTimestamp = &now + case "unready": + p.Status.Phase = corev1.PodPending + } + return nil + }}) + if state == "legacy-running" { + uid = "" + } + gone, err := backupQuiesceProcessGone(t.Context(), reader, "ns", "server", uid) + wantGone := state == "absent" || state == "replacement" || state == "terminal" + if gone != wantGone || (err != nil) != (state == "read-error") { + t.Fatalf("exit proof = %t, %v; want gone %t", gone, err, wantGone) + } + }) + } +} diff --git a/operator/internal/controller/backup_volumesnapshot.go b/operator/internal/controller/backup_volumesnapshot.go index b454f6fad..e90a0b6c9 100644 --- a/operator/internal/controller/backup_volumesnapshot.go +++ b/operator/internal/controller/backup_volumesnapshot.go @@ -38,6 +38,9 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( if err := r.backupAPIReader().Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: b.Name}, &existing); err != nil && !apierrors.IsNotFound(err) { return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot API unavailable: %v", err)) } + if legacyBackupOwesUnquiesce(b) && metav1.IsControlledBy(&existing, b) { + return r.observeVolumeSnapshot(ctx, b, &existing) + } // Quiesce first (idempotent across requeues). A CSI snapshot is only // crash-consistent; flushing the game to disk first (RCON save-all, // etc.) makes it application-consistent. No-op when the backup opts out @@ -69,6 +72,10 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot creation failed: %v", err)) } + return r.observeVolumeSnapshot(ctx, b, vs) +} + +func (r *BackupReconciler) observeVolumeSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup, vs *snapshotv1.VolumeSnapshot) (ctrl.Result, error) { switch { case vs.Status != nil && vs.Status.Error != nil && vs.Status.Error.Message != nil: // The CSI driver rejected the snapshot. Surface it and unquiesce so @@ -112,6 +119,7 @@ func (r *BackupReconciler) markVolumeSnapshotRunning( Reason: string(gameplanev1alpha1.BackupPhaseRunning), ObservedGeneration: b.Generation, }) + b.Status.Conditions = upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) if err := r.Status().Update(ctx, b); err != nil { return ctrl.Result{}, err } @@ -152,6 +160,7 @@ func (r *BackupReconciler) completeVolumeSnapshot( Reason: "Succeeded", ObservedGeneration: b.Generation, }) + b.Status.Conditions = upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) if err := r.Status().Update(ctx, b); err != nil { return ctrl.Result{}, err } diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 67e8be4e3..01b9ec971 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -68,6 +68,12 @@ type GameServerReconciler struct { // to info and existing StatefulSets don't roll on operator upgrade. AgentLogLevel string + // Auxiliary resources are complete install-level overrides. Nil uses the + // built-in requests; a non-nil empty object explicitly clears them. + AgentResources *corev1.ResourceRequirements + SentinelResources *corev1.ResourceRequirements + TunnelResources *corev1.ResourceRequirements + // ConfigInitImage is the image for the init container that copies // operator-rendered config files onto the data volume. Set from an // operator flag so air-gapped installs can point it at a private @@ -100,6 +106,8 @@ type GameServerReconciler struct { // during a soft stop. May be nil (or a disabled client) in dev clusters, // in which case the operator falls back to a timed scale-to-zero. AgentClient AgentStopper + // OperationQuiescer recovers orphaned backups before admitting a wipe. + OperationQuiescer AgentQuiescer // PodAttacher runs the module-declared stop sequence over a pod attach // to the game container's stdin, for consoleMode: pty games that @@ -151,6 +159,11 @@ type GameServerReconciler struct { // per-GameServer; when true, it can be toggled on/off per server. CaptureEnabled bool + // RestrictedWorkloads applies the opt-in PSA-compatible game profile. + RestrictedWorkloads bool + AgentTokenExpirationSeconds int64 + DiagnosticsNamespace string + // CaptureSidecarImage is the container image for the network capture sidecar // injected when capture is enabled on a GameServer. Set from the // --capture-sidecar-image operator flag so air-gapped installs can point it @@ -248,22 +261,35 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } else if changed { return ctrl.Result{Requeue: true}, nil } + // Exposure compatibility depends only on the server. Reject it before + // resolving/adopting a template, which may otherwise initiate a rollout. + if r.RestrictedWorkloads && gs.Spec.Networking.Expose == "Hostport" { + return ctrl.Result{}, r.setPhase(ctx, &gs, "restricted workload cannot use networking.expose=Hostport; use NodePort or LoadBalancer") + } - // Resolve the template this GameServer points at. Templates are - // cluster-scoped so no namespace is needed. - var tmpl gameplanev1alpha1.GameTemplate - if err := r.Get(ctx, types.NamespacedName{Name: gs.Spec.TemplateRef.Name}, &tmpl); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{RequeueAfter: 15 * time.Second}, r.setPhase(ctx, &gs, - fmt.Sprintf("GameTemplate %q not found", gs.Spec.TemplateRef.Name)) + // Snapshot resolution precedes config, images, init containers, storage, + // services and every other template-derived resource. + effective, changed, err := r.reconcileTemplateRevision(ctx, &gs) + if err != nil { + if apierrors.IsConflict(err) { + return requeueOnConflict(ctrl.Result{}, err) } - return ctrl.Result{}, err + return ctrl.Result{RequeueAfter: 15 * time.Second}, r.setPhase(ctx, &gs, err.Error()) } + if changed { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + tmpl := *effective // Resolve spec.config against the template's configSchema before // touching any children: invalid config must fail loudly (mirroring // the missing-template path) instead of materializing a pod that // silently ignores what the user asked for. + if r.RestrictedWorkloads { + if err := validateRestrictedTemplate(&tmpl); err != nil { + return ctrl.Result{}, r.setPhase(ctx, &gs, err.Error()) + } + } mc, err := materializeConfig(&gs, &tmpl) if err != nil { return ctrl.Result{}, r.setPhase(ctx, &gs, @@ -452,6 +478,9 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) // or nothing fires the transition until an unrelated event happens to // wake the reconciler. hints := []time.Duration{stopRequeue, idleRequeue} + if _, held, _ := lifecycleOwner(&gs); held { + hints = append(hints, 5*time.Second) + } if plan.wantSentinel { // Bounded backstop for planSentinel's wait on the sentinel or the // game pod. Both are owned (Owns(&appsv1.Deployment{}), @@ -482,6 +511,7 @@ func (r *GameServerReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&networkingv1.NetworkPolicy{}). Owns(&gameplanev1alpha1.BackupSchedule{}). Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). @@ -1194,6 +1224,53 @@ func (r *GameServerReconciler) desiredReplicas( idle idleState, inventory ...*wipeWorkerState, ) (int32, time.Duration, error) { + // Read ownership live before restart acknowledgment or any stop RPC. The + // stop timestamp uses a checked Update, so a claim racing that write wins + // either before the command or after the stop has become visible. + var live gameplanev1alpha1.GameServer + if err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &live); err != nil { + return 0, 0, err + } + if live.UID != gs.UID { + return 0, 0, fmt.Errorf("GameServer changed identity before lifecycle decision") + } + if op, held, err := lifecycleOwner(&live); err != nil { + return 0, 0, err + } else if held && op.Kind == "Rollout" && !op.Baseline && len(op.Recovery) == 0 && op.Token != live.Annotations[effectiveTemplateAnnotation] { + replicas, poll, err := r.softStop(ctx, gs, tmpl) + if poll == 0 { + poll = restartDrainPoll + } + return replicas, poll, err + } else if held && op.InitialStartup && len(op.Recovery) == 0 { + if live.Spec.Suspend { + return 0, 5 * time.Second, nil + } + return 1, 5 * time.Second, nil + } else if held && lifecycleRecoveryNeedsStop(op) { + return 0, 5 * time.Second, nil + } else if held && (op.Kind == "Backup" || op.Baseline || len(op.Recovery) != 0) { + var ss appsv1.StatefulSet + err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if apierrors.IsNotFound(err) { + return 0, 5 * time.Second, nil + } + if err != nil { + return 0, 0, err + } + if !metav1.IsControlledBy(&ss, gs) { + return 0, 0, fmt.Errorf("protected StatefulSet belongs to another GameServer") + } + if ss.Spec.Replicas == nil { + return 1, 5 * time.Second, nil + } + return *ss.Spec.Replicas, 5 * time.Second, nil + } else if held && (live.Annotations[restoreGuardAnnotation] != gs.Annotations[restoreGuardAnnotation] || live.Annotations[wipeGuardAnnotation] != gs.Annotations[wipeGuardAnnotation]) { + // This reconcile rendered config from an older GameServer. Use the + // live owner only for stopping; do not replace the caller's snapshot + // and mix a newly-read spec with already-materialized configuration. + return r.softStop(ctx, &live, tmpl) + } if gs.Annotations[wipeGuardAnnotation] != "" || pendingWipe(gs) { replicas, poll, err := r.softStop(ctx, gs, tmpl) if err == nil && r.settledWipeFailure(ctx, gs, wipeInventory(inventory)) { @@ -1453,9 +1530,44 @@ func (r *GameServerReconciler) reconcileStatefulSet( if live.UID != gs.UID { return fmt.Errorf("GameServer changed identity before StatefulSet mutation") } + if live.Annotations[effectiveTemplateAnnotation] != gs.Annotations[effectiveTemplateAnnotation] { + return fmt.Errorf("effective template changed before StatefulSet mutation") + } if owner := metav1.GetControllerOf(ss); owner != nil && owner.UID != live.UID { return fmt.Errorf("StatefulSet belongs to a different GameServer identity") } + op, held, err := lifecycleOwner(&live) + if err != nil { + return err + } + if held { + if ss.Annotations == nil { + ss.Annotations = map[string]string{} + } + ss.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if op.InitialStartup { + actualReplicas = replicas + delete(ss.Annotations, restoreGuardAnnotation) + } + if op.Kind == "Backup" || len(op.Recovery) != 0 { + if !ss.CreationTimestamp.IsZero() || ss.ResourceVersion != "" { + // Orphaned restores may leave a running initial workload. + // Stop it while preserving its complete pod template. Other + // protected operations also preserve the replica count. + if lifecycleRecoveryNeedsStop(op) { + zero := int32(0) + ss.Spec.Replicas = &zero + } + return nil + } + // No workload exists yet. Create it stopped; this gives backup + // admission a real object to fence against stale creates. + actualReplicas = 0 + replicas = 0 + } + } else { + delete(ss.Annotations, lifecycleOperationAnnotation) + } if live.Annotations[restoreGuardAnnotation] == "" { // Restore cleanup can race a callback that restamps the workload // before the checked GameServer release. That release guarantees @@ -1464,6 +1576,9 @@ func (r *GameServerReconciler) reconcileStatefulSet( delete(ss.Annotations, restoreGuardAnnotation) } for _, annotation := range []string{restoreGuardAnnotation, wipeGuardAnnotation} { + if annotation == restoreGuardAnnotation && held && op.InitialStartup { + continue + } if guard := live.Annotations[annotation]; guard != "" { actualReplicas = 0 if ss.Annotations == nil { @@ -1491,13 +1606,31 @@ func (r *GameServerReconciler) reconcileStatefulSet( } } } + preserveBaseline, err := r.preserveBaselinePodTemplate(ctx, &live, ss) + if err != nil { + return err + } ss.Spec.Replicas = &actualReplicas + if held && (op.Kind != "Rollout" || op.Token != live.Annotations[effectiveTemplateAnnotation] || len(op.Recovery) != 0) && (!ss.CreationTimestamp.IsZero() || ss.ResourceVersion != "") { + // Restore and wipe may stop the existing workload, but rollout + // waits until the operation and its worker cleanup have finished. + return nil + } + if preserveBaseline { + return nil + } ss.Spec.ServiceName = gs.Name ss.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} ss.Spec.Template.Labels = labels // Stamp (or clear) the config fingerprint without touching // annotations other actors may have set on the pod template. ann := ss.Spec.Template.Annotations + if revision := gs.Annotations[effectiveTemplateAnnotation]; revision != "" { + if ann == nil { + ann = map[string]string{} + } + ann[templateRevisionAnnotation] = revision + } if mc.hash != "" { if ann == nil { ann = map[string]string{} @@ -1509,7 +1642,7 @@ func (r *GameServerReconciler) reconcileStatefulSet( ss.Spec.Template.Annotations = ann ss.Spec.Template.Spec.Containers = []corev1.Container{ buildGameContainer(gs, tmpl, image, ver, mc), - buildAgentContainer(gs, tmpl, ver, r.AgentImage, r.AgentLogLevel, r.AgentImagePullPolicy), + buildAgentContainer(gs, tmpl, ver, r.AgentImage, r.AgentLogLevel, r.AgentImagePullPolicy, r.AgentResources), } volumes := []corev1.Volume{ { @@ -1569,7 +1702,7 @@ func (r *GameServerReconciler) reconcileStatefulSet( }, }, }) - ss.Spec.Template.Spec.InitContainers = []corev1.Container{buildConfigInitContainer(r.ConfigInitImage, tmpl)} + ss.Spec.Template.Spec.InitContainers = []corev1.Container{buildConfigInitContainer(r.ConfigInitImage, tmpl, r.AgentResources)} } else { ss.Spec.Template.Spec.InitContainers = nil } @@ -1596,6 +1729,15 @@ func (r *GameServerReconciler) reconcileStatefulSet( // Same unconditional-assign reasoning: removing template.spec.security // must clear a previously-set pod-level FSGroup, not leave it pinned. ss.Spec.Template.Spec.SecurityContext = gamePodSecurityContext(tmpl) + if r.RestrictedWorkloads { + if ss.Spec.Template.Annotations == nil { + ss.Spec.Template.Annotations = map[string]string{} + } + ss.Spec.Template.Annotations[workloadSecurityLabel] = "restricted" + } else { + delete(ss.Spec.Template.Annotations, workloadSecurityLabel) + } + applyWorkloadSecurity(&ss.Spec.Template.Spec, tmpl, r.RestrictedWorkloads, r.AgentTokenExpirationSeconds) // Default to the per-GameServer SA so the agent's heartbeat can // patch gameservers/status (see reconcileAgentRBAC); an explicit // spec.serviceAccountName still wins. @@ -1877,6 +2019,10 @@ func (r *GameServerReconciler) reconcileCapture(ctx context.Context, gs *gamepla return fmt.Errorf("get game pod for capture reconciliation: %w", podErr) } + if r.RestrictedWorkloads || pod.Annotations[workloadSecurityLabel] == "restricted" || (gs.Spec.Capture != nil && gs.Spec.Capture.NodeDiagnostic) { + gs.Status.Capture.Ready = r.DiagnosticsNamespace != "" && gs.Spec.Capture.NodeDiagnostic && pod.Status.Phase == corev1.PodRunning + return r.patchCaptureStatus(ctx, gs, base) + } if !hasCaptureEphemeralContainer(&pod) { image := r.CaptureSidecarImage pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, buildCaptureEphemeralContainer(image, string(gs.UID))) @@ -1983,22 +2129,31 @@ func configInitImageOrDefault(image string) string { // When the template sets security.fsGroup, config-init makes each file // and directory it copied group-writable so a non-root game sharing that // fsGroup can update them. -func buildConfigInitContainer(image string, tmpl *gameplanev1alpha1.GameTemplate) corev1.Container { +func buildConfigInitContainer(image string, tmpl *gameplanev1alpha1.GameTemplate, defaults ...*corev1.ResourceRequirements) corev1.Container { image = configInitImageOrDefault(image) mountPath := effectiveMountPath(tmpl) cpCmd := "cp -RL " + configFilesStagingPath + "/* '" + mountPath + "/'" + var security *corev1.SecurityContext - // If fsGroup is set, append a chmod step to make each file and directory - // config-init copied group-writable (entries that came from the staging tree; - // other files already on the volume keep their modes). + // Newly created files must inherit the declared shared group even on + // local-path volumes that do not implement kubelet fsGroup ownership. + // Only reconcile copied entries; unrelated server data keeps its modes. if tmpl.Spec.Security != nil && tmpl.Spec.Security.FSGroup != nil { - cpCmd += " && cd " + configFilesStagingPath + " && find ./* -follow \\( -type f -o -type d \\) | while IFS= read -r p; do chmod g+w '" + mountPath + "/'\"$p\"; done" + group := *tmpl.Spec.Security.FSGroup + security = &corev1.SecurityContext{RunAsGroup: ptrTo(group)} + // Restricted init runs as the game UID. It can overwrite an existing + // root-owned, shared-group-writable file, but cannot chmod that file. + // Inspect the current group/mode before mutating so repeat starts and + // profile upgrades do not require ownership of already-correct files. + cpCmd += fmt.Sprintf(" && cd %s && find ./* -follow \\( -type f -o -type d \\) | while IFS= read -r p; do dest='%s/'\"$p\"; group=$(stat -c %%g \"$dest\") || exit 1; [ \"$group\" = %d ] || chgrp %d \"$dest\" || exit 1; mode=$(stat -c %%a \"$dest\") || exit 1; [ $((0$mode & 0020)) -ne 0 ] || chmod g+w \"$dest\" || exit 1; done", configFilesStagingPath, mountPath, group, group) } return corev1.Container{ - Name: "config-init", - Image: image, - Command: []string{"/bin/sh", "-c"}, + Name: "config-init", + Image: image, + Resources: auxiliaryResources(optionalResources(defaults), "25m", "64Mi"), + Command: []string{"/bin/sh", "-c"}, + SecurityContext: security, // -L dereferences the kubelet's per-key symlinks; the * glob // skips the ..data/.. dot-entries of the Secret mount. Args: []string{cpCmd}, @@ -2242,14 +2397,17 @@ func gamePodSecurityContext(tmpl *gameplanev1alpha1.GameTemplate) *corev1.PodSec func buildAgentContainer( gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate, ver *gameplanev1alpha1.GameVersion, fallbackImage, logLevel, pullPolicy string, + defaults ...*corev1.ResourceRequirements, ) corev1.Container { image := fallbackImage - res := corev1.ResourceRequirements{} + res := auxiliaryResources(optionalResources(defaults), "25m", "64Mi") if tmpl.Spec.Agent != nil { if tmpl.Spec.Agent.Image != "" { image = tmpl.Spec.Agent.Image } - res = tmpl.Spec.Agent.Resources + if tmpl.Spec.Agent.Resources.Requests != nil || tmpl.Spec.Agent.Resources.Limits != nil || len(tmpl.Spec.Agent.Resources.Claims) != 0 { + res = *tmpl.Spec.Agent.Resources.DeepCopy() + } } mountPath := effectiveMountPath(tmpl) nonRoot := true @@ -2282,6 +2440,7 @@ func buildAgentContainer( env := []corev1.EnvVar{ {Name: "GAMEPLANE_SERVER_NAME", Value: gs.Name}, {Name: "GAMEPLANE_SERVER_UID", Value: string(gs.UID)}, + {Name: "GAMEPLANE_POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.uid"}}}, {Name: "GAMEPLANE_TEMPLATE", Value: tmpl.Name}, {Name: "GAMEPLANE_GAME", Value: tmpl.Spec.Game}, // Games without RCON (consoleMode pty/none) must not have the diff --git a/operator/internal/controller/gameserver_envtest_test.go b/operator/internal/controller/gameserver_envtest_test.go index 2b9265365..bbdb2e707 100644 --- a/operator/internal/controller/gameserver_envtest_test.go +++ b/operator/internal/controller/gameserver_envtest_test.go @@ -978,8 +978,15 @@ func TestGameServer_NoExtraVolumesNoCaptureComponent(t *testing.T) { types.NamespacedName{Namespace: ns, Name: "smp"}, &ss); err != nil { return false, "statefulset: " + err.Error() } - if len(ss.Spec.Template.Spec.Volumes) != 3 { - return false, fmt.Sprintf("pod volumes = %v, want exactly [data agent-tls captures]", sprintArgs(volumeNames(ss.Spec.Template.Spec.Volumes))) + if len(ss.Spec.Template.Spec.Volumes) != 4 { + return false, fmt.Sprintf("pod volumes = %v, want exactly [data agent-tls captures agent-api-token]", sprintArgs(volumeNames(ss.Spec.Template.Spec.Volumes))) + } + token := volumeByName(ss.Spec.Template.Spec.Volumes, agentTokenVolumeName) + if token == nil || token.Projected == nil || len(token.Projected.Sources) != 3 || token.Projected.Sources[0].ServiceAccountToken == nil { + return false, "agent credentials are not a rotating projection" + } + if ss.Spec.Template.Spec.AutomountServiceAccountToken == nil || *ss.Spec.Template.Spec.AutomountServiceAccountToken { + return false, "automatic ServiceAccount credentials are exposed" } captures := volumeByName(ss.Spec.Template.Spec.Volumes, "captures") if captures == nil { @@ -991,11 +998,18 @@ func TestGameServer_NoExtraVolumesNoCaptureComponent(t *testing.T) { if captures.EmptyDir.SizeLimit == nil || captures.EmptyDir.SizeLimit.IsZero() { return false, "captures emptyDir must carry a nonzero sizeLimit" } - for _, c := range ss.Spec.Template.Spec.Containers { + for _, c := range append(ss.Spec.Template.Spec.Containers, ss.Spec.Template.Spec.InitContainers...) { + tokenMounted := false for _, vm := range c.VolumeMounts { if vm.Name == "captures" { return false, "captures volume must not be mounted on container " + c.Name } + if vm.Name == agentTokenVolumeName { + tokenMounted = true + } + } + if tokenMounted != (c.Name == "agent") { + return false, "agent token mount boundary violated by " + c.Name } } if len(ss.Spec.Template.Spec.EphemeralContainers) != 0 { @@ -2031,9 +2045,8 @@ func TestGameServer_ConfigFilesMaterialize(t *testing.T) { return true, "" }) - // Dropping configFiles from the template must delete the Secret and - // strip the init container + volume. Template edits don't trigger a - // GameServer reconcile (no cross-watch), so poke the GameServer. + // Dropping configFiles from the template must adopt the new revision, + // delete the Secret, and strip the init container + volume. // Retry on conflict — reconcilers patch status concurrently, racing a // bare Get+Update's resourceVersion. if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -2056,14 +2069,19 @@ func TestGameServer_ConfigFilesMaterialize(t *testing.T) { t.Fatalf("update gameserver: %v", err) } eventually(t, func() (bool, string) { - // Re-poke each poll: the reconcile racing the template cache - // update must not strand the test. - poke := getGameServer(t, ns, "smp") - if poke.Annotations == nil { - poke.Annotations = map[string]string{} - } - poke.Annotations["test.gameplane.local/poke"] = time.Now().Format(time.RFC3339Nano) - _ = k8sClient.Update(context.Background(), poke) + // Envtest has no StatefulSet controller. Acknowledge only the + // actual stop request before the rollout may change its template. + var observed appsv1.StatefulSet + if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "smp"}, &observed); err != nil { + return false, "statefulset: " + err.Error() + } + if observed.Spec.Replicas != nil && *observed.Spec.Replicas == 0 && observed.Status.ObservedGeneration < observed.Generation { + observed.Status.ObservedGeneration = observed.Generation + observed.Status.Replicas, observed.Status.ReadyReplicas = 0, 0 + if err := k8sClient.Status().Update(context.Background(), &observed); err != nil { + return false, "observe stopped workload: " + err.Error() + } + } var sec corev1.Secret err := k8sClient.Get(context.Background(), diff --git a/operator/internal/controller/gameserver_files_test.go b/operator/internal/controller/gameserver_files_test.go index 815d7feea..4eb17f521 100644 --- a/operator/internal/controller/gameserver_files_test.go +++ b/operator/internal/controller/gameserver_files_test.go @@ -1,6 +1,10 @@ package controller import ( + "os" + "os/exec" + "path/filepath" + "runtime" "strings" "testing" @@ -95,12 +99,8 @@ func TestBuildConfigInitContainer_GroupWritableWithFSGroup(t *testing.T) { t.Errorf("args should not contain chmod -R, got %q", arg) } - // The whole command must be valid shell: the mount path is single-quoted - // and closed before the loop variable, so $p still expands. - want := "cp -RL " + configFilesStagingPath + "/* '/data/' && cd " + configFilesStagingPath + - " && find ./* -follow \\( -type f -o -type d \\) | while IFS= read -r p; do chmod g+w '/data/'\"$p\"; done" - if arg != want { - t.Errorf("config-init command =\n %q\nwant\n %q", arg, want) + if !strings.Contains(arg, "stat -c %g") || !strings.Contains(arg, "stat -c %a") { + t.Errorf("already-correct ownership and modes must be checked before mutation: %q", arg) } // Verify the command contains "find ./*" @@ -109,6 +109,65 @@ func TestBuildConfigInitContainer_GroupWritableWithFSGroup(t *testing.T) { } } +// The second start models files that are writable through fsGroup but cannot +// be chmodded by the init UID, as occurs on a root-to-Restricted transition. +func TestBuildConfigInitContainer_ReusesSharedFilesWithoutOwnership(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("config-init runs in Linux containers") + } + dir := t.TempDir() + staging, data := filepath.Join(dir, "staging"), filepath.Join(dir, "data") + for _, path := range []string{filepath.Join(staging, "cfg"), data} { + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + } + source := filepath.Join(staging, "cfg", "profile.txt") + if err := os.WriteFile(source, []byte("first"), 0o600); err != nil { + t.Fatal(err) + } + unrelated := filepath.Join(data, "world.dat") + if err := os.WriteFile(unrelated, []byte("world"), 0o600); err != nil { + t.Fatal(err) + } + tmpl := &gameplanev1alpha1.GameTemplate{} + tmpl.Spec.Security = &gameplanev1alpha1.GameSecuritySpec{FSGroup: ptrTo(int64(os.Getgid()))} + tmpl.Spec.Storage.MountPath = data + c := buildConfigInitContainer("", tmpl) + if c.SecurityContext == nil || c.SecurityContext.RunAsGroup == nil || *c.SecurityContext.RunAsGroup != *tmpl.Spec.Security.FSGroup { + t.Fatal("config bootstrap must create files in the shared volume group") + } + script := strings.ReplaceAll(c.Args[0], configFilesStagingPath, "'"+staging+"'") + if out, err := exec.CommandContext(t.Context(), "/bin/sh", "-c", script).CombinedOutput(); err != nil { + t.Fatalf("bootstrap: %v %s", err, out) + } + // A non-owner cannot change modes or groups, even when their requested + // values are unchanged. Reject both operations on the second invocation. + bin := filepath.Join(dir, "bin") + if err := os.Mkdir(bin, 0o755); err != nil { + t.Fatal(err) + } + for _, name := range []string{"chmod", "chgrp"} { + if err := os.WriteFile(filepath.Join(bin, name), []byte("#!/bin/sh\necho unexpected ownership mutation >&2\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(source, []byte("updated"), 0o600); err != nil { + t.Fatal(err) + } + command := exec.CommandContext(t.Context(), "/bin/sh", "-c", script) + command.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH")) + if out, err := command.CombinedOutput(); err != nil { + t.Fatalf("repeat copy required ownership of writable files: %v %s", err, out) + } + if got, err := os.ReadFile(filepath.Join(data, "cfg", "profile.txt")); err != nil || string(got) != "updated" { + t.Fatalf("config was not replaced: %q %v", got, err) + } + if info, err := os.Stat(unrelated); err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("unrelated data permissions changed: %v %v", info, err) + } +} + func TestBuildConfigInitContainer_NoChmodWithoutFSGroup(t *testing.T) { // Test 1: no Security block at all tmpl1 := &gameplanev1alpha1.GameTemplate{} diff --git a/operator/internal/controller/gameserver_idle.go b/operator/internal/controller/gameserver_idle.go index 9869ece72..055071975 100644 --- a/operator/internal/controller/gameserver_idle.go +++ b/operator/internal/controller/gameserver_idle.go @@ -292,6 +292,17 @@ func untilNext(next, now time.Time) time.Duration { func (r *GameServerReconciler) reconcileIdle( ctx context.Context, gs *gameplanev1alpha1.GameServer, ) (idleState, *gameplanev1alpha1.IdleStatus, time.Duration, error, error) { + // Keep both sleep and wake intent unchanged while an operation owns the + // workload. In particular, waking a stopped backup source is deferred. + if _, held, err := lifecycleOwner(gs); err != nil { + return idleAwake, nil, 0, nil, err + } else if held { + state := idleAwake + if parseAnnotationTime(gs, IdleAsleepSinceAnnotation) != nil { + state = idleAsleep + } + return state, gs.Status.Idle.DeepCopy(), idleReevaluate, nil, nil + } in := idleInputs{ spec: gs.Spec.Idle, suspended: gs.Spec.Suspend, @@ -349,7 +360,7 @@ func (r *GameServerReconciler) applyIdleTransition( gs.Annotations[IdleWakeCompletedAnnotation] = tok } } - if err := r.Patch(ctx, gs, client.MergeFrom(base)); err != nil { + if err := r.Patch(ctx, gs, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { return fmt.Errorf("patch idle annotations on %s/%s: %w", gs.Namespace, gs.Name, err) } return nil diff --git a/operator/internal/controller/gameserver_restart.go b/operator/internal/controller/gameserver_restart.go index 33159927d..af1ddf48d 100644 --- a/operator/internal/controller/gameserver_restart.go +++ b/operator/internal/controller/gameserver_restart.go @@ -79,5 +79,5 @@ func (r *GameServerReconciler) ackRestart(ctx context.Context, gs *gameplanev1al } gs.Annotations[RestartCompletedAnnotation] = gs.Annotations[RestartRequestedAnnotation] delete(gs.Annotations, stopRequestedAtAnnotation) - return r.Patch(ctx, gs, client.MergeFrom(base)) + return r.Patch(ctx, gs, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) } diff --git a/operator/internal/controller/gameserver_sentinel.go b/operator/internal/controller/gameserver_sentinel.go index b2c8a9edc..5415fbbf8 100644 --- a/operator/internal/controller/gameserver_sentinel.go +++ b/operator/internal/controller/gameserver_sentinel.go @@ -278,8 +278,9 @@ func (r *GameServerReconciler) reconcileSentinel( dep.Spec.Template.Spec.ServiceAccountName = sentinelSAName(gs) dep.Spec.Template.Spec.Containers = []corev1.Container{{ - Name: "sentinel", - Image: image, + Name: "sentinel", + Image: image, + Resources: auxiliaryResources(r.SentinelResources, "25m", "32Mi"), Env: []corev1.EnvVar{ {Name: "GAMESERVER_NAME", Value: gs.Name}, {Name: "GAMESERVER_NAMESPACE", Value: gs.Namespace}, diff --git a/operator/internal/controller/gameserver_template_revision.go b/operator/internal/controller/gameserver_template_revision.go new file mode 100644 index 000000000..53f8f8c44 --- /dev/null +++ b/operator/internal/controller/gameserver_template_revision.go @@ -0,0 +1,458 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + effectiveTemplateAnnotation = "gameplane.local/effective-template-revision" + templateRevisionAnnotation = "gameplane.local/template-revision" + templateSnapshotLabel = "gameplane.local/template-snapshot" + templateSnapshotKey = "template.json" + baselineTemplateAnnotation = "gameplane.local/template-baseline" + baselineWorkloadAnnotation = "gameplane.local/template-baseline-workload" + baselineInputsAnnotation = "gameplane.local/template-baseline-inputs" +) + +// Hash and persist content, not resourceVersion: status/metadata-only updates +// must not restart games. Name and the complete spec are all rendering inputs. +func templateSnapshot(tmpl *gameplanev1alpha1.GameTemplate) ([]byte, string, error) { + content := gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: tmpl.Name}, Spec: *tmpl.Spec.DeepCopy()} + data, err := json.Marshal(content) + if err != nil { + return nil, "", err + } + if len(data) > 900*1024 { + return nil, "", fmt.Errorf("template snapshot exceeds 900 KiB") + } + sum := sha256.Sum256(data) + return data, hex.EncodeToString(sum[:]), nil +} + +func validTemplateRevision(revision string) bool { + decoded, err := hex.DecodeString(revision) + return err == nil && len(decoded) == sha256.Size && hex.EncodeToString(decoded) == revision +} + +func templateSnapshotName(gs *gameplanev1alpha1.GameServer, revision string) string { + identity := sha256.Sum256([]byte(string(gs.UID) + "/" + gs.Namespace + "/" + gs.Name)) + return fmt.Sprintf("gpt-%x-%s", identity[:8], revision[:32]) +} + +func loadTemplateSnapshot(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer, revision string) (*gameplanev1alpha1.GameTemplate, error) { + if !validTemplateRevision(revision) { + return nil, fmt.Errorf("invalid effective template revision") + } + var cm corev1.ConfigMap + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: templateSnapshotName(gs, revision)}, &cm); err != nil { + return nil, fmt.Errorf("read pinned template snapshot: %w", err) + } + if !metav1.IsControlledBy(&cm, gs) || cm.Immutable == nil || !*cm.Immutable || !cm.DeletionTimestamp.IsZero() { + return nil, fmt.Errorf("template snapshot is foreign, mutable or deleting") + } + var tmpl gameplanev1alpha1.GameTemplate + if err := json.Unmarshal([]byte(cm.Data[templateSnapshotKey]), &tmpl); err != nil { + return nil, fmt.Errorf("decode template snapshot: %w", err) + } + _, digest, err := templateSnapshot(&tmpl) + if err != nil { + return nil, err + } + if digest != revision || tmpl.Name != gs.Spec.TemplateRef.Name { + return nil, fmt.Errorf("template snapshot content does not match pinned revision") + } + return &tmpl, nil +} + +// Existing pre-upgrade operations may not have a pin yet. Once a pin exists, +// absence/corruption must fail closed rather than silently reading latest. +func effectiveGameTemplate(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer) (*gameplanev1alpha1.GameTemplate, error) { + if revision := gs.Annotations[effectiveTemplateAnnotation]; revision != "" { + return loadTemplateSnapshot(ctx, reader, gs, revision) + } + var tmpl gameplanev1alpha1.GameTemplate + err := reader.Get(ctx, client.ObjectKey{Name: gs.Spec.TemplateRef.Name}, &tmpl) + return &tmpl, err +} + +func (r *GameServerReconciler) saveTemplateSnapshot(ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate) (string, error) { + data, revision, err := templateSnapshot(tmpl) + if err != nil { + return "", err + } + immutable := true + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: templateSnapshotName(gs, revision), Namespace: gs.Namespace, Labels: map[string]string{templateSnapshotLabel: string(gs.UID)}}, Immutable: &immutable, Data: map[string]string{templateSnapshotKey: string(data)}} + if err := controllerutil.SetControllerReference(gs, cm, r.Scheme); err != nil { + return "", err + } + if err := r.Create(ctx, cm); err != nil && !apierrors.IsAlreadyExists(err) { + return "", err + } + _, err = loadTemplateSnapshot(ctx, r.wipeReader(), gs, revision) + return revision, err +} + +// reconcileTemplateRevision runs before every template-dependent calculation. +// A true second result asks the caller to re-read admission state next pass. +func (r *GameServerReconciler) reconcileTemplateRevision(ctx context.Context, gs *gameplanev1alpha1.GameServer) (*gameplanev1alpha1.GameTemplate, bool, error) { + var installed gameplanev1alpha1.GameTemplate + err := r.wipeReader().Get(ctx, client.ObjectKey{Name: gs.Spec.TemplateRef.Name}, &installed) + if err != nil && !apierrors.IsNotFound(err) { + return nil, false, err + } + desired := "" + if err == nil { + _, desired, err = templateSnapshot(&installed) + if err != nil { + return nil, false, err + } + } + pin := gs.Annotations[effectiveTemplateAnnotation] + var effective *gameplanev1alpha1.GameTemplate + if pin != "" { + effective, err = loadTemplateSnapshot(ctx, r.wipeReader(), gs, pin) + if err != nil { + return nil, false, err + } + } else if desired != "" { + effective = &installed + } + op, held, err := lifecycleOwner(gs) + if err != nil { + return nil, false, err + } + if pin == "" && held && op.Kind == "Rollout" { + effective, err = loadTemplateSnapshot(ctx, r.wipeReader(), gs, op.Token) + if err != nil { + return nil, false, err + } + } + if effective == nil { + return nil, false, fmt.Errorf("GameTemplate %q not found", gs.Spec.TemplateRef.Name) + } + if err := r.reportTemplateRevision(ctx, gs, desired, op, held); err != nil { + return nil, false, err + } + if pin == "" && gs.Annotations[restoreTemplateAnnotation] != "" { + // The Restore must copy its durable source snapshot before even a + // stopped StatefulSet is rendered, otherwise InitialStartup could + // preserve a workload derived from the newest catalog template. + return effective, true, nil + } + if held && op.Kind == "Rollout" { + ready, err := recoverLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, op, r.OperationQuiescer) + if err != nil || !ready { + return effective, true, err + } + if op.Baseline { + return effective, true, r.initializeTemplateBaseline(ctx, gs, op) + } + if op.Token != pin { + // The admitted target survives a subsequent catalog update. + target, err := loadTemplateSnapshot(ctx, r.wipeReader(), gs, op.Token) + if err != nil { + return nil, false, err + } + if _, err := materializeConfig(gs, target); err != nil { + return nil, false, err + } + if _, err := resolveVersion(gs, target); err != nil { + return nil, false, err + } + if r.RestrictedWorkloads { + if err := validateRestrictedTemplate(target); err != nil { + return nil, false, err + } + } + drained, err := r.templateWorkloadDrained(ctx, gs) + if err != nil || !drained { + return effective, false, err + } + gs.Annotations[effectiveTemplateAnnotation] = op.Token + delete(gs.Annotations, stopRequestedAtAnnotation) + return effective, true, r.Update(ctx, gs) + } + observed, err := r.templateWorkloadAdopted(ctx, gs, pin) + if err != nil || !observed { + return effective, false, err + } + // Clean obsolete content before releasing, including when Automatic + // already has another catalog revision queued for the next pass. + if err := r.pruneTemplateSnapshots(ctx, gs); err != nil { + return nil, false, err + } + if err := clearLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, op.lifecycleIdentity); err != nil { + return nil, false, err + } + return effective, true, r.Update(ctx, gs) + } + // Existing owners continue rendering the old pin; initial startup of a + // restore may proceed even before the first post-upgrade pin is recorded. + if held { + return effective, false, nil + } + if desired != "" && desired != pin { + policy := gs.Spec.TemplateRollout + admit := pin == "" || policy == nil || policy.Policy == "" || policy.Policy == "Automatic" || policy.RequestedRevision == desired + if admit { + if _, err := materializeConfig(gs, &installed); err != nil { + return nil, false, err + } + if _, err := resolveVersion(gs, &installed); err != nil { + return nil, false, err + } + if r.RestrictedWorkloads { + if err := validateRestrictedTemplate(&installed); err != nil { + return nil, false, err + } + } + // Do not replace an unrelated restart or already-issued stop. + pending, err := backupLifecycleTransitionPending(ctx, r.wipeReader(), gs) + if err != nil || pending { + return effective, false, err + } + revision, err := r.saveTemplateSnapshot(ctx, gs, &installed) + if err != nil { + return nil, false, err + } + want := lifecycleIdentity{Kind: "Rollout", Name: gs.Name, UID: gs.UID, TargetUID: gs.UID, Generation: gs.Generation, Token: revision, Baseline: pin == ""} + claimed, err := claimLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, want, r.OperationQuiescer) + return effective, claimed, err + } + } + return effective, false, r.pruneTemplateSnapshots(ctx, gs) +} + +// Baseline captures render inputs rather than comparing reconstructed PodSpecs: +// API defaults/admission fields must not turn feature installation into a fleet +// rollout. These fields deliberately exclude power intent, service-only config, +// capture opt-in, scheduling backups and the rollout policy itself. +func (r *GameServerReconciler) baselineRenderInputs(gs *gameplanev1alpha1.GameServer, revision string) (string, error) { + spec := gs.Spec.DeepCopy() + spec.Suspend = false + spec.StopGracePeriodSeconds, spec.TemplateRollout, spec.BackupPolicy, spec.Idle, spec.Capture = nil, nil, nil, nil, nil + if spec.Networking.Expose == "Hostport" { + spec.Networking = gameplanev1alpha1.GameServerNetworking{Expose: "Hostport"} + } else { + spec.Networking = gameplanev1alpha1.GameServerNetworking{} + } + data, err := json.Marshal(struct { + Spec *gameplanev1alpha1.GameServerSpec + Revision, AgentImage, AgentLogLevel, ConfigInitImage string + AgentImagePullPolicy string + RestrictedWorkloads bool + AgentTokenExpirationSeconds int64 + AgentResources *corev1.ResourceRequirements + }{spec, revision, r.AgentImage, r.AgentLogLevel, r.ConfigInitImage, r.AgentImagePullPolicy, r.RestrictedWorkloads, r.AgentTokenExpirationSeconds, r.AgentResources}) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +func (r *GameServerReconciler) initializeTemplateBaseline(ctx context.Context, gs *gameplanev1alpha1.GameServer, op lifecycleOperation) error { + if _, err := loadTemplateSnapshot(ctx, r.wipeReader(), gs, op.Token); err != nil { + return err + } + var ss appsv1.StatefulSet + err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil { + if !metav1.IsControlledBy(&ss, gs) { + return fmt.Errorf("baseline workload belongs to another server") + } + // A real spec edit after admission must render normally. Recording + // its new inputs against the old workload would silently swallow it. + if gs.Generation == op.Generation { + inputs, err := r.baselineRenderInputs(gs, op.Token) + if err != nil { + return err + } + gs.Annotations[baselineTemplateAnnotation] = op.Token + gs.Annotations[baselineWorkloadAnnotation] = string(ss.UID) + gs.Annotations[baselineInputsAnnotation] = inputs + } + } + gs.Annotations[effectiveTemplateAnnotation] = op.Token + if err := clearLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, op.lifecycleIdentity); err != nil { + return err + } + return r.Update(ctx, gs) +} + +func (r *GameServerReconciler) preserveBaselinePodTemplate(ctx context.Context, gs *gameplanev1alpha1.GameServer, ss *appsv1.StatefulSet) (bool, error) { + pin := gs.Annotations[effectiveTemplateAnnotation] + if pin == "" || gs.Annotations[baselineTemplateAnnotation] != pin || string(ss.UID) != gs.Annotations[baselineWorkloadAnnotation] || ss.Spec.Template.Annotations[templateRevisionAnnotation] != "" { + return false, nil + } + inputs, err := r.baselineRenderInputs(gs, pin) + if err != nil || inputs != gs.Annotations[baselineInputsAnnotation] { + return false, err + } + // Provenance initialization must not suppress a real security upgrade. + // This also covers installing credential isolation and revisions together. + if ss.Spec.Template.Spec.AutomountServiceAccountToken == nil || *ss.Spec.Template.Spec.AutomountServiceAccountToken || r.RestrictedWorkloads != (ss.Spec.Template.Annotations[workloadSecurityLabel] == "restricted") { + return false, nil + } + projected := false + for _, volume := range ss.Spec.Template.Spec.Volumes { + if volume.Name == agentTokenVolumeName && volume.Projected != nil { + projected = true + } + } + if !projected { + return false, nil + } + // Stamp while already drained, before resume; never stamp an existing + // unannotated Pod or use its name as evidence of a known running revision. + if ss.Spec.Replicas != nil && *ss.Spec.Replicas == 0 && ss.Status.Replicas == 0 && ss.Status.ObservedGeneration >= ss.Generation { + var pod corev1.Pod + err := r.wipeReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + } + return true, nil +} + +func (r *GameServerReconciler) reportTemplateRevision(ctx context.Context, gs *gameplanev1alpha1.GameServer, desired string, op lifecycleOperation, held bool) error { + running, err := observedTemplateRevision(ctx, r.wipeReader(), gs) + if err != nil { + return err + } + base := gs.DeepCopy() + gs.Status.TemplateRevision = &gameplanev1alpha1.TemplateRevisionStatus{Desired: desired, Effective: gs.Annotations[effectiveTemplateAnnotation], Running: running} + switch { + case held && op.Kind == "Rollout": + setTemplateCondition(gs, metav1.ConditionTrue, "AdoptingRevision", "The admitted template revision is being adopted.") + case desired == "": + setTemplateCondition(gs, metav1.ConditionUnknown, "TemplateUnavailable", "The installed template is unavailable; the effective snapshot is retained.") + case desired == gs.Annotations[effectiveTemplateAnnotation]: + setTemplateCondition(gs, metav1.ConditionFalse, "RevisionCurrent", "The installed template revision is effective.") + case held: + setTemplateCondition(gs, metav1.ConditionTrue, "OperationInProgress", "Template adoption waits for the active lifecycle operation to finish.") + default: + setTemplateCondition(gs, metav1.ConditionTrue, "RevisionAvailable", "An installed template revision is waiting for admission.") + } + if reflect.DeepEqual(base.Status, gs.Status) { + return nil + } + return r.Status().Patch(ctx, gs, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) +} + +func setTemplateCondition(gs *gameplanev1alpha1.GameServer, status metav1.ConditionStatus, reason, message string) { + meta.SetStatusCondition(&gs.Status.Conditions, metav1.Condition{Type: "TemplateUpdatePending", Status: status, Reason: reason, Message: message, ObservedGeneration: gs.Generation}) +} + +func ownedTemplatePod(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer) (*corev1.Pod, error) { + var ss appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return nil, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&ss, gs) { + return nil, nil + } + var pod corev1.Pod + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + return nil, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&pod, &ss) || !pod.DeletionTimestamp.IsZero() { + return nil, nil + } + return &pod, nil +} + +func observedTemplateRevision(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer) (string, error) { + pod, err := ownedTemplatePod(ctx, reader, gs) + if err != nil || pod == nil { + return "", err + } + revision := pod.Annotations[templateRevisionAnnotation] + if !validTemplateRevision(revision) { + return "", nil + } + for _, container := range pod.Status.ContainerStatuses { + if container.Name == "game" && container.State.Running != nil { + return revision, nil + } + } + return "", nil +} + +func (r *GameServerReconciler) templateWorkloadDrained(ctx context.Context, gs *gameplanev1alpha1.GameServer) (bool, error) { + var ss appsv1.StatefulSet + if err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return false, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&ss, gs) { + return false, fmt.Errorf("template rollout workload belongs to another server") + } + if ss.Annotations[lifecycleOperationAnnotation] != gs.Annotations[lifecycleOperationAnnotation] || ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation { + return false, nil + } + var pods corev1.PodList + if err := r.wipeReader().List(ctx, &pods, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Name == gs.Name+"-0" || metav1.IsControlledBy(pod, &ss) || (liveRestorePod(pod) && podMountsRestoreTarget(pod, gs.Name)) { + return false, nil + } + } + return true, nil +} + +func (r *GameServerReconciler) templateWorkloadAdopted(ctx context.Context, gs *gameplanev1alpha1.GameServer, revision string) (bool, error) { + var ss appsv1.StatefulSet + if err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return false, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&ss, gs) || ss.Spec.Template.Annotations[templateRevisionAnnotation] != revision || ss.Status.ObservedGeneration < ss.Generation { + return false, nil + } + if ss.Spec.Replicas != nil && *ss.Spec.Replicas == 0 { + return r.templateWorkloadDrained(ctx, gs) + } + pod, err := ownedTemplatePod(ctx, r.wipeReader(), gs) + return pod != nil && pod.Annotations[templateRevisionAnnotation] == revision, err +} + +func (r *GameServerReconciler) pruneTemplateSnapshots(ctx context.Context, gs *gameplanev1alpha1.GameServer) error { + var snapshots corev1.ConfigMapList + if err := r.wipeReader().List(ctx, &snapshots, client.InNamespace(gs.Namespace), client.MatchingLabels{templateSnapshotLabel: string(gs.UID)}); err != nil { + return err + } + pin := gs.Annotations[effectiveTemplateAnnotation] + for i := range snapshots.Items { + cm := &snapshots.Items[i] + if !metav1.IsControlledBy(cm, gs) || (validTemplateRevision(pin) && cm.Name == templateSnapshotName(gs, pin)) { + continue + } + if err := r.Delete(ctx, cm, client.Preconditions{UID: &cm.UID, ResourceVersion: &cm.ResourceVersion}); client.IgnoreNotFound(err) != nil { + return err + } + } + return nil +} diff --git a/operator/internal/controller/gameserver_template_revision_test.go b/operator/internal/controller/gameserver_template_revision_test.go new file mode 100644 index 000000000..46706339c --- /dev/null +++ b/operator/internal/controller/gameserver_template_revision_test.go @@ -0,0 +1,732 @@ +package controller + +import ( + "context" + "encoding/json" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func revisionFixture(t *testing.T) (*GameServerReconciler, *gameplanev1alpha1.GameServer, *gameplanev1alpha1.GameTemplate) { + t.Helper() + s := testScheme(t) + if err := batchv1.AddToScheme(s); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}, TemplateRollout: &gameplanev1alpha1.TemplateRolloutSpec{Policy: "OnRequest"}}} + tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template"}, Spec: gameplanev1alpha1.GameTemplateSpec{Image: "example.invalid/game:old", Env: []corev1.EnvVar{{Name: "REVISION", Value: "old"}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, tmpl).WithStatusSubresource(gs, &appsv1.StatefulSet{}, &gameplanev1alpha1.Backup{}).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: s} + digest, err := r.saveTemplateSnapshot(t.Context(), gs, tmpl) + if err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{effectiveTemplateAnnotation: digest} + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + return r, gs, tmpl +} + +func TestTemplateOnRequestRetainsCompleteContents(t *testing.T) { + r, gs, installed := revisionFixture(t) + old := gs.Annotations[effectiveTemplateAnnotation] + installed.Spec.Image, installed.Spec.Env[0].Value = "example.invalid/game:new", "new" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if effective.Spec.Image != "example.invalid/game:old" || effective.Spec.Env[0].Value != "old" { + t.Fatalf("render escaped snapshot: %#v", effective.Spec) + } + if gs.Status.TemplateRevision.Effective != old || gs.Status.TemplateRevision.Desired == old { + t.Fatalf("untruthful revisions: %#v", gs.Status.TemplateRevision) + } + if c := meta.FindStatusCondition(gs.Status.Conditions, "TemplateUpdatePending"); c == nil || c.Status != metav1.ConditionTrue { + t.Fatalf("missing pending condition: %v", c) + } + // Recreating the reconciler and clearing status must not discard the pin. + gs.Status = gameplanev1alpha1.GameServerStatus{} + if err := r.Status().Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + r = &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + effective, _, err = r.reconcileTemplateRevision(t.Context(), gs) + if err != nil || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("restart lost snapshot: %v, %v", effective, err) + } +} + +func TestTemplateSnapshotRejectsMissingCorruptAndForeign(t *testing.T) { + for _, mode := range []string{"missing", "corrupt", "foreign"} { + t.Run(mode, func(t *testing.T) { + r, gs, _ := revisionFixture(t) + var cm corev1.ConfigMap + key := client.ObjectKey{Namespace: gs.Namespace, Name: templateSnapshotName(gs, gs.Annotations[effectiveTemplateAnnotation])} + if err := r.Get(t.Context(), key, &cm); err != nil { + t.Fatal(err) + } + switch mode { + case "missing": + if err := r.Delete(t.Context(), &cm); err != nil { + t.Fatal(err) + } + case "corrupt": + cm.Data[templateSnapshotKey] = `{}` + if err := r.Update(t.Context(), &cm); err != nil { + t.Fatal(err) + } + case "foreign": + cm.OwnerReferences[0].UID = "other-server" + if err := r.Update(t.Context(), &cm); err != nil { + t.Fatal(err) + } + } + if _, err := effectiveGameTemplate(t.Context(), r.Client, gs); err == nil { + t.Fatal("invalid snapshot fell through to live template") + } + }) + } +} + +func TestTemplateRequestMustMatchInstalledDigest(t *testing.T) { + r, gs, installed := revisionFixture(t) + gs.Spec.TemplateRollout.RequestedRevision = gs.Annotations[effectiveTemplateAnnotation] + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + installed.Spec.Image = "example.invalid/game:new" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + if _, _, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil { + t.Fatal(err) + } + if _, held, _ := lifecycleOwner(gs); held { + t.Fatal("old request admitted a newer template") + } +} + +func TestTemplateAdoptionWaitsForLifecycleOwner(t *testing.T) { + for _, kind := range []string{"Backup", "Restore", "Wipe"} { + t.Run(kind, func(t *testing.T) { + r, gs, installed := revisionFixture(t) + gs.Spec.TemplateRollout.Policy = "Automatic" + op := lifecycleIdentity{Kind: kind, Name: "owner", UID: "owner-uid", TargetUID: gs.UID, Token: "request"} + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: op}) + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + installed.Spec.Image = "example.invalid/game:new" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("protected operation adopted template: %v", err) + } + owner, _, _ := lifecycleOwner(gs) + if owner.Kind != kind { + t.Fatal("stole lifecycle owner") + } + }) + } +} + +func TestRunningTemplateRequiresOwnedRunningGame(t *testing.T) { + r, gs, _ := revisionFixture(t) + digest := gs.Annotations[effectiveTemplateAnnotation] + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "workload-uid", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, Annotations: map[string]string{templateRevisionAnnotation: digest}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(ss, appsv1.SchemeGroupVersion.WithKind("StatefulSet"))}}} + if err := r.Create(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + if got, err := observedTemplateRevision(t.Context(), r.Client, gs); err != nil || got != "" { + t.Fatalf("pending Pod reported running: %q %v", got, err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "game", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}} + if err := r.Status().Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + if got, err := observedTemplateRevision(t.Context(), r.Client, gs); err != nil || got != digest { + t.Fatalf("running Pod not observed: %q %v", got, err) + } + pod.OwnerReferences[0].UID = types.UID("replaced-workload") + if err := r.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + if got, err := observedTemplateRevision(context.Background(), r.Client, gs); err != nil || got != "" { + t.Fatalf("foreign Pod reported running: %q %v", got, err) + } +} + +func TestTemplateAdoptionDrainsAndKeepsAdmittedRevisionAcrossRestart(t *testing.T) { + r, gs, installed := revisionFixture(t) + old := gs.Annotations[effectiveTemplateAnnotation] + zero, one := int32(0), int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "workload", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}, Status: appsv1.StatefulSetStatus{Replicas: 1}} + if err := r.Create(t.Context(), ss); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, UID: "old-pod", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(ss, appsv1.SchemeGroupVersion.WithKind("StatefulSet"))}}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + installed.Spec.Image = "example.invalid/game:admitted" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + _, target, _ := templateSnapshot(installed) + gs.Spec.TemplateRollout.RequestedRevision = target + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("admission: %v %v", changed, err) + } + owner, held, _ := lifecycleOwner(gs) + if !held || owner.Kind != "Rollout" || owner.Token != target { + t.Fatalf("wrong owner: %#v", owner) + } + // A new leader and a newer catalog must retain the admitted target. + r = &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + installed.Spec.Image = "example.invalid/game:later" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + effective, changed, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil || changed || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("adopted before drain: %v %v", changed, err) + } + ss.Spec.Replicas = &zero + ss.Annotations = map[string]string{lifecycleOperationAnnotation: gs.Annotations[lifecycleOperationAnnotation]} + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.Replicas = 0 + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || changed || gs.Annotations[effectiveTemplateAnnotation] != old { + t.Fatalf("live old Pod escaped drain: %v %v", changed, err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed || gs.Annotations[effectiveTemplateAnnotation] != target { + t.Fatalf("pin not committed: %v %v", changed, err) + } + effective, err = effectiveGameTemplate(t.Context(), r.Client, gs) + if err != nil || effective.Spec.Image != "example.invalid/game:admitted" { + t.Fatalf("catalog replaced admitted content: %v", err) + } + // Suspended adoption releases after observing the new stopped template. + gs.Spec.Suspend = true + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + ss.Spec.Template.Annotations = map[string]string{templateRevisionAnnotation: target} + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("release: %v %v", changed, err) + } + if _, held, _ := lifecycleOwner(gs); held || !gs.Spec.Suspend { + t.Fatal("owner retained or user suspend lost") + } + if _, _, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil { + t.Fatal(err) + } + if gs.Status.TemplateRevision.Running != "" || gs.Status.TemplateRevision.Desired == target { + t.Fatalf("untruthful suspended/latest status: %#v", gs.Status.TemplateRevision) + } + var snapshots corev1.ConfigMapList + if err := r.List(t.Context(), &snapshots); err != nil { + t.Fatal(err) + } + if len(snapshots.Items) != 1 { + t.Fatalf("obsolete snapshots retained: %d", len(snapshots.Items)) + } +} + +func TestPinnedTemplateDrivesConfigInitStorageAndImage(t *testing.T) { + r, gs, old := revisionFixture(t) + old.Spec.Storage = gameplanev1alpha1.GameStorageSpec{MountPath: "/old-data", Size: resource.MustParse("1Gi"), Extra: []gameplanev1alpha1.ExtraVolumeSpec{{Name: "old", MountPath: "/old-extra", Size: resource.MustParse("1Gi")}}} + old.Spec.ConfigFiles = []gameplanev1alpha1.ConfigFile{{Path: "revision.txt", Template: "old contents"}} + revision, err := r.saveTemplateSnapshot(t.Context(), gs, old) + if err != nil { + t.Fatal(err) + } + gs.Annotations[effectiveTemplateAnnotation] = revision + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + latest := old.DeepCopy() + latest.Spec.Image, latest.Spec.Storage.MountPath = "example.invalid/game:new", "/new-data" + latest.Spec.Storage.Extra[0].Name, latest.Spec.Storage.Extra[0].MountPath = "new", "/new-extra" + latest.Spec.ConfigFiles[0].Template = "new contents" + if err := r.Update(t.Context(), latest); err != nil { + t.Fatal(err) + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + mc, err := materializeConfig(gs, effective) + if err != nil { + t.Fatal(err) + } + if err := r.reconcilePVC(t.Context(), gs, effective); err != nil { + t.Fatal(err) + } + if err := r.reconcileExtraPVCs(t.Context(), gs, effective); err != nil { + t.Fatal(err) + } + if err := r.reconcileFilesSecret(t.Context(), gs, mc); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, mc, 1); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Template.Spec.Containers[0].Image != old.Spec.Image { + t.Fatal("image used latest") + } + for _, container := range append(ss.Spec.Template.Spec.Containers, ss.Spec.Template.Spec.InitContainers...) { + for _, mount := range container.VolumeMounts { + if mount.MountPath == "/new-data" || mount.MountPath == "/new-extra" { + t.Fatalf("mount escaped snapshot: %#v", mount) + } + } + } + if len(ss.Spec.Template.Spec.InitContainers) == 0 { + t.Fatal("pinned config-init absent") + } + var files corev1.Secret + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-files"}, &files); err != nil { + t.Fatal(err) + } + found := false + for _, value := range files.Data { + if string(value) == "old contents" { + found = true + } + } + if !found { + t.Fatalf("files used latest: %#v", files.Data) + } + var pvcs corev1.PersistentVolumeClaimList + if err := r.List(t.Context(), &pvcs); err != nil { + t.Fatal(err) + } + if len(pvcs.Items) != 2 { + t.Fatalf("wrong effective PVC set: %d", len(pvcs.Items)) + } + for _, pvc := range pvcs.Items { + if pvc.Name == gs.Name+"-extra-new" { + t.Fatal("pending revision provisioned new extra storage") + } + } +} + +func TestSnapshotRestorePinsEffectiveSourceBeforeStartup(t *testing.T) { + r, source, installed := revisionFixture(t) + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: source.Namespace, UID: "restore-uid"}} + if err := r.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + target := source.DeepCopy() + target.Name, target.UID, target.ResourceVersion = "restored", "target-uid", "" + target.Annotations = map[string]string{} + restorer := &RestoreReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if err := restorer.planRestoredTemplate(t.Context(), rs, source, target); err != nil { + t.Fatal(err) + } + if err := r.Create(t.Context(), target); err != nil { + t.Fatal(err) + } + installed.Spec.Image = "example.invalid/game:new" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + if _, wait, err := r.reconcileTemplateRevision(t.Context(), target); err != nil || !wait { + t.Fatalf("target bypassed snapshot copy: wait=%v err=%v", wait, err) + } + if err := restorer.copyRestoredTemplate(t.Context(), rs, target); err != nil { + t.Fatal(err) + } + effective, err := effectiveGameTemplate(t.Context(), r.Client, target) + if err != nil || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("restore adopted latest content: %v", err) + } +} + +type revisionStopper struct{ calls int } + +func (s *revisionStopper) Stop(context.Context, string, string) error { s.calls++; return nil } +func (*revisionStopper) Enabled() bool { return true } + +func TestTemplateRolloutUsesOldGracefulStopAndBlocksBackup(t *testing.T) { + r, gs, old := revisionFixture(t) + old.Spec.RCON = &gameplanev1alpha1.RCONSpec{Protocol: "source"} + old.Spec.Capabilities = &gameplanev1alpha1.CapabilitiesSpec{Lifecycle: &gameplanev1alpha1.LifecycleSpec{Stop: []string{"save", "stop"}}} + pin, err := r.saveTemplateSnapshot(t.Context(), gs, old) + if err != nil { + t.Fatal(err) + } + gs.Annotations[effectiveTemplateAnnotation] = pin + gs.Spec.TemplateRollout.Policy = "Automatic" + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + newTemplate := old.DeepCopy() + newTemplate.Spec.Image = "example.invalid/game:new" + newTemplate.Spec.Capabilities = nil // latest no longer declares graceful stop + if err := r.Update(t.Context(), newTemplate); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("automatic admission: %v %v", changed, err) + } + if claimed, err := claimLifecycleOperation(t.Context(), r.Client, r.Client, gs, lifecycleIdentity{Kind: "Backup", Name: "copy", UID: "copy-uid", TargetUID: gs.UID}, nil); err != nil || claimed { + t.Fatalf("backup entered rollout: %v %v", claimed, err) + } + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "workload", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}, Status: appsv1.StatefulSetStatus{ReadyReplicas: 1, Replicas: 1}} + if err := r.Create(t.Context(), ss); err != nil { + t.Fatal(err) + } + stopper := &revisionStopper{} + r.AgentClient = stopper + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + replicas, poll, err := r.desiredReplicas(t.Context(), gs, effective, idleAwake) + if err != nil || replicas != 1 || poll <= 0 || stopper.calls != 1 { + t.Fatalf("old graceful contract bypassed: replicas=%d poll=%v calls=%d err=%v", replicas, poll, stopper.calls, err) + } +} + +func TestPinnedTemplateSurvivesInstalledTemplateDeletion(t *testing.T) { + r, gs, installed := revisionFixture(t) + if err := r.Delete(t.Context(), installed); err != nil { + t.Fatal(err) + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("installed deletion lost pinned content: %v", err) + } + if gs.Status.TemplateRevision.Desired != "" { + t.Fatal("missing installed revision reported as available") + } + condition := meta.FindStatusCondition(gs.Status.Conditions, "TemplateUpdatePending") + if condition == nil || condition.Status != metav1.ConditionUnknown { + t.Fatalf("missing unavailable condition: %#v", condition) + } +} + +func TestStaleTemplateRenderCannotUndoCommittedPin(t *testing.T) { + r, gs, old := revisionFixture(t) + stale := gs.DeepCopy() + newTemplate := old.DeepCopy() + newTemplate.Spec.Image = "example.invalid/game:new" + pin, err := r.saveTemplateSnapshot(t.Context(), gs, newTemplate) + if err != nil { + t.Fatal(err) + } + gs.Annotations[effectiveTemplateAnnotation] = pin + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), stale, old, nil, &materializedConfig{}, 1); err == nil { + t.Fatal("stale rendering created a workload after pin change") + } +} + +func TestTemplateBaselinePreservesLegacyWorkloadUntilAdmittedUpdate(t *testing.T) { + for _, policy := range []string{"Automatic", "OnRequest"} { + t.Run(policy, func(t *testing.T) { + r, gs, installed := revisionFixture(t) + delete(gs.Annotations, effectiveTemplateAnnotation) + gs.Spec.TemplateRollout.Policy = policy + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + installed.Spec.RCON = &gameplanev1alpha1.RCONSpec{Protocol: "source"} + installed.Spec.Capabilities = &gameplanev1alpha1.CapabilitiesSpec{Lifecycle: &gameplanev1alpha1.LifecycleSpec{Stop: []string{"save", "stop"}}} + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "legacy-workload", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Replicas: &one, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{"admission.example/default": "keep"}}, Spec: corev1.PodSpec{DNSPolicy: corev1.DNSClusterFirst, RestartPolicy: corev1.RestartPolicyAlways, SchedulerName: "default-scheduler", Containers: []corev1.Container{{Name: "game", Image: installed.Spec.Image, ImagePullPolicy: corev1.PullIfNotPresent, TerminationMessagePath: "/dev/termination-log", TerminationMessagePolicy: corev1.TerminationMessageReadFile}}}}}, Status: appsv1.StatefulSetStatus{Replicas: 1, ReadyReplicas: 1}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(ss, appsv1.SchemeGroupVersion.WithKind("StatefulSet"))}}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "game", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}} + for _, obj := range []client.Object{ss, pod} { + if err := r.Create(t.Context(), obj); err != nil { + t.Fatal(err) + } + } + // Credential isolation is already applied; provenance alone must + // preserve the API-defaulted template without restarting it. + applyWorkloadSecurity(&ss.Spec.Template.Spec, installed, false, 3600) + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + before, err := json.Marshal(ss.Spec.Template) + if err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("baseline admission: changed=%v err=%v", changed, err) + } + owner, held, err := lifecycleOwner(gs) + if err != nil || !held || !owner.Baseline { + t.Fatalf("missing durable baseline intent: %#v %v", owner, err) + } + // A restarted reconciler must preserve replicas while baseline ownership is held. + stopper := &revisionStopper{} + r = &GameServerReconciler{Client: r.Client, APIReader: r.APIReader, Scheme: r.Scheme, AgentClient: stopper} + if replicas, _, err := r.desiredReplicas(t.Context(), gs, installed, idleAwake); err != nil || replicas != 1 || stopper.calls != 0 { + t.Fatalf("baseline stopped legacy workload: replicas=%d calls=%d err=%v", replicas, stopper.calls, err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("baseline pin: changed=%v err=%v", changed, err) + } + pin := gs.Annotations[effectiveTemplateAnnotation] + if !validTemplateRevision(pin) { + t.Fatal("baseline did not save a pin") + } + if _, held, _ := lifecycleOwner(gs); held { + t.Fatal("baseline retained lifecycle owner") + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if gs.Status.TemplateRevision.Running != "" { + t.Fatal("legacy Pod assigned an invented running revision") + } + mc, err := materializeConfig(gs, effective) + if err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, mc, 1); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil { + t.Fatal(err) + } + after, err := json.Marshal(ss.Spec.Template) + if err != nil { + t.Fatal(err) + } + if string(after) != string(before) || stopper.calls != 0 || *ss.Spec.Replicas != 1 { + t.Fatal("baseline changed legacy Pod template or power state") + } + + installed.Spec.Image = "example.invalid/game:new" + installed.Spec.Capabilities = nil + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + _, desired, err := templateSnapshot(installed) + if err != nil { + t.Fatal(err) + } + if policy == "OnRequest" { + gs.Spec.TemplateRollout.RequestedRevision = desired + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("next revision admission: %v %v", changed, err) + } + effective, _, err = r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if replicas, _, err := r.desiredReplicas(t.Context(), gs, effective, idleAwake); err != nil || replicas != 1 || stopper.calls != 1 { + t.Fatalf("next revision bypassed old graceful stop: replicas=%d calls=%d err=%v", replicas, stopper.calls, err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, mc, 0); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || changed { + t.Fatalf("adopted before owned drain: changed=%v err=%v", changed, err) + } + if gs.Annotations[effectiveTemplateAnnotation] != pin { + t.Fatal("pin advanced while legacy Pod remains") + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil { + t.Fatal(err) + } + if ss.Annotations[lifecycleOperationAnnotation] != gs.Annotations[lifecycleOperationAnnotation] { + t.Fatal("drain not owned by admitted rollout") + } + ss.Status.Replicas, ss.Status.ReadyReplicas, ss.Status.ObservedGeneration = 0, 0, ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("drained adoption: %v %v", changed, err) + } + if gs.Annotations[effectiveTemplateAnnotation] != desired { + t.Fatal("drained rollout did not adopt requested pin") + } + }) + } +} + +func TestTemplateBaselineAllowsRealWorkloadChanges(t *testing.T) { + for _, change := range []string{"inputs", "stopped", "new-workload", "credential-isolation", "restricted-profile"} { + t.Run(change, func(t *testing.T) { + r, gs, _ := revisionFixture(t) + pin := gs.Annotations[effectiveTemplateAnnotation] + inputs, err := r.baselineRenderInputs(gs, pin) + if err != nil { + t.Fatal(err) + } + gs.Annotations[baselineTemplateAnnotation], gs.Annotations[baselineWorkloadAnnotation], gs.Annotations[baselineInputsAnnotation] = pin, "legacy", inputs + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{UID: "legacy"}, Spec: appsv1.StatefulSetSpec{Replicas: &one}} + applyWorkloadSecurity(&ss.Spec.Template.Spec, &gameplanev1alpha1.GameTemplate{}, false, 3600) + if keep, err := r.preserveBaselinePodTemplate(t.Context(), gs, ss); err != nil || !keep { + t.Fatalf("baseline not preserved: %v %v", keep, err) + } + switch change { + case "inputs": + gs.Spec.Image = "example.invalid/game:override" + case "stopped": + zero := int32(0) + ss.Spec.Replicas = &zero + case "new-workload": + ss.UID = "replacement" + case "credential-isolation": + ss.Spec.Template.Spec.AutomountServiceAccountToken = nil + case "restricted-profile": + r.RestrictedWorkloads = true + } + if keep, err := r.preserveBaselinePodTemplate(t.Context(), gs, ss); err != nil || keep { + t.Fatalf("real workload change frozen: %v %v", keep, err) + } + }) + } +} + +func TestTemplateBaselineWithoutWorkloadPinsBeforeFirstRender(t *testing.T) { + r, gs, _ := revisionFixture(t) + delete(gs.Annotations, effectiveTemplateAnnotation) + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + for range 2 { + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("initial pin: %v %v", changed, err) + } + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Template.Annotations[templateRevisionAnnotation] != gs.Annotations[effectiveTemplateAnnotation] || *ss.Spec.Replicas != 1 { + t.Fatal("first workload not stamped and started") + } +} + +func TestTemplateBaselineDoesNotSwallowConcurrentSpecEdit(t *testing.T) { + r, gs, installed := revisionFixture(t) + delete(gs.Annotations, effectiveTemplateAnnotation) + gs.Generation = 1 + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "legacy", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Replicas: &one, Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "game", Image: installed.Spec.Image}}}}}} + if err := r.Create(t.Context(), ss); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("baseline admission: %v %v", changed, err) + } + gs.Generation++ // Fake clients do not increment spec generations. + gs.Spec.Image = "example.invalid/game:requested" + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("baseline initialization: %v %v", changed, err) + } + if gs.Annotations[baselineInputsAnnotation] != "" { + t.Fatal("concurrent edit was recorded as already applied") + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Template.Spec.Containers[0].Image != gs.Spec.Image { + t.Fatal("baseline swallowed the requested image change") + } +} + +func TestTemplateRevisionRejectsRestrictedIncompatibleTargetBeforeAdmission(t *testing.T) { + r, gs, installed := revisionFixture(t) + r.RestrictedWorkloads = true + old := gs.Annotations[effectiveTemplateAnnotation] + installed.Spec.Image = "example.invalid/game:incompatible" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + gs.Spec.TemplateRollout.Policy = "Automatic" + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if _, _, err := r.reconcileTemplateRevision(t.Context(), gs); err == nil { + t.Fatal("admitted a template without Restricted-compatible identity") + } + if _, held, _ := lifecycleOwner(gs); held || gs.Annotations[effectiveTemplateAnnotation] != old { + t.Fatal("invalid security profile changed the pin or admitted a drain") + } +} diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index a950292b2..e5712d96b 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -392,6 +392,7 @@ func (r *GameServerReconciler) reconcileTunnel( dep.Spec.Template.Spec.Containers = []corev1.Container{{ Name: "tunnel", Image: image, + Resources: auxiliaryResources(r.TunnelResources, "25m", "64Mi"), Env: envVars, Ports: ports, VolumeMounts: volumeMounts, diff --git a/operator/internal/controller/gameserver_wipe.go b/operator/internal/controller/gameserver_wipe.go index ca96d7d63..c59ecf998 100644 --- a/operator/internal/controller/gameserver_wipe.go +++ b/operator/internal/controller/gameserver_wipe.go @@ -51,7 +51,7 @@ func (r *GameServerReconciler) reconcileWipe( if err != nil { return err } - if gs.Annotations[restoreGuardAnnotation] != "" { + if gs.Annotations[restoreGuardAnnotation] != "" || gs.Annotations[backupGuardAnnotation] != "" { return nil } req := gs.Annotations[WipeRequestedAnnotation] diff --git a/operator/internal/controller/gameserver_wipe_guard.go b/operator/internal/controller/gameserver_wipe_guard.go index f86a1969d..bdf00bfc8 100644 --- a/operator/internal/controller/gameserver_wipe_guard.go +++ b/operator/internal/controller/gameserver_wipe_guard.go @@ -165,8 +165,11 @@ func (r *GameServerReconciler) ensureWipeGuard(ctx context.Context, gs *gameplan return false, fmt.Errorf("GameServer changed identity before wipe acquisition") } *gs = live - if gs.Annotations[wipeGuardAnnotation] != "" || gs.Annotations[restoreGuardAnnotation] != "" { - return false, nil + if token := gs.Annotations[wipeGuardAnnotation]; token != "" { + ready, err := claimLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, wipeLifecycleIdentity(gs, token), r.OperationQuiescer) + // Upgrading an existing guard does not require another pass. Only + // unfinished predecessor cleanup must prevent this wipe proceeding. + return err == nil && !ready, err } state := wipeInventory(inventory) if err := r.loadWipeWorkers(ctx, gs, state); err != nil { @@ -189,20 +192,8 @@ func (r *GameServerReconciler) ensureWipeGuard(ctx context.Context, gs *gameplan if token == "" { token = wipeOrphanGuard } - if gs.Annotations == nil { - gs.Annotations = make(map[string]string) - } - gs.Annotations[wipeGuardAnnotation] = token - // Recovery without a pending request is cleanup, including legacy orphan - // pods with no token. The guard fences live workers without changing the - // current resume/stop intent. Pending and failed matching wipes suspend. - if pendingWipe(gs) { - gs.Spec.Suspend = true - } - if err := r.Update(ctx, gs); err != nil { - return false, err - } - return true, nil + _, err = claimLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, wipeLifecycleIdentity(gs, token), r.OperationQuiescer) + return err == nil && gs.Annotations[wipeGuardAnnotation] == token, err } // Stamp the owned workload with a checked write before workers launch. A stale @@ -299,6 +290,9 @@ func (r *GameServerReconciler) releaseWipeGuard(ctx context.Context, gs *gamepla if err := r.clearWipeWorkloadGuard(ctx, &live, token); err != nil { return err } + if err := clearLifecycleOperation(ctx, r.Client, r.wipeReader(), &live, wipeLifecycleIdentity(&live, token)); err != nil { + return err + } if success { live.Annotations[WipeCompletedAnnotation] = token live.Spec.Suspend = false @@ -307,6 +301,7 @@ func (r *GameServerReconciler) releaseWipeGuard(ctx context.Context, gs *gamepla // Transfer only after old Jobs and live pods have drained; next pass // fences the workload for this new token before launching a worker. live.Annotations[wipeGuardAnnotation] = live.Annotations[WipeRequestedAnnotation] + live.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: wipeLifecycleIdentity(&live, live.Annotations[WipeRequestedAnnotation])}) live.Spec.Suspend = true } else { delete(live.Annotations, wipeGuardAnnotation) diff --git a/operator/internal/controller/gameserver_wipe_inventory_test.go b/operator/internal/controller/gameserver_wipe_inventory_test.go index e4075147d..2ae64daf1 100644 --- a/operator/internal/controller/gameserver_wipe_inventory_test.go +++ b/operator/internal/controller/gameserver_wipe_inventory_test.go @@ -45,6 +45,22 @@ func TestGameServerReconcileInventoriesWipeWorkersOnce(t *testing.T) { c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, tmpl, caSecret).WithStatusSubresource(gs).Build() reader := &wipeInventoryReader{Reader: c} r := &GameServerReconciler{Client: c, APIReader: reader, Scheme: s, AgentCASecretName: caSecret.Name, AgentCASecretNamespace: caSecret.Namespace} + // This test measures an ordinary already-admitted reconcile. Template + // admission has its own live worker checks and transition tests. + revision, err := r.saveTemplateSnapshot(t.Context(), gs, tmpl) + if err != nil { + t.Fatal(err) + } + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[effectiveTemplateAnnotation] = revision + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := r.reportTemplateRevision(t.Context(), gs, revision, lifecycleOperation{}, false); err != nil { + t.Fatal(err) + } // Exercise both initial child/status creation and the next full pass. Idle // and capture reconciliation participate, rather than just lifecycle helpers. for range 2 { diff --git a/operator/internal/controller/helpers.go b/operator/internal/controller/helpers.go index be11001c0..8bd3890d6 100644 --- a/operator/internal/controller/helpers.go +++ b/operator/internal/controller/helpers.go @@ -13,8 +13,10 @@ import ( ) // upsertCondition replaces a condition by Type if present, otherwise -// appends. Sets LastTransitionTime to now when the status flips. +// appends. Sets LastTransitionTime to now when the status flips. Preserve the +// input slice so callers can compare old and new conditions before persisting. func upsertCondition(conds []metav1.Condition, c metav1.Condition) []metav1.Condition { + conds = append([]metav1.Condition(nil), conds...) now := metav1.Now() for i := range conds { if conds[i].Type != c.Type { diff --git a/operator/internal/controller/helpers_envtest_test.go b/operator/internal/controller/helpers_envtest_test.go index 7db1ea7c1..8069878a8 100644 --- a/operator/internal/controller/helpers_envtest_test.go +++ b/operator/internal/controller/helpers_envtest_test.go @@ -165,11 +165,10 @@ func startMgr(t *testing.T, ns string, setups ...setupReconciler) client.Client Scheme: scheme, Logger: logr.Discard(), Cache: cache.Options{ - // Include the test namespace AND cluster scope (""). Without - // the "" entry, controller-runtime's namespace-restricted - // cache will refuse to serve reads of cluster-scoped CRDs - // like GameTemplate. - DefaultNamespaces: map[string]cache.Config{ns: {}, "": {}}, + // Cluster-scoped kinds use the independent cluster cache. An + // empty namespace key means AllNamespaces and would make this + // manager reconcile fixtures left by every earlier test. + DefaultNamespaces: map[string]cache.Config{ns: {}}, }, Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0", @@ -571,12 +570,18 @@ func markBackupSucceeded(t *testing.T, ns, name, snapshotID, size string) { // reconciler waits for Suspended/Stopped before advancing to Running. func markGameServerPhase(t *testing.T, ns, name string, phase gameplanev1alpha1.GameServerPhase) { t.Helper() - var gs gameplanev1alpha1.GameServer - if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, &gs); err != nil { - t.Fatalf("get gameserver: %v", err) - } - gs.Status.Phase = phase - if err := k8sClient.Status().Update(context.Background(), &gs); err != nil { + // The Restore reconciler also writes the target's ownership annotations. + // Refetch after conflicts so this simulated status observation preserves + // concurrent controller writes without turning them into fixture failures. + ctx := t.Context() + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var gs gameplanev1alpha1.GameServer + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: name}, &gs); err != nil { + return err + } + gs.Status.Phase = phase + return k8sClient.Status().Update(ctx, &gs) + }); err != nil { t.Fatalf("status update gameserver: %v", err) } } @@ -774,6 +779,14 @@ func (f *fakeAgent) Unquiesce(_ context.Context, _, _ string) error { return f.unquiesceErr } +func (f *fakeAgent) CheckPod(context.Context, string, string, string) error { return nil } +func (f *fakeAgent) QuiescePod(ctx context.Context, namespace, server, _ string) error { + return f.Quiesce(ctx, namespace, server) +} +func (f *fakeAgent) UnquiescePod(ctx context.Context, namespace, server, _ string) error { + return f.Unquiesce(ctx, namespace, server) +} + func (f *fakeAgent) quiesceCount() int { f.mu.Lock() defer f.mu.Unlock() @@ -797,13 +810,32 @@ func (f *fakeLogReader) BackupLogs(_ context.Context, _, _ string) (io.ReadClose return io.NopCloser(strings.NewReader(f.body)), nil } -// seedGameServer creates a bare GameServer so reconcilers that resolve -// Backup/Restore serverRefs find their target. No GameServer reconciler -// runs in these suites, so the object stays inert (no children). +// seedGameServer creates a GameServer and observed workload for backup tests. +// No StatefulSet controller runs in envtest; report its generation explicitly +// so backup admission can exercise the real workload fence. func seedGameServer(t *testing.T, ns, name string) { t.Helper() gs := buildGameServer(ns, name, "seed-template") if err := k8sClient.Create(context.Background(), gs); err != nil { t.Fatalf("seed gameserver %s/%s: %v", ns, name, err) } + seedBackupWorkload(t, gs) +} + +func seedBackupWorkload(t *testing.T, gs *gameplanev1alpha1.GameServer) { + t.Helper() + name, ns := gs.Name, gs.Namespace + one := int32(1) + labels := map[string]string{"app": name} + ss := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "GameServer", Name: name, UID: gs.UID, Controller: ownerBoolPtr(true)}}}, + Spec: appsv1.StatefulSetSpec{Replicas: &one, ServiceName: name, Selector: &metav1.LabelSelector{MatchLabels: labels}, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "game", Image: "test/game:fixture"}}}}}, + } + if err := k8sClient.Create(context.Background(), ss); err != nil { + t.Fatalf("seed workload: %v", err) + } + ss.Status.ObservedGeneration = ss.Generation + if err := k8sClient.Status().Update(context.Background(), ss); err != nil { + t.Fatalf("observe workload: %v", err) + } } diff --git a/operator/internal/controller/helpers_extra_test.go b/operator/internal/controller/helpers_extra_test.go index 85a8d2ccf..ea6ab4ce0 100644 --- a/operator/internal/controller/helpers_extra_test.go +++ b/operator/internal/controller/helpers_extra_test.go @@ -53,6 +53,19 @@ func TestUpsertCondition_StatusFlip_BumpsTimestamp(t *testing.T) { } } +func TestUpsertConditionPreservesComparisonBase(t *testing.T) { + for _, status := range []metav1.ConditionStatus{metav1.ConditionFalse, metav1.ConditionTrue} { + t.Run(string(status), func(t *testing.T) { + original := metav1.Condition{Type: "SnapshotCleaned", Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: metav1.Now()} + conditions := []metav1.Condition{original} + updated := upsertCondition(conditions, metav1.Condition{Type: original.Type, Status: status, Reason: "Updated"}) + if conditions[0] != original || sameConditions(conditions, updated) { + t.Fatalf("upsert hid the change from status persistence: before=%+v after=%+v", conditions, updated) + } + }) + } +} + func TestSameConditions(t *testing.T) { a := []metav1.Condition{ {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Up"}, diff --git a/operator/internal/controller/lifecycle_operation.go b/operator/internal/controller/lifecycle_operation.go new file mode 100644 index 000000000..6445aa927 --- /dev/null +++ b/operator/internal/controller/lifecycle_operation.go @@ -0,0 +1,454 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const lifecycleOperationAnnotation = "operations.gameplane.local/owner" + +var errLifecycleGenerationChanged = errors.New("lifecycle operation spec changed after admission") + +// The GameServer is the arbitration object. A wipe has no separate CR, so its +// request token and GameServer UID identify the operation. Generation is pinned +// at admission; subsequent power requests must not change that identity. +type lifecycleIdentity struct { + Kind string `json:"kind"` + Name string `json:"name"` + UID types.UID `json:"uid"` + TargetUID types.UID `json:"targetUID"` + Generation int64 `json:"generation"` + Token string `json:"token,omitempty"` + NeedsUnquiesce bool `json:"needsUnquiesce,omitempty"` + QuiescePodUID types.UID `json:"quiescePodUID,omitempty"` + // InitialStartup permits only the first workload of a CSI restore's new + // target (including WaitForFirstConsumer PVCs), while other operations wait. + InitialStartup bool `json:"initialStartup,omitempty"` + // Baseline establishes a first template pin without replacing an existing + // workload. Durable intent prevents a leader restart turning it into a drain. + Baseline bool `json:"baseline,omitempty"` +} + +type lifecycleOperation struct { + lifecycleIdentity + // Recovery survives deletion of a successor while it is draining an older + // owner's workers. No new work is authorized until every entry is cleaned. + Recovery []lifecycleIdentity `json:"recovery,omitempty"` +} + +func lifecycleOperationJSON(op lifecycleOperation) string { + value, _ := json.Marshal(op) + return string(value) +} + +func parseLifecycleOperation(value string) (lifecycleOperation, error) { + var op lifecycleOperation + if err := json.Unmarshal([]byte(value), &op); err != nil { + return op, fmt.Errorf("decode lifecycle owner: %w", err) + } + for _, id := range append([]lifecycleIdentity{op.lifecycleIdentity}, op.Recovery...) { + if id.Name == "" || id.UID == "" || id.TargetUID == "" || (id.Kind != "Backup" && id.Kind != "Restore" && id.Kind != "Wipe" && id.Kind != "Rollout") || (id.Kind == "Wipe" && id.Token == "") || (id.Kind == "Rollout" && !validTemplateRevision(id.Token)) { + return op, fmt.Errorf("invalid lifecycle owner identity") + } + if id.TargetUID != op.TargetUID { + return op, fmt.Errorf("lifecycle recovery target changed identity") + } + } + return op, nil +} + +func backupLifecycleIdentity(gs *gameplanev1alpha1.GameServer, b *gameplanev1alpha1.Backup, quiesce bool) lifecycleIdentity { + return lifecycleIdentity{Kind: "Backup", Name: b.Name, UID: b.UID, TargetUID: gs.UID, Generation: b.Generation, NeedsUnquiesce: quiesce} +} + +func restoreLifecycleIdentity(gs *gameplanev1alpha1.GameServer, rs *gameplanev1alpha1.Restore) lifecycleIdentity { + return lifecycleIdentity{Kind: "Restore", Name: rs.Name, UID: rs.UID, TargetUID: gs.UID, Generation: rs.Generation} +} + +func wipeLifecycleIdentity(gs *gameplanev1alpha1.GameServer, token string) lifecycleIdentity { + return lifecycleIdentity{Kind: "Wipe", Name: gs.Name, UID: gs.UID, TargetUID: gs.UID, Generation: gs.Generation, Token: token} +} + +func sameLifecycleOwner(a, b lifecycleIdentity) bool { + return a.Kind == b.Kind && a.Name == b.Name && a.UID == b.UID && a.TargetUID == b.TargetUID && a.Token == b.Token +} + +// Existing guards remain readable during upgrades and provide the compatibility +// markers used by stop/drain cleanup. They are never independent permission to +// start new work. Contradictory legacy guards fail closed until their owners +// complete cleanup; migration must not select one concurrent writer as safe. +func lifecycleOwner(gs *gameplanev1alpha1.GameServer) (lifecycleOperation, bool, error) { + if value := gs.Annotations[lifecycleOperationAnnotation]; value != "" { + op, err := parseLifecycleOperation(value) + if err == nil && op.TargetUID != gs.UID { + err = fmt.Errorf("lifecycle owner belongs to a replaced GameServer") + } + return op, true, err + } + op := lifecycleOperation{} + count := 0 + if value := gs.Annotations[backupGuardAnnotation]; value != "" { + id, err := parseBackupIdentity(value) + if err != nil { + return op, true, err + } + op.lifecycleIdentity = lifecycleIdentity{Kind: "Backup", Name: id.Name, UID: id.UID, TargetUID: gs.UID, NeedsUnquiesce: true} + lineage, err := parseBackupRecoveryLineage(gs.Annotations[backupRecoveryLineageAnnotation]) + if err != nil { + return op, true, err + } + for _, predecessor := range lineage { + op.Recovery = append(op.Recovery, lifecycleIdentity{Kind: "Backup", Name: predecessor.Name, UID: predecessor.UID, TargetUID: gs.UID, NeedsUnquiesce: true}) + } + count++ + } + if value := gs.Annotations[restoreGuardAnnotation]; value != "" { + id, err := parseRestoreIdentity(value) + if err != nil { + return op, true, err + } + op.lifecycleIdentity = lifecycleIdentity{Kind: "Restore", Name: id.Name, UID: id.UID, TargetUID: gs.UID} + count++ + } + if token := gs.Annotations[wipeGuardAnnotation]; token != "" { + op.lifecycleIdentity = wipeLifecycleIdentity(gs, token) + count++ + } + if count > 1 { + return op, true, fmt.Errorf("overlapping legacy lifecycle owners require cleanup before new work") + } + return op, count != 0, nil +} + +func lifecycleOwnerExists(ctx context.Context, reader client.Reader, namespace string, id lifecycleIdentity) (bool, error) { + var obj client.Object + switch id.Kind { + case "Backup": + obj = &gameplanev1alpha1.Backup{} + case "Restore": + obj = &gameplanev1alpha1.Restore{} + case "Wipe", "Rollout": + // Only the GameServer reconciler may cancel/replace a wipe token after + // its workers drain. A removed request is not proof of worker absence. + return true, nil + default: + return false, fmt.Errorf("unknown lifecycle operation %q", id.Kind) + } + err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: id.Name}, obj) + if apierrors.IsNotFound(err) { + return false, nil + } + return err == nil && obj.GetUID() == id.UID, err +} + +// claimLifecycleOperation refreshes gs with the uncached reader and writes the +// shared owner plus compatibility guard in ONE resourceVersion-checked Update. +// Ownership is retained during recovery. A losing claimant performs no cleanup +// or worker creation. No timeout ever permits stealing from an existing owner. +func claimLifecycleOperation(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer, want lifecycleIdentity, quiescer AgentQuiescer) (bool, error) { + var live gameplanev1alpha1.GameServer + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &live); err != nil { + return false, err + } + if live.UID != want.TargetUID || !live.DeletionTimestamp.IsZero() { + return false, fmt.Errorf("lifecycle target changed identity or is being deleted") + } + if want.Kind == "Rollout" && live.Generation != want.Generation { + return false, errLifecycleGenerationChanged + } + *gs = live + op, occupied, err := lifecycleOwner(gs) + if err != nil { + return false, err + } + if occupied && !sameLifecycleOwner(op.lifecycleIdentity, want) { + exists, err := lifecycleOwnerExists(ctx, reader, gs.Namespace, op.lifecycleIdentity) + if err != nil || exists { + return false, err + } + op.Recovery = append(op.Recovery, op.lifecycleIdentity) + // A force-deleted restore may have left partial data. Preserve its + // suspension even when the next operation is a backup. + if op.Kind == "Restore" { + gs.Spec.Suspend = true + } + op.lifecycleIdentity = want + } else if occupied { + if gs.Annotations[lifecycleOperationAnnotation] != "" && op.Kind != "Wipe" && op.Kind != "Rollout" && op.Generation != want.Generation { + return false, errLifecycleGenerationChanged + } + if gs.Annotations[lifecycleOperationAnnotation] == "" { + op.lifecycleIdentity = want + } + } else { + if want.Kind != "Wipe" { + workers, err := wipeWorkers(ctx, reader, gs) + if err != nil || pendingWipe(gs) || workers.busy { + return false, err + } + } + // A stop command already sent may still save data and exit. Waiting + // here also lets pending restarts finish instead of freezing mid-stop. + if want.Kind == "Backup" || want.Kind == "Rollout" { + pending, err := backupLifecycleTransitionPending(ctx, reader, gs) + if err != nil || pending { + return false, err + } + } + busy, err := legacyLifecycleCopiesLive(ctx, c, reader, gs, want) + if err != nil || busy { + return false, err + } + op = lifecycleOperation{lifecycleIdentity: want} + } + before := gs.DeepCopy() + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + delete(gs.Annotations, backupGuardAnnotation) + delete(gs.Annotations, restoreGuardAnnotation) + delete(gs.Annotations, wipeGuardAnnotation) + switch want.Kind { + case "Backup": + gs.Annotations[backupGuardAnnotation] = backupIdentityJSON(want.Name, want.UID) + case "Restore": + gs.Annotations[restoreGuardAnnotation] = restoreIdentityJSON(want.Name, want.UID) + if !op.InitialStartup { + gs.Spec.Suspend = true + } + case "Wipe": + gs.Annotations[wipeGuardAnnotation] = want.Token + if pendingWipe(gs) { + gs.Spec.Suspend = true + } + } + // An acquired owner can be re-read many times while its worker runs. + // Avoid needless resourceVersion churn, but always CAS new admission. + if before.Annotations[lifecycleOperationAnnotation] != gs.Annotations[lifecycleOperationAnnotation] || before.Spec.Suspend != gs.Spec.Suspend { + if err := c.Update(ctx, gs); err != nil { + return false, err + } + } + return recoverLifecycleOperation(ctx, c, reader, gs, op, quiescer) +} + +// New code must also respect workers started before shared ownership existed, +// including quiesce=false copies which had no old backup guard at all. +func legacyLifecycleCopiesLive(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer, want lifecycleIdentity) (bool, error) { + var backups gameplanev1alpha1.BackupList + if err := reader.List(ctx, &backups, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + r := &BackupReconciler{Client: c, APIReader: reader, Scheme: c.Scheme()} + for i := range backups.Items { + b := &backups.Items[i] + if b.Spec.ServerRef.Name != gs.Name || (want.Kind == "Backup" && b.UID == want.UID) { + continue + } + if legacyBackupOwesUnquiesce(b) { + return true, nil + } + busy, err := r.backupWorkersLive(ctx, gs.Namespace, backupIdentity{Name: b.Name, UID: b.UID}, false) + if err != nil || busy { + return busy, err + } + } + return false, nil +} + +func lifecycleRecoveryNeedsStop(op lifecycleOperation) bool { + for _, predecessor := range op.Recovery { + if predecessor.Kind == "Restore" { + return true + } + } + return false +} + +func recoverLifecycleOperation(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer, op lifecycleOperation, quiescer AgentQuiescer) (bool, error) { + for _, predecessor := range op.Recovery { + var busy bool + var err error + switch predecessor.Kind { + case "Backup": + r := &BackupReconciler{Client: c, APIReader: reader, Scheme: c.Scheme()} + busy, err = r.backupWorkersLive(ctx, gs.Namespace, backupIdentity{Name: predecessor.Name, UID: predecessor.UID}, true) + case "Restore": + r := &RestoreReconciler{Client: c, APIReader: reader, Scheme: c.Scheme()} + busy, err = r.restoreWorkersLive(ctx, gs.Namespace, restoreIdentity{Name: predecessor.Name, UID: predecessor.UID}, true) + default: + return false, fmt.Errorf("unsupported orphan lifecycle owner %q", predecessor.Kind) + } + if err != nil || busy { + return false, err + } + } + if lifecycleRecoveryNeedsStop(op) { + r := &RestoreReconciler{Client: c, APIReader: reader, Scheme: c.Scheme()} + stopped, _, err := r.restoreTargetStopped(ctx, gs) + if err != nil || !stopped { + return false, err + } + } + for _, predecessor := range op.Recovery { + if !predecessor.NeedsUnquiesce { + continue + } + gone, err := backupQuiesceProcessGone(ctx, reader, gs.Namespace, gs.Name, predecessor.QuiescePodUID) + if err != nil { + return false, err + } + if gone { + continue + } + if !agentQuiesceAvailable(quiescer) { + return false, fmt.Errorf("cannot recover orphaned quiesce without an agent client") + } + if err := callBackupQuiesce(ctx, quiescer, gs.Namespace, gs.Name, predecessor.QuiescePodUID, true); err != nil { + return false, fmt.Errorf("recover orphaned lifecycle quiesce: %w", err) + } + } + if len(op.Recovery) > 0 { + op.Recovery = nil + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + delete(gs.Annotations, backupRecoveryLineageAnnotation) + if err := c.Update(ctx, gs); err != nil { + return false, err + } + } + return true, nil +} + +func backupLifecycleTransitionPending(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer) (bool, error) { + if tokenPending(gs, RestartRequestedAnnotation, RestartCompletedAnnotation) { + return true, nil + } + if gs.Annotations[stopRequestedAtAnnotation] == "" { + return false, nil + } + var ss appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return true, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&ss, gs) || ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation { + return true, nil + } + var pods corev1.PodList + if err := reader.List(ctx, &pods, client.InNamespace(gs.Namespace)); err != nil { + return true, err + } + for i := range pods.Items { + if liveRestorePod(&pods.Items[i]) && (pods.Items[i].Name == gs.Name+"-0" || podMountsRestoreTarget(&pods.Items[i], gs.Name)) { + return true, nil + } + } + return false, nil +} + +// Carry cleanup intent on the surviving arbitration object as well as Backup. +// A force-deleted Backup cannot hide a partial/ambiguous save-off RPC. +func (r *BackupReconciler) setLifecycleUnquiesceIntent(ctx context.Context, b *gameplanev1alpha1.Backup, needed bool) error { + gs, err := r.ownedBackupTarget(ctx, b) + if err != nil || gs == nil || gs.Annotations[lifecycleOperationAnnotation] == "" { + return err + } + op, err := parseLifecycleOperation(gs.Annotations[lifecycleOperationAnnotation]) + if err != nil { + return err + } + var podUID types.UID + if needed { + preparation, err := parseBackupPreparation(b) + if err != nil { + return err + } + if preparation != nil { + podUID = preparation.PodUID + } + } + if op.NeedsUnquiesce == needed && (!needed || op.QuiescePodUID == podUID) { + return nil + } + op.NeedsUnquiesce = needed + if needed { + op.QuiescePodUID = podUID + } + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + return r.Update(ctx, gs) +} + +// fenceBackupWorkload performs a checked write to the workload before a copy +// can start. A stale workload Update either precedes this fence (and is fully +// observed before admission) or conflicts with it. StatefulSet mutation also +// reads the live owner inside its callback and preserves the whole pod template. +func fenceBackupWorkload(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer) (bool, error) { + var ss appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + if apierrors.IsNotFound(err) { + return false, nil // wait for the workload reconciler to create a fenced set + } + return false, err + } + if !metav1.IsControlledBy(&ss, gs) { + return false, fmt.Errorf("backup StatefulSet is not controlled by the bound GameServer") + } + value := gs.Annotations[lifecycleOperationAnnotation] + if value == "" { + return false, fmt.Errorf("backup has no shared lifecycle owner") + } + if ss.Annotations[lifecycleOperationAnnotation] != value { + if ss.Annotations == nil { + ss.Annotations = map[string]string{} + } + ss.Annotations[lifecycleOperationAnnotation] = value + if err := c.Update(ctx, &ss); err != nil { + return false, err + } + } + if ss.Status.ObservedGeneration < ss.Generation || (ss.Status.CurrentRevision != "" && ss.Status.CurrentRevision != ss.Status.UpdateRevision) { + return false, nil + } + return true, nil +} + +// clearLifecycleOperation prepares the release; the caller removes its legacy +// guard and commits the GameServer Update only AFTER worker/agent cleanup and a +// durable completion decision. Release never changes the caller's power intent. +func clearLifecycleOperation(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer, want lifecycleIdentity) error { + value := gs.Annotations[lifecycleOperationAnnotation] + if value == "" { + return nil + } + op, err := parseLifecycleOperation(value) + if err != nil { + return err + } + if !sameLifecycleOwner(op.lifecycleIdentity, want) || len(op.Recovery) != 0 { + return fmt.Errorf("cannot release lifecycle ownership before recovery completes") + } + var ss appsv1.StatefulSet + err = reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil && metav1.IsControlledBy(&ss, gs) && ss.Annotations[lifecycleOperationAnnotation] == value { + delete(ss.Annotations, lifecycleOperationAnnotation) + if err := c.Update(ctx, &ss); err != nil { + return err + } + } + delete(gs.Annotations, lifecycleOperationAnnotation) + return nil +} diff --git a/operator/internal/controller/lifecycle_operation_envtest_test.go b/operator/internal/controller/lifecycle_operation_envtest_test.go new file mode 100644 index 000000000..7dc623a1a --- /dev/null +++ b/operator/internal/controller/lifecycle_operation_envtest_test.go @@ -0,0 +1,73 @@ +//go:build envtest + +package controller + +import ( + "sync" + "testing" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// A real apiserver arbitrates across controller types; no process-local mutex +// or informer timing can allow both claimants to create workers. +func TestLifecycleCASAcrossControllers(t *testing.T) { + ns := newNamespace(t) + seedGameServer(t, ns, "target") + b := buildBackup(ns, "backup", "target", "repo") + b.Spec.Quiesce = false + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: ns}, Spec: gameplanev1alpha1.RestoreSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: "target"}, BackupRef: gameplanev1alpha1.LocalObjectRef{Name: "source"}}} + if err := k8sClient.Create(t.Context(), b); err != nil { + t.Fatal(err) + } + if err := k8sClient.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + var gs gameplanev1alpha1.GameServer + if err := k8sClient.Get(t.Context(), client.ObjectKey{Namespace: ns, Name: "target"}, &gs); err != nil { + t.Fatal(err) + } + type outcome struct { + claimed bool + err error + } + results := make(chan outcome, 2) + start := make(chan struct{}) + var ready sync.WaitGroup + ready.Add(2) + for _, want := range []lifecycleIdentity{backupLifecycleIdentity(&gs, b, false), restoreLifecycleIdentity(&gs, rs)} { + go func() { + target := gs.DeepCopy() + ready.Done() + <-start + claimed, err := claimLifecycleOperation(t.Context(), k8sClient, k8sClient, target, want, nil) + results <- outcome{claimed: claimed, err: err} + }() + } + ready.Wait() + close(start) + winners := 0 + for range 2 { + result := <-results + if result.err != nil && !apierrors.IsConflict(result.err) { + t.Fatalf("claim: %v", result.err) + } + if result.claimed { + winners++ + } + } + if winners != 1 { + t.Fatalf("exclusive admission had %d winners", winners) + } + if err := k8sClient.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + op, held, err := lifecycleOwner(&gs) + if err != nil || !held || op.TargetUID != gs.UID { + t.Fatalf("owner was not durable: held=%t owner=%+v err=%v", held, op, err) + } +} diff --git a/operator/internal/controller/lifecycle_operation_test.go b/operator/internal/controller/lifecycle_operation_test.go new file mode 100644 index 000000000..b41e61cc7 --- /dev/null +++ b/operator/internal/controller/lifecycle_operation_test.go @@ -0,0 +1,387 @@ +package controller + +import ( + "context" + "fmt" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestLifecycleAllBackupVariantsExcludeRestoreAndWipe(t *testing.T) { + for _, quiesce := range []bool{false, true} { + for _, strategy := range []string{"restic-snapshot", "volume-snapshot"} { + t.Run(fmt.Sprintf("%s/quiesce=%t", strategy, quiesce), func(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce, b.Spec.Strategy = quiesce, strategy + r, _ := guardFixture(t, b) + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim backup: claimed=%t err=%v", claimed, err) + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns", UID: "restore-uid"}} + if err := r.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + rr := &RestoreReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if claimed, err := rr.claimRestoreTarget(t.Context(), rs, &gs); err != nil || claimed { + t.Fatalf("restore overlapped backup: claimed=%t err=%v", claimed, err) + } + gs.Annotations[WipeRequestedAnnotation] = "wipe" + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if changed, err := gr.ensureWipeGuard(t.Context(), &gs); err != nil || changed || gs.Annotations[wipeGuardAnnotation] != "" { + t.Fatalf("wipe overlapped backup: changed=%t err=%v", changed, err) + } + }) + } + } +} + +func TestLifecycleRestoreExcludesBackupCreatedAfterDrainCheck(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = false + r, _ := guardFixture(t, b) + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns", UID: "restore-uid"}} + if err := r.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + rr := &RestoreReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if claimed, err := rr.claimRestoreTarget(t.Context(), rs, &gs); err != nil || !claimed { + t.Fatalf("claim restore: claimed=%t err=%v", claimed, err) + } + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || claimed { + t.Fatalf("unquiesced backup entered restore: claimed=%t err=%v", claimed, err) + } +} + +func TestLifecycleUnquiescedCleanupRetainsOwnerUntilWorkerDrains(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = false + r, q := guardFixture(t, b) + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orphan", Namespace: "ns", Labels: map[string]string{backupUIDLabel: string(b.UID)}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + b.Status.Phase = gameplanev1alpha1.BackupPhaseFailed + if err := r.Status().Update(t.Context(), b); err != nil { + t.Fatal(err) + } + if res, err := r.runUnquiesce(t.Context(), b); err != nil || res.RequeueAfter == 0 { + t.Fatalf("cleanup released live worker: result=%+v err=%v", res, err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if _, err := r.runUnquiesce(t.Context(), b); err != nil { + t.Fatal(err) + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || q.unquiesced != 0 { + t.Fatalf("unquiesced cleanup: owner=%q agent calls=%d", gs.Annotations[lifecycleOperationAnnotation], q.unquiesced) + } +} + +func TestLifecycleBackupFencePreservesReplicasAndTemplate(t *testing.T) { + s := wipeScheme(t) + gs := wipeGameServer(false, "") + ss := wipeStoppedSet(gs) + one := int32(1) + ss.Spec.Replicas = &one + ss.Spec.Template.Spec.Containers = []corev1.Container{{Name: "game", Image: "before"}} + b := guardBackup("copy") + b.Spec.ServerRef.Name = gs.Name + gs.Annotations[backupGuardAnnotation] = backupIdentityJSON(b.Name, b.UID) + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: backupLifecycleIdentity(gs, b, false)}) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, ss, b).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: s} + stale := gs.DeepCopy() + stale.Annotations = nil + stale.Spec.Suspend = true + if err := r.reconcileStatefulSet(t.Context(), stale, &gameplanev1alpha1.GameTemplate{}, nil, &materializedConfig{}, 0); err != nil { + t.Fatal(err) + } + var current appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(ss), ¤t); err != nil { + t.Fatal(err) + } + if *current.Spec.Replicas != 1 || current.Spec.Template.Spec.Containers[0].Image != "before" { + t.Fatal("stale reconcile stopped or rolled protected source") + } + if replicas, delay, err := r.desiredReplicas(t.Context(), stale, &gameplanev1alpha1.GameTemplate{}, idleAsleep); err != nil || replicas != 1 || delay == 0 { + t.Fatalf("stop/idle bypassed live owner: replicas=%d delay=%s err=%v", replicas, delay, err) + } +} + +func TestLifecycleConcurrentClaimsUseResourceVersion(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = false + r, _ := guardFixture(t, b) + intercepted := false + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if gs, ok := obj.(*gameplanev1alpha1.GameServer); ok && !intercepted && gs.Annotations[lifecycleOperationAnnotation] != "" { + intercepted = true + var current gameplanev1alpha1.GameServer + if err := c.Get(ctx, client.ObjectKeyFromObject(gs), ¤t); err != nil { + return err + } + if current.Annotations == nil { + current.Annotations = map[string]string{} + } + current.Annotations[WipeRequestedAnnotation] = "winner" + current.Annotations[wipeGuardAnnotation] = "winner" + returnAfter := c.Update(ctx, ¤t) + if returnAfter != nil { + return returnAfter + } + } + return c.Update(ctx, obj, opts...) + }, + }) + if claimed, err := r.claimBackupTarget(t.Context(), b); claimed || !apierrors.IsConflict(err) { + t.Fatalf("lost CAS started backup: claimed=%t err=%v", claimed, err) + } + var jobs batchv1.JobList + if err := r.List(t.Context(), &jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("losing claimant started workers: jobs=%d err=%v", len(jobs.Items), err) + } +} + +func TestLifecycleIdentityRejectsReplacementAndMalformedOwner(t *testing.T) { + for _, owner := range []string{"broken", lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: lifecycleIdentity{Kind: "Backup", Name: "other", UID: types.UID("other"), TargetUID: "replaced"}})} { + t.Run(owner, func(t *testing.T) { + b := guardBackup("copy") + r, _ := guardFixture(t, b) + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{lifecycleOperationAnnotation: owner} + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + if claimed, err := r.claimBackupTarget(t.Context(), b); claimed || err == nil { + t.Fatalf("invalid owner admitted backup: claimed=%t err=%v", claimed, err) + } + }) + } +} + +func TestLifecycleOrphanBackupRecoveryFencesRestoreUntilDrain(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = false + r, _ := guardFixture(t, b) + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orphan", Namespace: "ns", Labels: map[string]string{backupUIDLabel: string(b.UID)}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + forceDeleteGuardBackup(t, r, b.Name) + rs := lifecycleRestore("restore", "restore-uid") + if err := r.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "server-uid"}} + rr := &RestoreReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if claimed, err := rr.claimRestoreTarget(t.Context(), rs, gs); err != nil || claimed { + t.Fatalf("restore admitted before orphan drained: %t %v", claimed, err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if replicas, _, err := gr.desiredReplicas(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, idleAwake); err != nil || replicas != 1 { + t.Fatalf("recovery stopped source while copy worker alive: replicas=%d err=%v", replicas, err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if claimed, err := rr.claimRestoreTarget(t.Context(), rs, gs); err != nil || !claimed { + t.Fatalf("restore did not recover after drain: %t %v", claimed, err) + } +} + +func TestLifecycleIdleTransitionConflictsWithNewBackupOwner(t *testing.T) { + b := guardBackup("copy") + r, _ := guardFixture(t, b) + var stale gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &stale); err != nil { + t.Fatal(err) + } + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if err := gr.applyIdleTransition(t.Context(), &stale, idleOutcome{sleep: true}, time.Now()); !apierrors.IsConflict(err) { + t.Fatalf("stale idle transition bypassed ownership CAS: %v", err) + } + var current gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&stale), ¤t); err != nil { + t.Fatal(err) + } + if current.Annotations[IdleAsleepSinceAnnotation] != "" { + t.Fatal("backup acquisition allowed stale sleep") + } +} + +func TestLifecycleBackupFenceConflictsWithStaleWorkloadWrite(t *testing.T) { + b := guardBackup("copy") + r, _ := guardFixture(t, b) + var stale appsv1.StatefulSet + key := client.ObjectKey{Namespace: "ns", Name: "server"} + if err := r.Get(t.Context(), key, &stale); err != nil { + t.Fatal(err) + } + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + gs, err := r.ownedBackupTarget(t.Context(), b) + if err != nil || gs == nil { + t.Fatalf("owned target: %v", err) + } + if ready, err := fenceBackupWorkload(t.Context(), r.Client, r.Client, gs); err != nil || !ready { + t.Fatalf("fence: %t %v", ready, err) + } + zero := int32(0) + stale.Spec.Replicas = &zero + stale.Spec.Template.Spec.Containers = []corev1.Container{{Name: "game", Image: "new-revision"}} + if err := r.Update(t.Context(), &stale); !apierrors.IsConflict(err) { + t.Fatalf("stale workload write defeated fence: %v", err) + } +} + +func TestLifecycleCSIInitialStartupExcludesBackupAndRollout(t *testing.T) { + r, gs, rs := restoreLifecycleFixture(t) + gs.Spec.Suspend = false + gs.Annotations = map[string]string{restoreGuardAnnotation: restoreIdentityJSON(rs.Name, rs.UID)} + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + want := restoreLifecycleIdentity(gs, rs) + want.InitialStartup = true + if ready, err := claimLifecycleOperation(t.Context(), r.Client, r.apiReader(), gs, want, nil); err != nil || !ready { + t.Fatalf("claim initial startup: %t %v", ready, err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + ss.Spec.Template.Spec.Containers = []corev1.Container{{Name: "game", Image: "snapshot-version"}} + if err := r.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.apiReader(), Scheme: r.Scheme} + replicas, _, err := gr.desiredReplicas(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, idleAwake) + if err != nil || replicas != 1 { + t.Fatalf("CSI initial startup blocked first consumer: replicas=%d err=%v", replicas, err) + } + if err := gr.reconcileStatefulSet(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, nil, &materializedConfig{}, replicas); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + if *ss.Spec.Replicas != 1 || ss.Spec.Template.Spec.Containers[0].Image != "snapshot-version" { + t.Fatal("CSI restore either blocked startup or adopted a different template") + } + b := guardBackup("new-copy") + if err := r.Create(t.Context(), b); err != nil { + t.Fatal(err) + } + br := &BackupReconciler{Client: r.Client, APIReader: r.apiReader(), Scheme: r.Scheme} + if claimed, err := br.claimBackupTarget(t.Context(), b); err != nil || claimed { + t.Fatalf("backup entered CSI restore startup: %t %v", claimed, err) + } +} + +func TestLifecycleIndependentServersRemainConcurrent(t *testing.T) { + first, second := guardBackup("first"), guardBackup("second") + second.Spec.ServerRef.Name = "other" + r, _ := guardFixture(t, first, second) + other := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: "ns", UID: "other-uid"}} + if err := r.Create(t.Context(), other); err != nil { + t.Fatal(err) + } + for _, b := range []*gameplanev1alpha1.Backup{first, second} { + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("independent target %s blocked: %t %v", b.Spec.ServerRef.Name, claimed, err) + } + } +} + +type lifecycleStopper struct{ calls int } + +func (s *lifecycleStopper) Enabled() bool { return true } +func (s *lifecycleStopper) Stop(context.Context, string, string) error { + s.calls++ + return nil +} + +func TestLifecycleBackupDefersRestartAndStopRPC(t *testing.T) { + for _, quiesce := range []bool{false, true} { + t.Run(fmt.Sprintf("quiesce=%t", quiesce), func(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = quiesce + r, _ := guardFixture(t, b) + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + var gs gameplanev1alpha1.GameServer + key := client.ObjectKey{Namespace: "ns", Name: "server"} + if err := r.Get(t.Context(), key, &gs); err != nil { + t.Fatal(err) + } + gs.Annotations[RestartRequestedAnnotation] = "restart-after-copy" + gs.Spec.Suspend = true + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), key, &ss); err != nil { + t.Fatal(err) + } + ss.Status.Replicas, ss.Status.ReadyReplicas = 1, 1 + if err := r.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + stopper := &lifecycleStopper{} + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme, AgentClient: stopper} + tmpl := &gameplanev1alpha1.GameTemplate{Spec: gameplanev1alpha1.GameTemplateSpec{RCON: &gameplanev1alpha1.RCONSpec{Protocol: "source"}, Capabilities: &gameplanev1alpha1.CapabilitiesSpec{Lifecycle: &gameplanev1alpha1.LifecycleSpec{Stop: []string{"stop"}}}}} + if replicas, poll, err := gr.desiredReplicas(t.Context(), &gs, tmpl, idleAsleep); err != nil || replicas != 1 || poll == 0 { + t.Fatalf("restart changed protected source: replicas=%d poll=%s err=%v", replicas, poll, err) + } + if err := r.Get(t.Context(), key, &gs); err != nil { + t.Fatal(err) + } + if stopper.calls != 0 || gs.Annotations[RestartCompletedAnnotation] != "" || gs.Annotations[stopRequestedAtAnnotation] != "" { + t.Fatal("restart issued or acknowledged a stop during the copy") + } + }) + } +} diff --git a/operator/internal/controller/manager_scope_envtest_test.go b/operator/internal/controller/manager_scope_envtest_test.go new file mode 100644 index 000000000..4a4330f69 --- /dev/null +++ b/operator/internal/controller/manager_scope_envtest_test.go @@ -0,0 +1,39 @@ +//go:build envtest + +package controller + +import ( + "testing" + + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// The envtest API has no namespace garbage collector. Each new manager must +// ignore earlier fixtures while continuing to read cluster-scoped templates. +func TestManagerCacheKeepsNamespaceIsolationAndClusterScope(t *testing.T) { + ns, other := newNamespace(t), newNamespace(t) + tmpl := buildGameTemplate(uniqueName("cache-scope")) + if err := k8sClient.Create(t.Context(), tmpl); err != nil { + t.Fatal(err) + } + deleteCleanup(t, tmpl) + for _, namespace := range []string{ns, other} { + if err := k8sClient.Create(t.Context(), buildGameServer(namespace, "server", tmpl.Name)); err != nil { + t.Fatal(err) + } + } + cached := startMgr(t, ns) + var templates gameplanev1alpha1.GameTemplate + if err := cached.Get(t.Context(), client.ObjectKeyFromObject(tmpl), &templates); err != nil { + t.Fatalf("cluster-scoped template unavailable: %v", err) + } + var servers gameplanev1alpha1.GameServerList + if err := cached.List(t.Context(), &servers); err != nil { + t.Fatal(err) + } + if len(servers.Items) != 1 || servers.Items[0].Namespace != ns { + t.Fatalf("manager cache included resources outside its test namespace: %+v", servers.Items) + } +} diff --git a/operator/internal/controller/networkcapture_controller.go b/operator/internal/controller/networkcapture_controller.go index 4b1f46447..a7ada7fa4 100644 --- a/operator/internal/controller/networkcapture_controller.go +++ b/operator/internal/controller/networkcapture_controller.go @@ -127,9 +127,15 @@ const captureFileCleanupBudget = 5 * time.Minute // over mTLS through the -agent Service, and monitors completion. type NetworkCaptureReconciler struct { client.Client + APIReader client.Reader Scheme *runtime.Scheme SidecarClient SidecarCaptureClient CaptureEnabled bool + RestrictedWorkloads bool + DiagnosticsNamespace string + DiagnosticsStorageClass string + AgentCASecretName string + AgentCASecretNamespace string CaptureSidecarImage string CaptureDefaultMaxDurationSeconds int64 CaptureDefaultMaxSizeBytes int64 @@ -166,6 +172,11 @@ func (r *NetworkCaptureReconciler) reconcile(ctx context.Context, req ctrl.Reque if err := r.Get(ctx, req.NamespacedName, &nc); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + if use, err := r.useNodeCapture(ctx, &nc); err != nil { + return ctrl.Result{}, err + } else if use { + return r.reconcileNodeCapture(ctx, &nc) + } // A user asked for this capture to stop (the API's StopNetworkCapture // set stopRequestedAnnotation). Stop the sidecar first, then complete diff --git a/operator/internal/controller/networkcapture_envtest_test.go b/operator/internal/controller/networkcapture_envtest_test.go index f7cf85007..5c5d2e12d 100644 --- a/operator/internal/controller/networkcapture_envtest_test.go +++ b/operator/internal/controller/networkcapture_envtest_test.go @@ -510,12 +510,19 @@ func TestNetworkCapture_UserStopCallsSidecar(t *testing.T) { // Emulate the API's StopNetworkCapture: patch status directly to // Completed with its exact message, bypassing the reconciler. - nc := getNetworkCapture(t, ns, "cap-stop-001") - now := metav1.NewTime(time.Now().UTC()) - nc.Status.Phase = gameplanev1alpha1.CapturePhaseCompleted - nc.Status.CompletionTime = &now - nc.Status.Message = "stopped by user request" - if err := k8sClient.Status().Update(context.Background(), nc); err != nil { + // The active reconciler may refresh capture progress between Get and Update. + ctx := t.Context() + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + var nc gameplanev1alpha1.NetworkCapture + if err := k8sClient.Get(ctx, types.NamespacedName{Namespace: ns, Name: "cap-stop-001"}, &nc); err != nil { + return err + } + now := metav1.NewTime(time.Now().UTC()) + nc.Status.Phase = gameplanev1alpha1.CapturePhaseCompleted + nc.Status.CompletionTime = &now + nc.Status.Message = "stopped by user request" + return k8sClient.Status().Update(ctx, &nc) + }); err != nil { t.Fatalf("patch capture to user-stopped: %v", err) } diff --git a/operator/internal/controller/networkcapture_filter.go b/operator/internal/controller/networkcapture_filter.go index c3d0bfd49..8252a16ed 100644 --- a/operator/internal/controller/networkcapture_filter.go +++ b/operator/internal/controller/networkcapture_filter.go @@ -7,7 +7,6 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -50,14 +49,14 @@ func (r *NetworkCaptureReconciler) captureFilter( return nc.Spec.Filter, "", nil } name := gs.Spec.TemplateRef.Name - var tmpl gameplanev1alpha1.GameTemplate - if err := r.Get(ctx, types.NamespacedName{Name: name}, &tmpl); err != nil { + tmpl, err := effectiveGameTemplate(ctx, r.Client, gs) + if err != nil { if apierrors.IsNotFound(err) { return nil, fmt.Sprintf("gametemplate %s not found; cannot build the default capture filter", name), nil } return nil, "", fmt.Errorf("get gametemplate %s for the default capture filter: %w", name, err) } - f := defaultCaptureFilter(&tmpl) + f := defaultCaptureFilter(tmpl) if f == "" { return nil, fmt.Sprintf("gametemplate %s advertises no ports; supply a capture filter", name), nil } diff --git a/operator/internal/controller/networkcapture_node.go b/operator/internal/controller/networkcapture_node.go new file mode 100644 index 000000000..ab6231797 --- /dev/null +++ b/operator/internal/controller/networkcapture_node.go @@ -0,0 +1,570 @@ +package controller + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const nodeCaptureNamespaceAnnotation = "gameplane.local/capture-node-namespace" +const nodeCaptureNodeAnnotation = "gameplane.local/capture-node" +const nodeCaptureContainerAnnotation = "gameplane.local/capture-container-id" +const nodeCaptureFinishedAnnotation = "gameplane.local/capture-worker-finished" +const nodeCaptureFinalizer = "gameplane.local/node-capture-cleanup" +const nodeCaptureUIDLabel = "gameplane.local/capture-uid" +const nodeCaptureComponentLabel = "gameplane.local/capture-component" + +func nodeCaptureName(nc *gameplanev1alpha1.NetworkCapture) string { return "capture-" + string(nc.UID) } +func (r *NetworkCaptureReconciler) nodeReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +func (r *NetworkCaptureReconciler) useNodeCapture(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture) (bool, error) { + if nc.Annotations[nodeCaptureNamespaceAnnotation] != "" || controllerutil.ContainsFinalizer(nc, nodeCaptureFinalizer) { + return true, nil + } + // A profile/opt-in change must not reinterpret an already-started legacy + // capture as a new node worker or abandon its stop/retention protocol. + if nc.Annotations[capturePodUIDAnnotation] != "" || nc.Status.Phase == gameplanev1alpha1.CapturePhaseRunning || isTerminalCapturePhase(nc.Status.Phase) { + return false, nil + } + if r.RestrictedWorkloads { + return true, nil + } + var gs gameplanev1alpha1.GameServer + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: nc.Namespace, Name: nc.Spec.ServerRef.Name}, &gs); err != nil { + return false, client.IgnoreNotFound(err) + } + if gs.Spec.Capture != nil && gs.Spec.Capture.NodeDiagnostic { + return true, nil + } + var pod corev1.Pod + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: nc.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + return false, client.IgnoreNotFound(err) + } + return pod.Annotations[workloadSecurityLabel] == "restricted", nil +} + +func nodeGameContainerID(pod *corev1.Pod) (string, error) { + if pod.Spec.NodeName == "" || pod.UID == "" || pod.Spec.HostNetwork || pod.DeletionTimestamp != nil { + return "", errors.New("node capture requires a live scheduled Pod with its own network namespace") + } + for _, c := range pod.Status.ContainerStatuses { + if c.Name == "game" && c.State.Running != nil { + id := strings.TrimPrefix(c.ContainerID, "containerd://") + decoded, err := hex.DecodeString(id) + if err != nil || len(decoded) != 32 || !strings.HasPrefix(c.ContainerID, "containerd://") { + return "", errors.New("node capture currently supports Linux containerd targets only") + } + return c.ContainerID, nil + } + } + return "", errors.New("game container is not running") +} + +func nodeAffinity(name string) *corev1.Affinity { + return &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{name}}}}}}}} +} + +func nodeWorkerPodSpec(nc *gameplanev1alpha1.NetworkCapture, gs *gameplanev1alpha1.GameServer, pod *corev1.Pod, image, pvc, filter string, duration, size int64) (corev1.PodSpec, error) { + id, err := nodeGameContainerID(pod) + if err != nil { + return corev1.PodSpec{}, err + } + if duration < 1 || duration > 3600 || size < 1 || size > 900*1024*1024 || filter == "" { + return corev1.PodSpec{}, errors.New("node capture requires a filter, duration <= 3600s and size <= 900Mi") + } + raw, err := json.Marshal(map[string]any{"serverUID": string(gs.UID), "captureUID": string(nc.UID), "captureID": nc.Name, "podUID": string(pod.UID), "containerID": id, "node": pod.Spec.NodeName, "filter": filter, "durationSeconds": duration, "maxSizeBytes": size}) + if err != nil { + return corev1.PodSpec{}, err + } + return corev1.PodSpec{ + HostPID: true, RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: ptrTo(false), Affinity: nodeAffinity(pod.Spec.NodeName), NodeSelector: map[string]string{"kubernetes.io/os": "linux"}, + Tolerations: append([]corev1.Toleration(nil), pod.Spec.Tolerations...), + TerminationGracePeriodSeconds: ptrTo(int64(10)), + SecurityContext: &corev1.PodSecurityContext{RunAsUser: ptrTo(int64(0)), RunAsGroup: ptrTo(int64(65532)), FSGroup: ptrTo(int64(65532))}, + Volumes: []corev1.Volume{{Name: "captures", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc}}}}, + Containers: []corev1.Container{{Name: "capture", Image: image, Command: []string{"/capture-tool"}, Args: []string{"--node-worker"}, Env: []corev1.EnvVar{{Name: "NODE_CAPTURE_CONFIG", Value: string(raw)}}, VolumeMounts: []corev1.VolumeMount{{Name: "captures", MountPath: "/tmp/captures"}}, + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("50m"), corev1.ResourceMemory: resource.MustParse("64Mi")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("256Mi")}}, + SecurityContext: &corev1.SecurityContext{RunAsUser: ptrTo(int64(0)), ReadOnlyRootFilesystem: ptrTo(true), AllowPrivilegeEscalation: ptrTo(false), Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}, Add: []corev1.Capability{"SYS_ADMIN", "SYS_PTRACE", "NET_RAW"}}, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeUnconfined}}, + }}, + }, nil +} + +// Each capture gets a same-namespace anchor. Cross-namespace ownerReferences +// cannot link diagnostics to the NetworkCapture; its finalizer deletes this +// anchor in the recorded namespace and waits for foreground GC to finish. +func (r *NetworkCaptureReconciler) createNodeObject(ctx context.Context, obj client.Object, anchor *corev1.ConfigMap, nc *gameplanev1alpha1.NetworkCapture) error { + obj.SetNamespace(anchor.Namespace) + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels[nodeCaptureUIDLabel] = string(nc.UID) + obj.SetLabels(labels) + if err := controllerutil.SetControllerReference(anchor, obj, r.Scheme); err != nil { + return err + } + if err := r.Create(ctx, obj); err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } + live, ok := obj.DeepCopyObject().(client.Object) + if !ok { + return errors.New("diagnostic resource is not a Kubernetes object") + } + if err := r.nodeReader().Get(ctx, client.ObjectKeyFromObject(obj), live); err != nil { + return err + } + owner := metav1.GetControllerOf(live) + if owner == nil || owner.UID != anchor.UID || live.GetLabels()[nodeCaptureUIDLabel] != string(nc.UID) { + return errors.New("diagnostic resource identity collision") + } + } + return nil +} + +func (r *NetworkCaptureReconciler) ensureNodeArtifacts(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture, gs *gameplanev1alpha1.GameServer, pod *corev1.Pod) (*corev1.ConfigMap, error) { + ns, name := nc.Annotations[nodeCaptureNamespaceAnnotation], nodeCaptureName(nc) + anchor := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{nodeCaptureUIDLabel: string(nc.UID)}}} + if err := r.Create(ctx, anchor); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + if err := r.nodeReader().Get(ctx, client.ObjectKeyFromObject(anchor), anchor); err != nil { + return nil, err + } + if anchor.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return nil, errors.New("diagnostic anchor identity collision") + } + } + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}}}} + if r.DiagnosticsStorageClass != "" { + pvc.Spec.StorageClassName = &r.DiagnosticsStorageClass + } + if err := r.createNodeObject(ctx, pvc, anchor, nc); err != nil { + return nil, err + } + if err := r.ensureNodeCertificate(ctx, anchor, nc); err != nil { + return nil, err + } + image := r.CaptureSidecarImage + if image == "" { + image = DefaultCaptureSidecarImage + } + files := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name + "-files", Labels: map[string]string{nodeCaptureComponentLabel: "artifacts"}}, Spec: corev1.PodSpec{ + AutomountServiceAccountToken: ptrTo(false), Affinity: nodeAffinity(pod.Spec.NodeName), + Tolerations: append([]corev1.Toleration(nil), pod.Spec.Tolerations...), + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: ptrTo(true), RunAsUser: ptrTo(int64(65532)), RunAsGroup: ptrTo(int64(65532)), FSGroup: ptrTo(int64(65532)), SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + Volumes: []corev1.Volume{{Name: "captures", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: name}}}, {Name: "tls", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: name + "-tls"}}}}, + Containers: []corev1.Container{{Name: "artifacts", Image: image, Command: []string{"/capture-tool"}, Args: []string{"--artifacts-only"}, Env: []corev1.EnvVar{{Name: "GAMEPLANE_SERVER_UID", Value: string(gs.UID)}}, Ports: []corev1.ContainerPort{{Name: "capture", ContainerPort: 9091}}, VolumeMounts: []corev1.VolumeMount{{Name: "captures", MountPath: "/tmp/captures"}, {Name: "tls", MountPath: "/etc/tls", ReadOnly: true}}, + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("10m"), corev1.ResourceMemory: resource.MustParse("32Mi")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("128Mi")}}, + SecurityContext: &corev1.SecurityContext{RunAsNonRoot: ptrTo(true), ReadOnlyRootFilesystem: ptrTo(true), AllowPrivilegeEscalation: ptrTo(false), Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + }}, + }} + if err := r.createNodeObject(ctx, files, anchor, nc); err != nil { + return nil, err + } + svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: name + "-agent"}, Spec: corev1.ServiceSpec{Selector: map[string]string{nodeCaptureUIDLabel: string(nc.UID), nodeCaptureComponentLabel: "artifacts"}, Ports: []corev1.ServicePort{{Name: "capture", Port: 9091, TargetPort: intstr.FromInt32(9091)}}}} + if err := r.createNodeObject(ctx, svc, anchor, nc); err != nil { + return nil, err + } + return anchor, nil +} + +func (r *NetworkCaptureReconciler) ensureNodeCertificate(ctx context.Context, anchor *corev1.ConfigMap, nc *gameplanev1alpha1.NetworkCapture) error { + var ca corev1.Secret + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: r.AgentCASecretNamespace, Name: r.AgentCASecretName}, &ca); err != nil { + return err + } + fingerprint := caCertFingerprint(ca.Data["ca.crt"]) + key := types.NamespacedName{Namespace: anchor.Namespace, Name: anchor.Name + "-tls"} + var existing corev1.Secret + err := r.nodeReader().Get(ctx, key, &existing) + if err == nil { + owner := metav1.GetControllerOf(&existing) + if owner == nil || owner.UID != anchor.UID { + return errors.New("diagnostic certificate identity collision") + } + if existing.Annotations["gameplane.local/ca-fingerprint"] == fingerprint { + return nil + } + } else if !apierrors.IsNotFound(err) { + return err + } + cert, keyCA, err := parseAgentCA(ca.Data["ca.crt"], ca.Data["ca.key"]) + if err != nil { + return err + } + fake := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: anchor.Name, Namespace: anchor.Namespace}} + crt, private, err := signAgentServerCert(cert, keyCA, fake) + if err != nil { + return err + } + sec := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace, Annotations: map[string]string{"gameplane.local/ca-fingerprint": fingerprint}}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{"tls.crt": crt, "tls.key": private, "ca.crt": ca.Data["ca.crt"]}} + if existing.UID != "" { + existing.Data = sec.Data + existing.Annotations = sec.Annotations + return r.Update(ctx, &existing) + } + return r.createNodeObject(ctx, sec, anchor, nc) +} + +func (r *NetworkCaptureReconciler) deleteNodeWorkers(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture) (bool, error) { + ns := nc.Annotations[nodeCaptureNamespaceAnnotation] + if ns == "" { + return true, nil + } + var job batchv1.Job + err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: nodeCaptureName(nc) + "-worker"}, &job) + if err == nil { + if job.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return false, errors.New("diagnostic worker identity collision") + } + if job.DeletionTimestamp == nil { + if err := r.Delete(ctx, &job, client.PropagationPolicy(metav1.DeletePropagationForeground), client.Preconditions{UID: &job.UID}); err != nil && !apierrors.IsNotFound(err) { + return false, err + } + } + } else if !apierrors.IsNotFound(err) { + return false, err + } + var pods corev1.PodList + if err := r.nodeReader().List(ctx, &pods, client.InNamespace(ns), client.MatchingLabels{nodeCaptureUIDLabel: string(nc.UID), nodeCaptureComponentLabel: "worker"}); err != nil { + return false, err + } + // Independently drain orphan worker Pods too, including a Job deleted by + // another actor with orphan propagation. Never free a capture lock early. + for i := range pods.Items { + p := &pods.Items[i] + if p.DeletionTimestamp == nil { + if err := r.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) { + return false, err + } + } + } + return apierrors.IsNotFound(err) && len(pods.Items) == 0, nil +} + +func (r *NetworkCaptureReconciler) cleanupNodeCapture(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture) (bool, error) { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil || !drained { + return false, err + } + ns := nc.Annotations[nodeCaptureNamespaceAnnotation] + if ns == "" { + return true, nil + } + var anchor corev1.ConfigMap + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: nodeCaptureName(nc)}, &anchor); err != nil { + return apierrors.IsNotFound(err), client.IgnoreNotFound(err) + } + if anchor.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return false, errors.New("diagnostic anchor identity collision") + } + if anchor.DeletionTimestamp == nil { + if err := r.Delete(ctx, &anchor, client.PropagationPolicy(metav1.DeletePropagationForeground), client.Preconditions{UID: &anchor.UID}); err != nil && !apierrors.IsNotFound(err) { + return false, err + } + } + return false, nil +} + +func (r *NetworkCaptureReconciler) nodeFail(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture, message string) (ctrl.Result, error) { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return r.fail(ctx, nc, message) +} + +func (r *NetworkCaptureReconciler) reconcileNodeCapture(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture) (ctrl.Result, error) { + if nc.DeletionTimestamp != nil { + clean, err := r.cleanupNodeCapture(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !clean { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + if err := r.releaseNodeCaptureLock(ctx, nc, ptrTo(metav1.Now())); err != nil { + return ctrl.Result{}, err + } + controllerutil.RemoveFinalizer(nc, nodeCaptureFinalizer) + return ctrl.Result{}, r.Update(ctx, nc) + } + if isTerminalCapturePhase(nc.Status.Phase) { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + if nc.Status.CompletionTime == nil { + nc.Status.CompletionTime = ptrTo(metav1.Now()) + return ctrl.Result{RequeueAfter: retentionPollInterval}, r.Status().Update(ctx, nc) + } + ttl := effectiveRetentionSeconds(nc.Spec.TTLSecondsAfterFinished, r.CaptureDefaultRetentionSeconds, r.CaptureMaxRetentionSeconds) + if time.Now().Before(nc.Status.CompletionTime.Add(time.Duration(ttl) * time.Second)) { + // Retained artifacts need certificate rotation even after the target + // game Pod has gone away. No target or worker is recreated here. + if ns := nc.Annotations[nodeCaptureNamespaceAnnotation]; ns != "" { + var anchor corev1.ConfigMap + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: nodeCaptureName(nc)}, &anchor); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } else if anchor.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return ctrl.Result{}, errors.New("diagnostic anchor identity collision") + } else if err := r.ensureNodeCertificate(ctx, &anchor, nc); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: retentionPollInterval}, nil + } + return ctrl.Result{}, r.Delete(ctx, nc, client.Preconditions{UID: &nc.UID}) + } + var gs gameplanev1alpha1.GameServer + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: nc.Namespace, Name: nc.Spec.ServerRef.Name}, &gs); err != nil { + if apierrors.IsNotFound(err) { + return r.nodeFail(ctx, nc, "target GameServer no longer exists") + } + return ctrl.Result{}, err + } + if owner := metav1.GetControllerOf(nc); owner != nil && owner.UID != gs.UID { + return r.nodeFail(ctx, nc, "target GameServer identity changed") + } + if err := r.ensureOwnerReference(ctx, nc, &gs); err != nil { + return ctrl.Result{}, err + } + if gs.DeletionTimestamp != nil { + return r.nodeFail(ctx, nc, "target GameServer is terminating") + } + if !r.CaptureEnabled || r.DiagnosticsNamespace == "" || gs.Spec.Capture == nil || !gs.Spec.Capture.Enabled || !gs.Spec.Capture.NodeDiagnostic { + return r.nodeFail(ctx, nc, "node diagnostics require cluster deployment opt-in and administrator spec.capture.nodeDiagnostic opt-in on this server") + } + if _, stop := nc.Annotations[stopRequestedAnnotation]; stop { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return r.finishNodeCapture(ctx, nc, true) + } + if nc.Annotations[nodeCaptureFinishedAnnotation] != "" { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return r.finishNodeCapture(ctx, nc, false) + } + var pod corev1.Pod + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + if apierrors.IsNotFound(err) { + return r.nodeFail(ctx, nc, "target game Pod no longer exists") + } + return ctrl.Result{}, err + } + var ss appsv1.StatefulSet + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: gs.Namespace, Name: gs.Name}, &ss); err != nil { + return ctrl.Result{}, err + } + ssOwner, podOwner := metav1.GetControllerOf(&ss), metav1.GetControllerOf(&pod) + if ssOwner == nil || ssOwner.UID != gs.UID || podOwner == nil || podOwner.UID != ss.UID { + return r.nodeFail(ctx, nc, "target Pod is not controlled by this GameServer") + } + containerID, err := nodeGameContainerID(&pod) + if err != nil { + return r.nodeFail(ctx, nc, err.Error()) + } + if bound := nc.Annotations[capturePodUIDAnnotation]; bound != "" && (bound != string(pod.UID) || nc.Annotations[nodeCaptureNodeAnnotation] != pod.Spec.NodeName || nc.Annotations[nodeCaptureContainerAnnotation] != containerID) { + return r.nodeFail(ctx, nc, "target Pod, node or container identity changed") + } + if nc.Annotations[nodeCaptureNamespaceAnnotation] == "" { + if r.DiagnosticsNamespace == gs.Namespace { + return r.nodeFail(ctx, nc, "node diagnostics namespace must be separate from the games namespace") + } + var namespace corev1.Namespace + if err := r.nodeReader().Get(ctx, types.NamespacedName{Name: r.DiagnosticsNamespace}, &namespace); err != nil { + return ctrl.Result{}, err + } + if namespace.Labels["pod-security.kubernetes.io/enforce"] != "privileged" { + return r.nodeFail(ctx, nc, "diagnostics namespace must explicitly permit privileged node diagnostics") + } + var captures gameplanev1alpha1.NetworkCaptureList + if err := r.nodeReader().List(ctx, &captures, client.InNamespace(nc.Namespace)); err != nil { + return ctrl.Result{}, err + } + for i := range captures.Items { + o := &captures.Items[i] + if o.UID != nc.UID && o.Spec.ServerRef.Name == gs.Name && !isTerminalCapturePhase(o.Status.Phase) && (o.CreationTimestamp.Before(&nc.CreationTimestamp) || (o.CreationTimestamp.Equal(&nc.CreationTimestamp) && o.Name < nc.Name)) { + return r.nodeFail(ctx, nc, "another capture is already in progress on this GameServer") + } + } + if nc.Annotations == nil { + nc.Annotations = map[string]string{} + } + nc.Annotations[nodeCaptureNamespaceAnnotation] = r.DiagnosticsNamespace + nc.Annotations[capturePodUIDAnnotation] = string(pod.UID) + nc.Annotations[nodeCaptureNodeAnnotation] = pod.Spec.NodeName + nc.Annotations[nodeCaptureContainerAnnotation] = containerID + controllerutil.AddFinalizer(nc, nodeCaptureFinalizer) + if err := r.Update(ctx, nc); err != nil { + return ctrl.Result{}, err + } + } + filter, failMessage, err := r.captureFilter(ctx, nc, &gs) + if err != nil { + return ctrl.Result{}, err + } + if failMessage != "" { + return r.nodeFail(ctx, nc, failMessage) + } + duration, size := r.CaptureDefaultMaxDurationSeconds, r.CaptureDefaultMaxSizeBytes + if nc.Spec.MaxDuration != nil { + duration = int64(nc.Spec.MaxDuration.Seconds()) + } + if nc.Spec.MaxSize != nil { + size = nc.Spec.MaxSize.Value() + } + image := r.CaptureSidecarImage + if image == "" { + image = DefaultCaptureSidecarImage + } + spec, err := nodeWorkerPodSpec(nc, &gs, &pod, image, nodeCaptureName(nc), *filter, duration, size) + if err != nil { + return r.nodeFail(ctx, nc, err.Error()) + } + anchor, err := r.ensureNodeArtifacts(ctx, nc, &gs, &pod) + if err != nil { + return ctrl.Result{}, err + } + var job batchv1.Job + jobKey := types.NamespacedName{Namespace: anchor.Namespace, Name: anchor.Name + "-worker"} + err = r.nodeReader().Get(ctx, jobKey, &job) + if apierrors.IsNotFound(err) { + if nc.Status.Phase == gameplanev1alpha1.CapturePhaseRunning { + return r.nodeFail(ctx, nc, "node diagnostic worker disappeared") + } + job = batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: jobKey.Name}, Spec: batchv1.JobSpec{BackoffLimit: ptrTo(int32(0)), ActiveDeadlineSeconds: ptrTo(duration + 30), Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{nodeCaptureUIDLabel: string(nc.UID), nodeCaptureComponentLabel: "worker"}}, Spec: spec}}} + if err := r.createNodeObject(ctx, &job, anchor, nc); err != nil { + return ctrl.Result{}, err + } + } else if err != nil { + return ctrl.Result{}, err + } + if job.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return ctrl.Result{}, errors.New("diagnostic worker identity collision") + } + if job.Status.Succeeded > 0 || job.Status.Failed > 0 { + if job.Status.Failed > 0 { + return r.nodeFail(ctx, nc, "node diagnostic worker failed; verify Linux containerd cgroups, node capabilities and storage") + } + nc.Annotations[nodeCaptureFinishedAnnotation] = time.Now().UTC().Format(time.RFC3339) + if err := r.Update(ctx, nc); err != nil { + return ctrl.Result{}, err + } + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return r.finishNodeCapture(ctx, nc, false) + } + if nc.Status.Phase != gameplanev1alpha1.CapturePhaseRunning { + nc.Status.Phase = gameplanev1alpha1.CapturePhaseRunning + nc.Status.StartTime = ptrTo(metav1.Now()) + nc.Status.Message = "node diagnostic capture running" + if err := r.patchGameServerActiveCapture(ctx, &gs, nc, nil); err != nil { + return ctrl.Result{}, err + } + if err := r.Status().Update(ctx, nc); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: time.Second}, nil +} + +func (r *NetworkCaptureReconciler) finishNodeCapture(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture, stopped bool) (ctrl.Result, error) { + if nc.Annotations[nodeCaptureNamespaceAnnotation] == "" { + return r.fail(ctx, nc, "capture stopped before the node diagnostic worker started") + } + phase, packets, bytes, message, err := r.SidecarClient.GetCaptureStatus(ctx, nc.Annotations[nodeCaptureNamespaceAnnotation], nodeCaptureName(nc), nc.Name, captureOwnerUID(nc.OwnerReferences, nc.Spec.ServerRef.Name), string(nc.UID)) + if err != nil { + if stopped { + return r.fail(ctx, nc, "capture stopped before a completed artifact was available") + } + finished, parseErr := time.Parse(time.RFC3339, nc.Annotations[nodeCaptureFinishedAnnotation]) + if parseErr != nil || time.Since(finished) > 2*time.Minute { + return r.fail(ctx, nc, "node diagnostic worker finished without an available artifact") + } + return ctrl.Result{RequeueAfter: time.Second}, nil + } + nc.Status.Phase = gameplanev1alpha1.CapturePhaseCompleted + if phase != "completed" { + nc.Status.Phase = gameplanev1alpha1.CapturePhaseFailed + } + nc.Status.PacketsWritten = packets + nc.Status.BytesWritten = resource.NewQuantity(bytes, resource.BinarySI) + nc.Status.Message = message + nc.Status.CompletionTime = ptrTo(metav1.Now()) + if err := r.releaseNodeCaptureLock(ctx, nc, nc.Status.CompletionTime); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: retentionPollInterval}, r.Status().Update(ctx, nc) +} + +// A direct CR deletion can arrive before terminal status. Release only after +// worker drain and only for this capture/server incarnation; optimistic locking +// prevents a concurrent successor from losing its newly acquired pointer. +func (r *NetworkCaptureReconciler) releaseNodeCaptureLock(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture, terminalTime *metav1.Time) error { + var live gameplanev1alpha1.NetworkCapture + if err := r.nodeReader().Get(ctx, client.ObjectKeyFromObject(nc), &live); err != nil { + return client.IgnoreNotFound(err) + } + if live.UID != nc.UID { + return nil + } + var gs gameplanev1alpha1.GameServer + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: nc.Namespace, Name: nc.Spec.ServerRef.Name}, &gs); err != nil { + return client.IgnoreNotFound(err) + } + if captureOwnerUID(nc.OwnerReferences, nc.Spec.ServerRef.Name) != string(gs.UID) || gs.Status.Capture == nil || gs.Status.Capture.ActiveCapture == nil || *gs.Status.Capture.ActiveCapture != nc.Name { + return nil + } + base := gs.DeepCopy() + gs.Status.Capture.ActiveCapture = nil + gs.Status.Capture.LastCaptureTime = terminalTime + return r.Status().Patch(ctx, &gs, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) +} diff --git a/operator/internal/controller/networkcapture_node_test.go b/operator/internal/controller/networkcapture_node_test.go new file mode 100644 index 000000000..ddd9a333d --- /dev/null +++ b/operator/internal/controller/networkcapture_node_test.go @@ -0,0 +1,213 @@ +package controller + +import ( + "context" + "reflect" + "strings" + "testing" + "time" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestNodeCaptureWorkerBoundAndNoCredentials(t *testing.T) { + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap-test", UID: "capture-uid"}} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "server-uid"}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{UID: "11111111-2222-3333-4444-555555555555"}, Spec: corev1.PodSpec{NodeName: "target-node"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "game", ContainerID: "containerd://" + strings.Repeat("a", 64), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}} + spec, err := nodeWorkerPodSpec(nc, gs, pod, "image", "pvc", "tcp port 25565", 10, 1024) + if err != nil { + t.Fatal(err) + } + if !spec.HostPID || spec.HostNetwork || spec.AutomountServiceAccountToken == nil || *spec.AutomountServiceAccountToken || spec.RestartPolicy != corev1.RestartPolicyNever { + t.Fatalf("unsafe worker isolation: %+v", spec) + } + if len(spec.Volumes) != 1 || spec.Volumes[0].PersistentVolumeClaim == nil { + t.Fatalf("worker acquired host paths or credentials: %+v", spec.Volumes) + } + if len(spec.Containers) != 1 || *spec.Containers[0].SecurityContext.RunAsUser != 0 || *spec.Containers[0].SecurityContext.AllowPrivilegeEscalation { + t.Fatalf("unexpected worker privileges: %+v", spec.Containers) + } + node := spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields[0] + if node.Key != "metadata.name" || len(node.Values) != 1 || node.Values[0] != "target-node" { + t.Fatalf("worker not bound to exact node: %+v", node) + } + pod.Spec.HostNetwork = true + if _, err := nodeWorkerPodSpec(nc, gs, pod, "image", "pvc", "tcp port 25565", 10, 1024); err == nil { + t.Fatal("host network target accepted") + } + pod.Spec.HostNetwork = false + pod.Status.ContainerStatuses[0].ContainerID = "docker://" + strings.Repeat("a", 64) + if _, err := nodeWorkerPodSpec(nc, gs, pod, "image", "pvc", "tcp port 25565", 10, 1024); err == nil { + t.Fatal("unsupported runtime accepted") + } +} + +func nodeReconcilerFixture(t *testing.T, objects ...client.Object) *NetworkCaptureReconciler { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, appsv1.AddToScheme, batchv1.AddToScheme, gameplanev1alpha1.AddToScheme} { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&gameplanev1alpha1.NetworkCapture{}, &gameplanev1alpha1.GameServer{}).WithObjects(objects...).Build() + return &NetworkCaptureReconciler{Client: c, APIReader: c, Scheme: scheme, CaptureEnabled: true, RestrictedWorkloads: true, DiagnosticsNamespace: "diagnostics"} +} + +type staleCaptureListClient struct{ client.Client } + +func (c staleCaptureListClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if captures, ok := list.(*gameplanev1alpha1.NetworkCaptureList); ok { + captures.Items = nil + return nil + } + return c.Client.List(ctx, list, opts...) +} + +func TestNodeCaptureExclusivityUsesLiveList(t *testing.T) { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "server"}, Spec: gameplanev1alpha1.GameServerSpec{Capture: &gameplanev1alpha1.CaptureConfiguration{Enabled: true, NodeDiagnostic: true}}} + owner := metav1.OwnerReference{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: gs.Name, UID: gs.UID, Controller: ptrTo(true)} + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "newer", Namespace: "games", UID: "newer", CreationTimestamp: metav1.NewTime(time.Now()), OwnerReferences: []metav1.OwnerReference{owner}}} + nc.Spec.ServerRef.Name = gs.Name + older := nc.DeepCopy() + older.Name = "older" + older.UID = "older" + older.CreationTimestamp = metav1.NewTime(time.Now().Add(-time.Minute)) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "statefulset", OwnerReferences: []metav1.OwnerReference{owner}}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "game-0", Namespace: "games", UID: "pod", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: ss.Name, UID: ss.UID, Controller: ptrTo(true)}}}, Spec: corev1.PodSpec{NodeName: "node"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "game", ContainerID: "containerd://" + strings.Repeat("a", 64), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}} + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "diagnostics", Labels: map[string]string{"pod-security.kubernetes.io/enforce": "privileged"}}} + r := nodeReconcilerFixture(t, gs, nc, older, ss, pod, ns) + r.Client = staleCaptureListClient{r.Client} + if _, err := r.reconcileNodeCapture(t.Context(), nc); err != nil { + t.Fatal(err) + } + var live gameplanev1alpha1.NetworkCapture + if err := r.Get(t.Context(), client.ObjectKeyFromObject(nc), &live); err != nil { + t.Fatal(err) + } + if live.Status.Phase != gameplanev1alpha1.CapturePhaseFailed || !strings.Contains(live.Status.Message, "another capture") { + t.Fatalf("stale cache bypassed exclusivity: %+v", live.Status) + } + if live.Annotations[nodeCaptureNamespaceAnnotation] != "" { + t.Fatal("new worker bound despite older request") + } +} + +func TestDeletingNodeCaptureReleasesOnlyItsOwnLock(t *testing.T) { + for _, test := range []struct { + name, active, serverUID string + clear bool + }{{"matching", "cap", "server", true}, {"other-capture", "other", "server", false}, {"replaced-server", "cap", "replacement", false}} { + t.Run(test.name, func(t *testing.T) { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: types.UID(test.serverUID)}, Status: gameplanev1alpha1.GameServerStatus{Capture: &gameplanev1alpha1.CaptureStatus{ActiveCapture: ptrTo(test.active)}}} + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap", Namespace: "games", UID: "capture", Finalizers: []string{nodeCaptureFinalizer}, DeletionTimestamp: ptrTo(metav1.Now()), Annotations: map[string]string{nodeCaptureNamespaceAnnotation: "diagnostics"}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: "game", UID: "server", Controller: ptrTo(true)}}}} + nc.Spec.ServerRef.Name = gs.Name + r := nodeReconcilerFixture(t, gs, nc) + if _, err := r.reconcileNodeCapture(t.Context(), nc); err != nil { + t.Fatal(err) + } + var live gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &live); err != nil { + t.Fatal(err) + } + if (live.Status.Capture.ActiveCapture == nil) != test.clear { + t.Fatalf("wrong lock release: %+v", live.Status.Capture) + } + }) + } +} + +func TestNodeWorkerCopiesTargetTolerations(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{UID: "pod"}, Spec: corev1.PodSpec{NodeName: "tainted-node", Tolerations: []corev1.Toleration{{Key: "game-nodes", Operator: corev1.TolerationOpEqual, Value: "dedicated", Effect: corev1.TaintEffectNoSchedule}}}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "game", ContainerID: "containerd://" + strings.Repeat("a", 64), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}} + spec, err := nodeWorkerPodSpec(&gameplanev1alpha1.NetworkCapture{}, &gameplanev1alpha1.GameServer{}, pod, "image", "pvc", "tcp", 10, 1024) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(spec.Tolerations, pod.Spec.Tolerations) { + t.Fatal("worker lost target tolerations") + } +} + +func TestNodeCaptureRefusesWithoutServerOptIn(t *testing.T) { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Capture: &gameplanev1alpha1.CaptureConfiguration{Enabled: true}}} + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap", Namespace: "games", UID: "capture-uid"}} + nc.Spec.ServerRef.Name = gs.Name + r := nodeReconcilerFixture(t, gs, nc) + if _, err := r.reconcileNodeCapture(t.Context(), nc); err != nil { + t.Fatal(err) + } + var live gameplanev1alpha1.NetworkCapture + if err := r.Get(t.Context(), client.ObjectKeyFromObject(nc), &live); err != nil { + t.Fatal(err) + } + if live.Status.Phase != gameplanev1alpha1.CapturePhaseFailed || !strings.Contains(live.Status.Message, "administrator") { + t.Fatalf("opt-in refusal missing: %+v", live.Status) + } + var jobs batchv1.JobList + if err := r.List(t.Context(), &jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("unauthorized worker: %v %+v", err, jobs) + } +} + +func TestNodeCaptureDrainsOrphanWorkersBeforeCleanup(t *testing.T) { + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap", Namespace: "games", UID: "capture-uid", Annotations: map[string]string{nodeCaptureNamespaceAnnotation: "diagnostics"}}} + labels := map[string]string{nodeCaptureUIDLabel: string(nc.UID), nodeCaptureComponentLabel: "worker"} + anchor := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: nodeCaptureName(nc), Namespace: "diagnostics", UID: "anchor", Labels: labels}} + worker := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orphan-worker", Namespace: "diagnostics", UID: "worker", Labels: labels}} + other := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "other-worker", Namespace: "diagnostics", UID: "other", Labels: map[string]string{nodeCaptureUIDLabel: "other", nodeCaptureComponentLabel: "worker"}}} + r := nodeReconcilerFixture(t, nc, anchor, worker, other) + ctx := context.Background() + if clean, err := r.cleanupNodeCapture(ctx, nc); err != nil || clean { + t.Fatalf("cleanup freed lock before observed worker deletion: %t %v", clean, err) + } + var live corev1.ConfigMap + if err := r.Get(ctx, client.ObjectKeyFromObject(anchor), &live); err != nil { + t.Fatal("anchor deleted before draining worker") + } + if clean, err := r.cleanupNodeCapture(ctx, nc); err != nil || clean { + t.Fatalf("cleanup did not wait for anchor GC: %t %v", clean, err) + } + if clean, err := r.cleanupNodeCapture(ctx, nc); err != nil || !clean { + t.Fatalf("cleanup failed to finish: %t %v", clean, err) + } + var p corev1.Pod + if err := r.Get(ctx, client.ObjectKeyFromObject(other), &p); err != nil { + t.Fatal("deleted unrelated worker") + } +} + +func TestNodeCaptureDoesNotDeleteNamesakeJob(t *testing.T) { + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap", UID: "capture-uid", Annotations: map[string]string{nodeCaptureNamespaceAnnotation: "diagnostics"}}} + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: nodeCaptureName(nc) + "-worker", Namespace: "diagnostics", UID: "foreign"}} + r := nodeReconcilerFixture(t, job) + if _, err := r.deleteNodeWorkers(t.Context(), nc); err == nil { + t.Fatal("accepted foreign job") + } + var live batchv1.Job + if err := r.Get(t.Context(), types.NamespacedName{Namespace: job.Namespace, Name: job.Name}, &live); apierrors.IsNotFound(err) { + t.Fatal("deleted foreign job") + } +} + +func TestNodeOptInPreservesLegacyCaptureLifecycle(t *testing.T) { + r := nodeReconcilerFixture(t) + for _, phase := range []gameplanev1alpha1.CapturePhase{gameplanev1alpha1.CapturePhasePending, gameplanev1alpha1.CapturePhaseRunning, gameplanev1alpha1.CapturePhaseCompleted} { + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{capturePodUIDAnnotation: "legacy-pod"}}, Status: gameplanev1alpha1.NetworkCaptureStatus{Phase: phase}} + if node, err := r.useNodeCapture(t.Context(), nc); err != nil || node { + t.Fatalf("legacy phase %s rerouted: %t %v", phase, node, err) + } + nc.Annotations[nodeCaptureNamespaceAnnotation] = "diagnostics" + if node, err := r.useNodeCapture(t.Context(), nc); err != nil || !node { + t.Fatalf("bound node phase %s lost lifecycle: %t %v", phase, node, err) + } + } +} diff --git a/operator/internal/controller/restore_controller.go b/operator/internal/controller/restore_controller.go index 4cbf94e10..9d134b578 100644 --- a/operator/internal/controller/restore_controller.go +++ b/operator/internal/controller/restore_controller.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "fmt" "time" @@ -36,11 +37,16 @@ type RestoreReconciler struct { // operator flag so air-gapped installs can point it at a private // registry mirror. Empty falls back to DefaultResticImage. ResticImage string + // BackupResources shares the backup worker resource configuration. + BackupResources *corev1.ResourceRequirements // JobBackoffLimit / JobActiveDeadlineSeconds bound the restic restore Job. // Set from operator flags; nil / zero fall back to // DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds. JobBackoffLimit *int32 JobActiveDeadlineSeconds int64 + // OperationQuiescer completes an orphaned backup's cleanup before this + // restore can acquire the same target. It never quiesces a restore. + OperationQuiescer AgentQuiescer } // apiReader returns the uncached reader for live-consistency checks, @@ -125,7 +131,10 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } return ctrl.Result{}, err } - if src.Spec.Strategy == "volume-snapshot" && rs.Annotations[restoreTargetAnnotation] == "" { + // Binding the target does not change the restore strategy. CSI's own + // preparation path checks that binding before awaiting startup, repairing + // references and enforcing the deadline on every subsequent reconcile. + if src.Spec.Strategy == "volume-snapshot" { return r.reconcileVolumeSnapshotRestore(ctx, &rs, &src) } @@ -156,6 +165,9 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } claimed, err := r.claimRestoreTarget(ctx, &rs, &gs) if err != nil { + if errors.Is(err, errLifecycleGenerationChanged) { + return r.fail(ctx, &rs, err.Error()) + } return ctrl.Result{}, err } if !claimed { @@ -168,12 +180,13 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } } - // A missing template is not fatal: it only refines the FSGroup, and - // buildBackupPodSecurityContext falls back to 65532 for a zero template. - var tmpl gameplanev1alpha1.GameTemplate - if err := r.Get(ctx, types.NamespacedName{Name: gs.Spec.TemplateRef.Name}, &tmpl); err != nil && !apierrors.IsNotFound(err) { + // A pinned snapshot is mandatory once established. Legacy targets retain + // the historical missing-template fallback for their initial operation. + effective, err := effectiveGameTemplate(ctx, r.apiReader(), &gs) + if err != nil && (gs.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { return ctrl.Result{}, err } + tmpl := *effective if rs.Status.Phase == gameplanev1alpha1.RestorePhaseSuspending { if gs.Status.Phase != gameplanev1alpha1.GameServerPhaseSuspended && @@ -327,8 +340,9 @@ func (r *RestoreReconciler) buildRestorePodSpec( return corev1.PodSpec{ SecurityContext: buildBackupPodSecurityContext(tmpl), Containers: []corev1.Container{{ - Name: "restic", - Image: resticImageOrDefault(r.ResticImage), + Name: "restic", + Image: resticImageOrDefault(r.ResticImage), + Resources: auxiliaryResources(r.BackupResources, "100m", "128Mi"), // --target / restores each snapshot entry at its original // absolute path. The companion backup runs `restic backup // /data`, so the snapshot tree is rooted at /data/...; with diff --git a/operator/internal/controller/restore_lifecycle.go b/operator/internal/controller/restore_lifecycle.go index a405b967a..91916ea54 100644 --- a/operator/internal/controller/restore_lifecycle.go +++ b/operator/internal/controller/restore_lifecycle.go @@ -70,7 +70,15 @@ func (r *RestoreReconciler) bindRestoreTarget(ctx context.Context, rs *gameplane // cannot both claim the same GameServer, even across operator processes. Suspend // is written in that same update; a crash cannot leave a lock without suspension. func (r *RestoreReconciler) claimRestoreTarget(ctx context.Context, rs *gameplanev1alpha1.Restore, gs *gameplanev1alpha1.GameServer) (bool, error) { - owner := restoreIdentityJSON(rs.Name, rs.UID) + // Refresh before both compatibility checks and shared-owner admission. + var live gameplanev1alpha1.GameServer + if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(gs), &live); err != nil { + return false, err + } + if live.UID != gs.UID { + return false, fmt.Errorf("restore target changed identity") + } + *gs = live workers, err := wipeWorkers(ctx, r.apiReader(), gs) if err != nil { return false, err @@ -95,38 +103,7 @@ func (r *RestoreReconciler) claimRestoreTarget(ctx context.Context, rs *gameplan } return false, nil } - if guard := gs.Annotations[restoreGuardAnnotation]; guard != "" && guard != owner { - identity, err := parseRestoreIdentity(guard) - if err != nil { - return false, err - } // Malformed guards fail closed. - var holder gameplanev1alpha1.Restore - err = r.apiReader().Get(ctx, types.NamespacedName{Namespace: gs.Namespace, Name: identity.Name}, &holder) - if err == nil && holder.UID == identity.UID { - return false, nil - } - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - // Force-deleted owners can leave their Job and pods behind. Cancel the - // old UID's Job and wait for its workers before reclaiming the guard. - busy, err := r.restoreWorkersLive(ctx, gs.Namespace, identity, true) - if err != nil || busy { - return false, err - } - } - if gs.Annotations[restoreGuardAnnotation] == owner && gs.Spec.Suspend { - return true, nil - } - if gs.Annotations == nil { - gs.Annotations = make(map[string]string) - } - gs.Annotations[restoreGuardAnnotation] = owner - gs.Spec.Suspend = true - if err := r.Update(ctx, gs); err != nil { - return false, err - } - return true, nil + return claimLifecycleOperation(ctx, r.Client, r.apiReader(), gs, restoreLifecycleIdentity(gs, rs), r.OperationQuiescer) } func liveRestorePod(pod *corev1.Pod) bool { @@ -280,6 +257,51 @@ func (r *RestoreReconciler) cleanupRestore(ctx context.Context, rs *gameplanev1a return ctrl.Result{}, err } if err == nil && gs.UID == target.UID && gs.Annotations[restoreGuardAnnotation] == restoreIdentityJSON(rs.Name, rs.UID) { + if value := gs.Annotations[lifecycleOperationAnnotation]; value != "" { + op, err := parseLifecycleOperation(value) + if err != nil { + return ctrl.Result{}, err + } + if !sameLifecycleOwner(op.lifecycleIdentity, restoreLifecycleIdentity(&gs, rs)) { + return ctrl.Result{}, fmt.Errorf("restore cleanup target has a different lifecycle owner") + } + if !success && (op.InitialStartup || !gs.Spec.Suspend) { + // A CSI restore may have started the game to bind its PVC. + // Retire that permission before asking the workload to stop; + // a replacement operation must not overlap the old process. + op.InitialStartup = false + gs.Spec.Suspend = true + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if err := r.Update(ctx, &gs); err != nil { + return ctrl.Result{}, err + } + } + ready, err := recoverLifecycleOperation(ctx, r.Client, r.apiReader(), &gs, op, r.OperationQuiescer) + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + } + if !success { + if !gs.Spec.Suspend { + gs.Spec.Suspend = true + if err := r.Update(ctx, &gs); err != nil { + return ctrl.Result{}, err + } + } + stopped, progressed, err := r.restoreTargetStopped(ctx, &gs) + if err != nil { + return ctrl.Result{}, err + } + if progressed { + return ctrl.Result{Requeue: true}, nil + } + if !stopped { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + } + if err := clearLifecycleOperation(ctx, r.Client, r.apiReader(), &gs, restoreLifecycleIdentity(&gs, rs)); err != nil { + return ctrl.Result{}, err + } var ss appsv1.StatefulSet ssErr := r.apiReader().Get(ctx, client.ObjectKeyFromObject(&gs), &ss) if ssErr != nil && !apierrors.IsNotFound(ssErr) { diff --git a/operator/internal/controller/restore_startup_cleanup_test.go b/operator/internal/controller/restore_startup_cleanup_test.go new file mode 100644 index 000000000..78da66458 --- /dev/null +++ b/operator/internal/controller/restore_startup_cleanup_test.go @@ -0,0 +1,181 @@ +package controller + +import ( + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func startupCleanupFixture(t *testing.T) (*RestoreReconciler, *gameplanev1alpha1.GameServer, *gameplanev1alpha1.Restore, *appsv1.StatefulSet, *corev1.Pod) { + t.Helper() + r, gs, rs := restoreLifecycleFixture(t) + rs.Annotations = map[string]string{restoreTargetAnnotation: restoreIdentityJSON(gs.Name, gs.UID)} + rs.Finalizers = []string{restoreCleanupFinalizer} + if err := r.Update(t.Context(), rs); err != nil { + t.Fatal(err) + } + op := lifecycleOperation{lifecycleIdentity: restoreLifecycleIdentity(gs, rs)} + op.InitialStartup = true + gs.Spec.Suspend = false + gs.Annotations = map[string]string{restoreGuardAnnotation: restoreIdentityJSON(rs.Name, rs.UID), lifecycleOperationAnnotation: lifecycleOperationJSON(op)} + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + ss := &appsv1.StatefulSet{} + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), ss); err != nil { + t.Fatal(err) + } + one := int32(1) + ss.Spec.Replicas = &one + ss.Spec.Template.Spec.Containers = []corev1.Container{{Name: "game", Image: "restored-image"}} + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.Replicas = 1 + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, UID: "startup-pod", Finalizers: []string{"test/hold"}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: ss.Name, UID: ss.UID, Controller: ownerBoolPtr(true)}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + return r, gs, rs, ss, pod +} + +func TestRestoreStartupCleanupDrainsBeforeRelease(t *testing.T) { + for _, outcome := range []string{"failed", "deleted", "succeeded"} { + t.Run(outcome, func(t *testing.T) { + r, _, rs, ss, pod := startupCleanupFixture(t) + rs.Status.Phase = gameplanev1alpha1.RestorePhaseFailed + if outcome == "succeeded" { + rs.Status.Phase = gameplanev1alpha1.RestorePhaseSucceeded + } + if err := r.Status().Update(t.Context(), rs); err != nil { + t.Fatal(err) + } + if outcome == "deleted" { + if err := r.Delete(t.Context(), rs); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(rs), rs); err != nil { + t.Fatal(err) + } + } + res, err := r.cleanupRestore(t.Context(), rs) + if err != nil { + t.Fatal(err) + } + gs := lifecycleServer(t, r) + if outcome == "succeeded" { + if gs.Spec.Suspend || gs.Annotations[lifecycleOperationAnnotation] != "" || res.RequeueAfter != 0 { + t.Fatalf("successful startup was stopped: %+v %+v", gs, res) + } + return + } + op, held, err := lifecycleOwner(gs) + if err != nil || !held || op.InitialStartup || !gs.Spec.Suspend || res.RequeueAfter == 0 { + t.Fatalf("failed startup released or retained permission to start: %+v %+v %v", gs, res, err) + } + // An observed scale-down does not prove a terminating Pod drained. + zero := int32(0) + ss.Spec.Replicas = &zero + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.Replicas, ss.Status.ObservedGeneration = 0, ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if res, err := r.cleanupRestore(t.Context(), rs); err != nil || res.RequeueAfter == 0 || lifecycleServer(t, r).Annotations[lifecycleOperationAnnotation] == "" { + t.Fatalf("terminating startup Pod lost exclusion: %+v %v", res, err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(pod), pod); err != nil { + t.Fatal(err) + } + pod.Finalizers = nil + if err := r.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + for range 2 { + if _, err := r.cleanupRestore(t.Context(), rs); err != nil { + t.Fatal(err) + } + } + gs = lifecycleServer(t, r) + if !gs.Spec.Suspend || gs.Annotations[lifecycleOperationAnnotation] != "" || gs.Annotations[restoreGuardAnnotation] != "" { + t.Fatalf("drained failed startup did not settle suspended: %+v", gs) + } + }) + } +} + +func TestOrphanRestoreStartupDrainsBeforeNextOwnerProceeds(t *testing.T) { + r, gs, rs, ss, pod := startupCleanupFixture(t) + rs.Finalizers = nil + if err := r.Update(t.Context(), rs); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), rs); err != nil { + t.Fatal(err) + } + b := guardBackup("next") + if err := r.Create(t.Context(), b); err != nil { + t.Fatal(err) + } + want := backupLifecycleIdentity(gs, b, false) + if ready, err := claimLifecycleOperation(t.Context(), r.Client, r.apiReader(), gs, want, nil); err != nil || ready { + t.Fatalf("orphan startup admitted another operation: %t %v", ready, err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.apiReader(), Scheme: r.Scheme} + if replicas, _, err := gr.desiredReplicas(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, idleAwake); err != nil || replicas != 0 { + t.Fatalf("orphan recovery did not request zero replicas: %d %v", replicas, err) + } + // Even a stale callback asking for one replica must stop the old process + // without changing the template before the successor owns clean data. + if err := gr.reconcileStatefulSet(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil { + t.Fatal(err) + } + if *ss.Spec.Replicas != 0 || ss.Spec.Template.Spec.Containers[0].Image != "restored-image" { + t.Fatalf("orphan recovery did not preserve the stopped template: %+v", ss.Spec) + } + ss.Status.Replicas, ss.Status.ObservedGeneration = 0, ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if ready, err := claimLifecycleOperation(t.Context(), r.Client, r.apiReader(), gs, want, nil); err != nil || ready { + t.Fatalf("observed scale-down bypassed live startup Pod: %t %v", ready, err) + } + pod.Finalizers = nil + if err := r.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + ready := false + for range 2 { + var err error + ready, err = claimLifecycleOperation(t.Context(), r.Client, r.apiReader(), gs, want, nil) + if err != nil { + t.Fatal(err) + } + if ready { + break + } + } + if !ready { + t.Fatal("drained predecessor did not permit successor") + } +} diff --git a/operator/internal/controller/restore_template.go b/operator/internal/controller/restore_template.go new file mode 100644 index 000000000..039fa3dff --- /dev/null +++ b/operator/internal/controller/restore_template.go @@ -0,0 +1,84 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const restoreTemplateAnnotation = "restore.gameplane.local/template-snapshot" + +func (r *RestoreReconciler) planRestoredTemplate(ctx context.Context, rs *gameplanev1alpha1.Restore, source, target *gameplanev1alpha1.GameServer) error { + // Pre-upgrade sources retain historical restore behavior. New snapshots + // cannot reconstruct the template used when an older backup was taken. + if source.Annotations[effectiveTemplateAnnotation] == "" { + return nil + } + identity := sha256.Sum256([]byte(string(rs.UID))) + name := fmt.Sprintf("restore-template-%x", identity[:16]) + var existing corev1.ConfigMap + err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: rs.Namespace, Name: name}, &existing) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil { + if !metav1.IsControlledBy(&existing, rs) || existing.Immutable == nil || !*existing.Immutable { + return fmt.Errorf("restore template plan is foreign or mutable") + } + } else { + tmpl, err := effectiveGameTemplate(ctx, r.apiReader(), source) + if err != nil { + return err + } + data, revision, err := templateSnapshot(tmpl) + if err != nil { + return err + } + immutable := true + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: rs.Namespace, Annotations: map[string]string{templateRevisionAnnotation: revision}}, Immutable: &immutable, Data: map[string]string{templateSnapshotKey: string(data)}} + if err := controllerutil.SetControllerReference(rs, cm, r.Scheme); err != nil { + return err + } + if err := r.Create(ctx, cm); err != nil { + return err + } + } + target.Annotations[restoreTemplateAnnotation] = name + return nil +} + +func (r *RestoreReconciler) copyRestoredTemplate(ctx context.Context, rs *gameplanev1alpha1.Restore, target *gameplanev1alpha1.GameServer) error { + var cm corev1.ConfigMap + if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: target.Namespace, Name: target.Annotations[restoreTemplateAnnotation]}, &cm); err != nil { + return err + } + if !metav1.IsControlledBy(&cm, rs) || cm.Immutable == nil || !*cm.Immutable || !cm.DeletionTimestamp.IsZero() { + return fmt.Errorf("restore template plan is foreign, mutable or deleting") + } + var tmpl gameplanev1alpha1.GameTemplate + if err := json.Unmarshal([]byte(cm.Data[templateSnapshotKey]), &tmpl); err != nil { + return err + } + _, revision, err := templateSnapshot(&tmpl) + if err != nil { + return err + } + if tmpl.Name != target.Spec.TemplateRef.Name || revision != cm.Annotations[templateRevisionAnnotation] { + return fmt.Errorf("restore template plan content changed") + } + writer := &GameServerReconciler{Client: r.Client, APIReader: r.apiReader(), Scheme: r.Scheme} + if _, err := writer.saveTemplateSnapshot(ctx, target, &tmpl); err != nil { + return err + } + target.Annotations[effectiveTemplateAnnotation] = revision + return r.Update(ctx, target) +} diff --git a/operator/internal/controller/restore_volumesnapshot.go b/operator/internal/controller/restore_volumesnapshot.go index c2d1819f5..ea1207787 100644 --- a/operator/internal/controller/restore_volumesnapshot.go +++ b/operator/internal/controller/restore_volumesnapshot.go @@ -76,6 +76,13 @@ func (r *RestoreReconciler) reconcileVolumeSnapshotRestore( "target server %q already exists; volume-snapshot restores create a new server", rs.Spec.ServerRef.Name)) } + ready, err := r.prepareSnapshotRestoreTarget(ctx, rs, &newGS) + if errors.Is(err, errLifecycleGenerationChanged) { + return r.fail(ctx, rs, err.Error()) + } + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } return r.awaitRestoredServer(ctx, rs, src, &newGS) case !apierrors.IsNotFound(err): return ctrl.Result{}, err @@ -94,9 +101,14 @@ func (r *RestoreReconciler) reconcileVolumeSnapshotRestore( created := &gameplanev1alpha1.GameServer{ ObjectMeta: metav1.ObjectMeta{ - Name: rs.Spec.ServerRef.Name, - Namespace: rs.Namespace, - Annotations: map[string]string{annoRestoreCreatedBy: rs.Name}, + Name: rs.Spec.ServerRef.Name, + Namespace: rs.Namespace, + Annotations: map[string]string{ + annoRestoreCreatedBy: rs.Name, + // Reserve the target atomically with creation. Once its UID is + // assigned, bind the full shared owner before permitting startup. + restoreGuardAnnotation: restoreIdentityJSON(rs.Name, rs.UID), + }, }, } orig.Spec.DeepCopyInto(&created.Spec) @@ -129,6 +141,12 @@ func (r *RestoreReconciler) reconcileVolumeSnapshotRestore( return r.failOrRequeue(ctx, rs, err) } rewriteRefs(&created.Spec, copies) + // Preserve the source's effective template before creating a target. A + // durable Restore-owned copy survives source adoption/deletion and retries; + // the target receives its own immutable copy before InitialStartup. + if err := r.planRestoredTemplate(ctx, rs, &orig, created); err != nil { + return r.failOrRequeue(ctx, rs, err) + } // Persist the copy plan on the Restore now, before creating anything. // ensureOwnedRefCopies below can fail transiently (e.g. a write @@ -211,6 +229,32 @@ func (r *RestoreReconciler) ensureRestoredRefsFromSpec( return nil } +func (r *RestoreReconciler) prepareSnapshotRestoreTarget(ctx context.Context, rs *gameplanev1alpha1.Restore, gs *gameplanev1alpha1.GameServer) (bool, error) { + if gs.Annotations[effectiveTemplateAnnotation] == "" && gs.Annotations[restoreTemplateAnnotation] != "" { + if err := r.copyRestoredTemplate(ctx, rs, gs); err != nil { + return false, err + } + return false, nil + } + changed, err := r.bindRestoreTarget(ctx, rs, gs) + if err != nil { + return false, err + } + if changed { + return false, nil + } + target, err := parseRestoreIdentity(rs.Annotations[restoreTargetAnnotation]) + if err != nil { + return false, err + } + if target.UID != gs.UID || target.Name != gs.Name { + return false, fmt.Errorf("CSI restore target changed identity") + } + want := restoreLifecycleIdentity(gs, rs) + want.InitialStartup = true + return claimLifecycleOperation(ctx, r.Client, r.apiReader(), gs, want, r.OperationQuiescer) +} + // awaitRestoredServer drives the Restore to a terminal phase based on the // newly-provisioned server: Running (with its references copied) → // Succeeded, Failed → Failed, otherwise keep polling while it starts up diff --git a/operator/internal/controller/restore_volumesnapshot_dispatch_test.go b/operator/internal/controller/restore_volumesnapshot_dispatch_test.go new file mode 100644 index 000000000..5ca38869b --- /dev/null +++ b/operator/internal/controller/restore_volumesnapshot_dispatch_test.go @@ -0,0 +1,81 @@ +package controller + +import ( + "encoding/json" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// Exercise the public dispatch after target binding. Calling the CSI helper +// directly misses a regression that diverted later passes into Restic restore. +func TestVolumeSnapshotRestoreReconcileKeepsBoundStrategy(t *testing.T) { + for _, state := range []string{"running", "repair-reference", "deadline"} { + t.Run(state, func(t *testing.T) { + scheme := scrapeScheme(t) + if err := appsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + orig := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "source", Namespace: "ns", UID: "source-uid"}} + target := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "target", Namespace: "ns", UID: "target-uid", Annotations: map[string]string{annoRestoreCreatedBy: "restore", restoreGuardAnnotation: restoreIdentityJSON("restore", "restore-uid")}}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}} + backup := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup", Namespace: "ns", UID: "backup-uid"}, Spec: gameplanev1alpha1.BackupSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: orig.Name}, Strategy: "volume-snapshot"}, Status: gameplanev1alpha1.BackupStatus{Phase: gameplanev1alpha1.BackupPhaseSucceeded, SnapshotID: "snapshot", VolumeSnapshotContentName: "content"}} + started := metav1.Now() + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns", UID: "restore-uid", Finalizers: []string{restoreCleanupFinalizer}, Annotations: map[string]string{restoreTargetAnnotation: restoreIdentityJSON(target.Name, target.UID)}}, Spec: gameplanev1alpha1.RestoreSpec{BackupRef: gameplanev1alpha1.LocalObjectRef{Name: backup.Name}, ServerRef: gameplanev1alpha1.LocalObjectRef{Name: target.Name}}, Status: gameplanev1alpha1.RestoreStatus{Phase: gameplanev1alpha1.RestorePhaseRunning, SnapshotID: "snapshot", StartTime: &started}} + objects := []client.Object{orig, target, backup, rs} + want := gameplanev1alpha1.RestorePhaseRunning + switch state { + case "running": + target.Status.Phase = gameplanev1alpha1.GameServerPhaseRunning + want = gameplanev1alpha1.RestorePhaseSucceeded + case "repair-reference": + plan, err := json.Marshal([]refCopy{{Kind: secretRefKind, OrigName: "source-secret", CopyName: "target-secret"}}) + if err != nil { + t.Fatal(err) + } + rs.Annotations["restore.gameplane.local/copy-plan"] = string(plan) + target.Spec.Env = []corev1.EnvVar{{Name: "SECRET", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "target-secret"}, Key: "key"}}}} + objects = append(objects, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "source-secret", Namespace: "ns", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(orig, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Data: map[string][]byte{"key": []byte("preserved-value")}}) + case "deadline": + started = metav1.NewTime(time.Now().Add(-VolumeSnapshotRestoreDeadline - time.Minute)) + want = gameplanev1alpha1.RestorePhaseFailed + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).WithStatusSubresource(rs, target, backup).Build() + r := &RestoreReconciler{Client: cl, APIReader: cl, Scheme: scheme} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(rs)}); err != nil { + t.Fatal(err) + } + var got gameplanev1alpha1.Restore + if err := cl.Get(t.Context(), client.ObjectKeyFromObject(rs), &got); err != nil { + t.Fatal(err) + } + if got.Status.Phase != want { + t.Fatalf("bound CSI restore phase = %s, want %s: %s", got.Status.Phase, want, got.Status.Message) + } + if state == "repair-reference" { + var copied corev1.Secret + if err := cl.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "target-secret"}, &copied); err != nil { + t.Fatalf("bound CSI restore did not repair reference: %v", err) + } + if !metav1.IsControlledBy(&copied, target) || string(copied.Data["key"]) != "preserved-value" { + t.Fatal("repaired reference lost ownership or data") + } + } + var jobs batchv1.JobList + if err := cl.List(t.Context(), &jobs); err != nil { + t.Fatal(err) + } + if len(jobs.Items) != 0 { + t.Fatal("CSI restore dispatched into Restic Job creation") + } + }) + } +} diff --git a/operator/internal/controller/restore_volumesnapshot_requeue_test.go b/operator/internal/controller/restore_volumesnapshot_requeue_test.go index 78b963678..64ebfb51b 100644 --- a/operator/internal/controller/restore_volumesnapshot_requeue_test.go +++ b/operator/internal/controller/restore_volumesnapshot_requeue_test.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -75,7 +76,7 @@ func TestVolumeSnapshotRestore_TransientErrorsRequeueNotFail(t *testing.T) { }, } restore := &gameplanev1alpha1.Restore{ - ObjectMeta: metav1.ObjectMeta{Name: "rs", Namespace: ns}, + ObjectMeta: metav1.ObjectMeta{Name: "rs", Namespace: ns, UID: "uid-restore"}, Spec: gameplanev1alpha1.RestoreSpec{ BackupRef: gameplanev1alpha1.LocalObjectRef{Name: "smp-vs"}, ServerRef: gameplanev1alpha1.LocalObjectRef{Name: restoredName}, @@ -105,6 +106,8 @@ func TestVolumeSnapshotRestore_TransientErrorsRequeueNotFail(t *testing.T) { switch o := obj.(type) { case *gameplanev1alpha1.GameServer: if o.Name == restoredName { + // The fake client does not assign API-server identities. + o.UID = "uid-restored" restoredCreated = true } case *corev1.Secret: @@ -149,13 +152,32 @@ func TestVolumeSnapshotRestore_TransientErrorsRequeueNotFail(t *testing.T) { if err := pass("stale-cache pass"); err != nil { t.Fatalf("stale-cache pass: want requeue without error, got %v", err) } - // Pass 3: cache caught up; the await path re-ensures the copy but the - // write still fails transiently. + // Pass 3: cache caught up; persist the new target's UID and cleanup + // finalizer before claiming its durable lifecycle owner on the next pass. staleRestored = false + if err := pass("bind target pass"); err != nil { + t.Fatalf("bind target pass: unexpected error: %v", err) + } + var bound gameplanev1alpha1.Restore + if err := cl.Get(ctx, client.ObjectKeyFromObject(restore), &bound); err != nil { + t.Fatalf("get bound restore: %v", err) + } + target, err := parseRestoreIdentity(bound.Annotations[restoreTargetAnnotation]) + if err != nil || target.Name != restoredName || target.UID != "uid-restored" { + t.Fatalf("bound target = %+v, error = %v; want restored server identity", target, err) + } + if !controllerutil.ContainsFinalizer(&bound, restoreCleanupFinalizer) { + t.Fatal("bound restore is missing its cleanup finalizer") + } + if failSecretCreates != 1 { + t.Fatalf("binding attempted a copy before admission: remaining failures = %d, want 1", failSecretCreates) + } + // Pass 4: the shared owner is acquired; the await path re-ensures the + // copy but the write still fails transiently. if err := pass("await pass with write error"); err == nil { t.Fatal("await pass: want the transient copy error returned for requeue, got nil") } - // Pass 4: errors cleared; the copy is made. + // Pass 5: errors cleared; the copy is made. if err := pass("await pass"); err != nil { t.Fatalf("await pass: unexpected error: %v", err) } @@ -165,6 +187,10 @@ func TestVolumeSnapshotRestore_TransientErrorsRequeueNotFail(t *testing.T) { if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: restoredName}, &restored); err != nil { t.Fatalf("get restored server: %v", err) } + owner, occupied, err := lifecycleOwner(&restored) + if err != nil || !occupied || !sameLifecycleOwner(owner.lifecycleIdentity, restoreLifecycleIdentity(&restored, restore)) || !owner.InitialStartup { + t.Fatalf("restored lifecycle owner = %+v, occupied = %v, error = %v; want initial-startup restore owner", owner, occupied, err) + } if got := restored.Spec.Env[0].ValueFrom.SecretKeyRef.Name; got != copyName { t.Errorf("restored env secret ref = %q, want copy %q", got, copyName) } diff --git a/operator/internal/controller/workload_security.go b/operator/internal/controller/workload_security.go new file mode 100644 index 000000000..d1471444c --- /dev/null +++ b/operator/internal/controller/workload_security.go @@ -0,0 +1,67 @@ +package controller + +import ( + "fmt" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" +) + +const agentTokenVolumeName = "agent-api-token" +const workloadSecurityLabel = "gameplane.local/workload-security" + +// Images must declare their compatible numeric identity; guessing from an +// image name or trusting an unknown image USER only moves failure to kubelet. +func validateRestrictedTemplate(tmpl *gameplanev1alpha1.GameTemplate) error { + sec := tmpl.Spec.Security + if sec == nil || sec.RunAsUser == nil || *sec.RunAsUser <= 0 || sec.FSGroup == nil || *sec.FSGroup <= 0 || (sec.RunAsGroup != nil && *sec.RunAsGroup <= 0) { + return fmt.Errorf("restricted workload requires template security.runAsUser and security.fsGroup greater than zero, and a positive security.runAsGroup when specified; use an image compatible with that identity") + } + return nil +} + +func restrictedContainerSecurity(tmpl *gameplanev1alpha1.GameTemplate) *corev1.SecurityContext { + group := tmpl.Spec.Security.RunAsGroup + if group == nil { + group = tmpl.Spec.Security.FSGroup + } + return &corev1.SecurityContext{ + RunAsUser: tmpl.Spec.Security.RunAsUser, RunAsGroup: group, + RunAsNonRoot: ptrTo(true), AllowPrivilegeEscalation: ptrTo(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } +} + +// All profiles restrict the mounted API credentials to the agent. Shared PID +// remains intentional for metrics: this is mount isolation, not a hard trust +// boundary against a process that can inspect another process through /proc. +func applyWorkloadSecurity(pod *corev1.PodSpec, tmpl *gameplanev1alpha1.GameTemplate, restricted bool, expirationSeconds int64) { + if expirationSeconds < 600 || expirationSeconds > 3600 { + expirationSeconds = 3600 + } + pod.AutomountServiceAccountToken = ptrTo(false) + pod.Volumes = append(pod.Volumes, corev1.Volume{Name: agentTokenVolumeName, VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptrTo(int32(0o644)), Sources: []corev1.VolumeProjection{ + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Path: "token", ExpirationSeconds: &expirationSeconds}}, + {ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "kube-root-ca.crt"}, Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}}}, + {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}}, + }, + }}}) + for i := range pod.Containers { + c := &pod.Containers[i] + if c.Name == "agent" { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{Name: agentTokenVolumeName, MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", ReadOnly: true}) + } + if restricted && c.Name == "game" { + c.SecurityContext = restrictedContainerSecurity(tmpl) + } + } + if restricted { + pod.SecurityContext.RunAsNonRoot = ptrTo(true) + pod.SecurityContext.SeccompProfile = &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault} + for i := range pod.InitContainers { + pod.InitContainers[i].SecurityContext = restrictedContainerSecurity(tmpl) + } + } +} diff --git a/operator/internal/controller/workload_security_test.go b/operator/internal/controller/workload_security_test.go new file mode 100644 index 000000000..8ba74505b --- /dev/null +++ b/operator/internal/controller/workload_security_test.go @@ -0,0 +1,125 @@ +package controller + +import ( + "strings" + "testing" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestRestrictedHostportFailsBeforeWorkloadCreation(t *testing.T) { + scheme := testScheme(t) + if err := batchv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "hostport", Namespace: "games", UID: "server-uid"}} + gs.Spec.TemplateRef.Name = "compatible" + gs.Spec.Networking.Expose = "Hostport" + tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: gs.Spec.TemplateRef.Name}} + tmpl.Spec.Security = &gameplanev1alpha1.GameSecuritySpec{RunAsUser: ptrTo(int64(1000)), FSGroup: ptrTo(int64(1000))} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gs, tmpl).WithStatusSubresource(gs).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: scheme, RestrictedWorkloads: true} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Status.Phase != gameplanev1alpha1.GameServerPhaseFailed || len(gs.Status.Conditions) == 0 || !strings.Contains(gs.Status.Conditions[0].Message, "use NodePort or LoadBalancer") { + t.Fatalf("incompatible exposure did not report an actionable failure: %+v", gs.Status) + } + var workloads appsv1.StatefulSetList + if err := c.List(t.Context(), &workloads); err != nil || len(workloads.Items) != 0 { + t.Fatalf("created a workload before validation: %+v %v", workloads.Items, err) + } + var claims corev1.PersistentVolumeClaimList + if err := c.List(t.Context(), &claims); err != nil || len(claims.Items) != 0 { + t.Fatalf("created storage before validation: %+v %v", claims.Items, err) + } + var configs corev1.ConfigMapList + if err := c.List(t.Context(), &configs); err != nil || len(configs.Items) != 0 { + t.Fatalf("created template/config state before validation: %+v %v", configs.Items, err) + } + // Exposure validation must also precede template lookup. This keeps + // future template admission/rollout work behind the same rejection. + if err := c.Delete(t.Context(), tmpl); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if len(gs.Status.Conditions) == 0 || !strings.Contains(gs.Status.Conditions[0].Message, "use NodePort or LoadBalancer") { + t.Fatalf("template lookup ran before exposure validation: %+v", gs.Status) + } +} + +func TestRestrictedTemplateIdentity(t *testing.T) { + for _, test := range []struct { + name string + uid, gid, fs *int64 + valid bool + }{ + {"absent", nil, nil, nil, false}, + {"root", ptrTo(int64(0)), nil, ptrTo(int64(1000)), false}, + {"missing-volume-group", ptrTo(int64(1000)), nil, nil, false}, + {"root-group", ptrTo(int64(1000)), ptrTo(int64(0)), ptrTo(int64(1000)), false}, + {"compatible", ptrTo(int64(1000)), ptrTo(int64(1000)), ptrTo(int64(1000)), true}, + {"default-primary-group", ptrTo(int64(1000)), nil, ptrTo(int64(1000)), true}, + } { + t.Run(test.name, func(t *testing.T) { + tmpl := &gameplanev1alpha1.GameTemplate{} + tmpl.Spec.Security = &gameplanev1alpha1.GameSecuritySpec{RunAsUser: test.uid, RunAsGroup: test.gid, FSGroup: test.fs} + if err := validateRestrictedTemplate(tmpl); (err == nil) != test.valid { + t.Fatalf("valid=%t: %v", test.valid, err) + } + }) + } +} + +func TestWorkloadTokenOnlyMountedInAgent(t *testing.T) { + tmpl := &gameplanev1alpha1.GameTemplate{} + tmpl.Spec.Security = &gameplanev1alpha1.GameSecuritySpec{RunAsUser: ptrTo(int64(1000)), FSGroup: ptrTo(int64(1000))} + gs := &gameplanev1alpha1.GameServer{} + pod := corev1.PodSpec{SecurityContext: gamePodSecurityContext(tmpl), Containers: []corev1.Container{{Name: "game"}, buildAgentContainer(gs, tmpl, nil, "agent", "", "")}, InitContainers: []corev1.Container{buildConfigInitContainer("", tmpl)}} + applyWorkloadSecurity(&pod, tmpl, true, 3600) + if pod.AutomountServiceAccountToken == nil || *pod.AutomountServiceAccountToken { + t.Fatal("automatic credentials exposed") + } + for _, c := range append(pod.Containers, pod.InitContainers...) { + sc := c.SecurityContext + if sc == nil || sc.RunAsNonRoot == nil || !*sc.RunAsNonRoot || sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation || sc.SeccompProfile == nil || sc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault || sc.Capabilities == nil || len(sc.Capabilities.Drop) != 1 || sc.Capabilities.Drop[0] != "ALL" || len(sc.Capabilities.Add) != 0 { + t.Fatalf("%s not Restricted: %+v", c.Name, sc) + } + mounted := false + for _, m := range c.VolumeMounts { + if m.Name == agentTokenVolumeName { + mounted = true + if !m.ReadOnly || m.MountPath != "/var/run/secrets/kubernetes.io/serviceaccount" { + t.Fatalf("unsafe token mount: %+v", m) + } + } + } + if mounted != (c.Name == "agent") { + t.Fatalf("%s token mount=%t", c.Name, mounted) + } + } + var projection *corev1.ProjectedVolumeSource + for _, v := range pod.Volumes { + if v.Name == agentTokenVolumeName { + projection = v.Projected + } + } + if projection == nil || len(projection.Sources) != 3 || projection.Sources[0].ServiceAccountToken == nil || *projection.Sources[0].ServiceAccountToken.ExpirationSeconds != 3600 { + t.Fatalf("missing rotating token projection: %+v", projection) + } +} diff --git a/operator/specs.md b/operator/specs.md index c7aea8225..60308563c 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -6,6 +6,35 @@ ## Purpose +### Restricted workloads and node capture (#714) + +`--restricted-workloads` applies nonroot, no escalation, drop ALL and +RuntimeDefault to game/config-init; incompatible numeric identities and Hostport exposure fail with +the existing Ready/Failed condition before workload creation. Every game Pod +disables token automount and projects token/CA/namespace only into agent. +Token lifetime is 600–3600s (default 3600); shared PID remains for metrics. +Use NodePort or LoadBalancer for Restricted workloads. Config-init creates +files with the declared fsGroup as its primary group in both profiles and +reconciles copied entries' group-write permission only when needed, so an +already-correct root-owned file can be reused by the Restricted init UID. +Existing volumes still need compatible ownership before enabling Restricted. + +Restricted Pods never receive the legacy capture ephemeral container. +`spec.capture.nodeDiagnostic` additionally requires cluster capture enablement +and `--diagnostics-namespace`. The dedicated namespace must explicitly enforce +privileged and be separate from games/control plane. Live owner UIDs establish +GameServer→StatefulSet→Pod provenance; Pod UID, node and exact containerd ID are +pinned before worker creation. A target change drains work before releasing +the capture lock. A bounded, no-retry Job has hostPID and only SYS_ADMIN, +SYS_PTRACE/NET_RAW additions, no token/socket/hostPath. Duration <=3600s, bytes +<=900Mi, PVC 1Gi; deployment quota bounds aggregate retention. Linux/containerd +with eth0 is the supported target. Artifact serving uses a separate Restricted +Pod with per-capture mTLS. Direct reads cover diagnostic Secrets and resources. +Cross-namespace owner references are never used: a NetworkCapture finalizer +drains worker Pods, deletes a same-namespace ConfigMap anchor with foreground +GC, then releases finalization. TTL and GameServer owner GC drive this cleanup. +Terminal captures refresh artifact certificates without recreating targets. + The operator is a Kubernetes controller-runtime-based process that reconciles Gameplane Custom Resource Definitions (CRDs) into operational Kubernetes objects. It is the authoritative owner of all state transformations: a user must be able to `kubectl apply` a GameServer and achieve the same outcome as creating it via the dashboard API. The operator drives the full lifecycle of game servers, backups, restores, and module installation through a set of specialized reconcilers watching CRD status. ## Responsibilities @@ -30,6 +59,41 @@ start/stop and expiry remain reconciler responsibilities; the gateway cannot bypass them or operate an arbitrary pod. Existing game resources and storage are not recreated to enable remote parity. +## Template revision admission + +- `GameServer.spec.templateRollout.policy` defaults to Automatic; OnRequest + admits only `requestedRevision == status.templateRevision.desired`. Initial + provisioning/baseline always adopts the installed content. No fleet scheduler, + canary or maintenance-window controller is implied. +- First-pin initialization uses shared ownership and a checked metadata update + without stopping an existing workload. Its StatefulSet UID and render-input + fingerprint preserve the existing Pod template byte-for-byte, including API + defaults/admission fields, while those inputs are unchanged. A real workload + input change, replaced StatefulSet or observed stopped drain allows revision + stamping. An existing unannotated Pod never receives inferred provenance. + New servers pin before first rendering. Existing lifecycle owners defer both. +- Complete template Name/spec content is SHA256-addressed in immutable owned + ConfigMaps (900 KiB maximum). The effective annotation is authoritative; status + is reconstructible. A missing/foreign/mutable/corrupt pin fails closed. +- All GameServer config, version/image, init, storage, agent, networking and + workload calculations use the effective snapshot. Restic backup resolves the + pin after shared ownership, restore resolves its owned target pin, and default + capture filters use pinned ports. CSI restore planning persists a Restore-owned + source snapshot and copies it to the new server before startup permission. +- Rollout extends the shared operation owner: no admission during backup, + restore or wipe; old effective graceful stop and live observed Pod drain precede + a checked pin update. The owner survives restart and catalog changes, releasing + after the matching new Pod is observed or a suspended zero-replica template is + observed. Startup health remains independent. Per-server power intent survives. +- Desired/effective/running digests are distinct. Running requires the owned + StatefulSet/Pod chain and a running game container; legacy or stopped Pods do + not claim the desired revision. TemplateUpdatePending has RevisionAvailable, + OperationInProgress, AdoptingRevision, RevisionCurrent and TemplateUnavailable + reasons, included in the generated condition translation reference. +- Obsolete snapshots are pruned only after old workload drain and replacement + observation. PVCs are retained; data compatibility and runtime/data rollback + remain administrator responsibilities. See `docs/template-rollouts.md`. + ## Non-goals / boundaries The operator **does not** handle: @@ -131,6 +195,20 @@ Nine CRD kinds under `gameplane.local/v1alpha1`: ## Reconcilers +### Auxiliary resource reservations + +Install flags `--agent-resources`, `--sentinel-resources`, `--tunnel-resources` +and `--backup-resources` accept complete JSON ResourceRequirements objects. +Absent flags use requests of 25m/64Mi for agent/config-init, 25m/32Mi for sentinel, +25m/64Mi for each tunnel provider, and 100m/128Mi for Restic backup/init/restore. +No new limits are set. Forget/prune retains 50m/128Mi and its existing 512Mi +memory limit when no override is configured. Explicit `{}` clears defaults. +Builders deep-copy resource maps. Template agent requests/limits, when declared, +replace install agent defaults; image-only overrides keep those defaults. +Game resources and GameServer placement retain their existing semantics. +Chart placement and optional PDBs are documented in +[`docs/production-placement.md`](../docs/production-placement.md). + Primary reconcilers register with the manager in `cmd/main.go` and handle CRD lifecycle: ### GameServerReconciler @@ -218,11 +296,17 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - **Responsibility:** Drive Backup to completion: coordinate quiesce (if agent available), spawn restic Job, track snapshot ID. - **Key functions:** - Quiesce orchestration: call agent Quiesce before Job, Unquiesce after completion. - - Quiesced copies are serialized per target GameServer UID across schedules and on-demand requests. The Backup pins its target in `backup.gameplane.local/target`; the GameServer's `backup.gameplane.local/owner` stores the owning Backup name/UID. Live API reads and resourceVersion updates protect acquisition/release across reconciler restarts. Quiesce cleanup intent is recorded before calling the agent, including partial failures. Cleanup waits for live copy workers; deletion cancels the copy Job with foreground propagation first. Force-deleted owners are recovered under exclusive ownership, draining their workers and restoring saving before the next copy. Pre-upgrade quiesced Backups without a target annotation finish and release all legacy workers before new protected copies may start. - - Orphan recovery preserves all predecessor names/UIDs in the GameServer's `backup.gameplane.local/recovering-owners` annotation, atomically with ownership transfers, so deleting a recovering Backup cannot hide an earlier worker. Normal deletion, orphan recovery, and legacy cleanup also wait for pending owned CSI captures to report ready or an explicit error; VolumeSnapshots are reclaimed by owner-reference GC after cleanup. + - With quiesce and an agent client enabled, preparation is recorded once in `backup.gameplane.local/preparation`, bound to the target UID and StatefulSet UID/generation. `already-stopped` requires uncached proof of desired and observed zero replicas, caught-up observed generation, and no live game/PVC user pods or pending writer Jobs. Current copy workers are excluded by Job ownership. This path sends no agent calls and preserves suspend/idle/wake intent under the shared workload fence. It applies equally to restic and CSI copies. + - Live preparation also pins the game Pod UID. The operator checks the agent's read-only `/v1/pods/{uid}/identity` endpoint before recording the first cleanup intent, then uses pod-bound quiesce/unquiesce routes; the agent rejects a different UID before RCON. A missing endpoint or identity mismatch fails closed without a legacy-route fallback. Ambiguous quiesce errors retain cleanup intent and the original Pod UID on both the Backup and shared owner. Only confirmed save-on or uncached evidence that the original pod is absent/replaced/terminal permits cleanup; an HTTP timeout, unready pod, or terminating pod is insufficient. Legacy attempts without a UID retire only on absence/terminal proof and never adopt a new UID. `Unquiesced=True, Reason=SourceProcessGone` distinguishes process exit from an actual save-on RPC. + - Source proof is checked before and after quiesce and on each copy reconcile. Observed workload/process replacement fails the Backup before it can report a restorable success. Owner recovery retains the pod UID even if the Backup is force-deleted, drains predecessor workers, and never sends its cleanup to a successor pod. + - Upgrade the operator and agent image together and allow existing GameServer pods to roll out the new agent and `GAMEPLANE_POD_UID` DownwardAPI env. Older agents fail the identity preflight before cleanup debt, so the workload can continue its upgrade after terminal cleanup releases the owner. Retry failed copies after rollout; preflight never bypasses protection. This does not automatically stop a running game. + - Every copy, including `quiesce=false` and CSI snapshots, acquires the shared lifecycle owner before starting. The Backup pins its target in `backup.gameplane.local/target`; `backup.gameplane.local/owner` remains a compatibility marker. The shared owner is described below. Quiesce cleanup intent is recorded on both the Backup and the surviving GameServer before calling the agent, including partial failures. Cleanup waits for live copy workers; deletion cancels the copy Job with foreground propagation first. Every Backup carries the cleanup finalizer. Pre-upgrade quiesced workers are observed through completion even when overlapping legacy copies cannot acquire the new exclusive owner. + - Orphan recovery preserves predecessor operation identities in the shared owner's `recovery` array, atomically with ownership transfer. Existing `backup.gameplane.local/recovering-owners` lineage is imported during migration. Deleting a recovering owner cannot hide an earlier worker. Normal deletion, orphan recovery, and legacy cleanup also wait for pending owned CSI captures to report ready or an explicit error; VolumeSnapshots are reclaimed by owner-reference GC after cleanup. - VolumeSnapshot API availability is checked before quiescing. Snapshot creation errors fail the Backup and run unquiesce cleanup immediately instead of retrying creation indefinitely with saving disabled. API/creation failures persist `backup.gameplane.local/snapshot-aborted` to allow cleanup without waiting for a failed/nonrestorable capture. Failed unquiesce remains visible and retried. A transport failure can leave a CSI snapshot object despite a Failed Backup; this does not make that Backup restorable. - Restic Job: mount data PVC, run restic backup to configured destination (S3, B2, rest-server, local fs). Job carries BackoffLimit and ActiveDeadlineSeconds from the reconciler's JobBackoffLimit / JobActiveDeadlineSeconds fields (operator flags --backup-job-backoff-limit / --backup-job-active-deadline-seconds, Helm operator.backupJobBackoffLimit / operator.backupJobActiveDeadlineSeconds; defaults 2 and 86400, applied when unset; spec 018 OD-027); only a Failed=True Job condition (set by jobPermanentlyFailed check) marks the Backup Failed. Pod SecurityContext takes FSGroup from the target GameTemplate's spec.security.fsGroup (fallback 65532 if template missing). - Snapshot tracking: parse restic JSON output from container logs to extract snapshot ID. + - Capture metadata: after ownership, workload fencing and quiesce admission, record `status.captureManifest` once before creating the first copy worker. It pins the GameServer UID, strategy, observed Pod/StatefulSet mount layout, PVC names/UIDs, included primary claim and excluded extra/mod/retained claims. `storage.extra[].backupRole` optionally declares `data` or `rebuildable`; absent declarations remain unclassified and excluded. Later template edits never rewrite the manifest. Existing workers and old terminal Backups have unknown historical coverage, never reconstructed from the current template. + - Achieved consistency: the `Consistency` condition is separate from copy phase and volume coverage. It stays Unknown until a usable snapshot identity is recorded; failed copies report `CaptureFailed`. Agent-confirmed quiesce, already-stopped/drained sources, unsupported/no-agent raw copy and unquiesced CSI snapshots have distinct reasons. Quiesce depends on the module commands and does not imply that excluded volumes were captured. The API exposes the stored status and the backup drawer shows consistency and included/excluded claims. `StoppedSource` records an observation, without adding a stop/copy/restart policy or multi-volume backup. - Snapshot cleanup on delete (`backup_forget.go`): restic-strategy Backups carry `BackupSnapshotFinalizer` (`gameplane.local/backup-snapshot-finalizer`, added on first reconcile, also for pre-existing non-Failed Backups; not added for volume-snapshot or Failed Backups). `finalizeDelete` runs the unquiesce stage (`BackupFinalizer`) first, then `finalizeSnapshot`: it deletes the Backup's own still-running Job (foreground, awaited), waits while a non-terminal Restore pins the Backup or its snapshot id, then runs Job `-forget` (`restic unlock || true; restic forget --prune --retry-lock 10m`, backoff 3, deadline 1800s, 50m/128Mi requests and 512Mi memory limit, no PVC; the snapshot id is passed through env `FORGET_SNAPSHOT_ID` and must match `^[0-9a-f]{8,64}$`). Success sets `SnapshotForgotten=True` and a Normal `SnapshotForgotten` event; a Failed Job, a missing/incomplete repo Secret, a terminating namespace, or `maxSnapshotForgetWait` (45m, from annotation `backup.gameplane.local/forget-started-at`) sets `SnapshotForgotten=False` with a Warning `SnapshotForgetAbandoned` event and releases the Backup, leaving the snapshot in the repository. A Backup with no recorded snapshot id is released with `SnapshotForgetSkipped` when it may have written one. Backup Jobs run `restic backup /data --json --retry-lock 10m ...`. RBAC markers on `backup_controller.go` grant `backups` update/patch, `backups/status`, `backups/finalizers` and `jobs` create/update/patch/delete. Tests: `backup_forget_test.go`, `backup_forget_envtest_test.go`, `retention_terminating_test.go`; e2e `TestBackup_DeleteForgetsSnapshot` and the extended `TestBackupSchedule_RetentionTrimsPast`. Spec: `specs/done_020-backup-snapshot-forget/`. - Status: report phase (Pending/Running/Succeeded/Failed), error details. - **Integrations:** Agent client (quiesce), Kubernetes Pod logs (restic output), VolumeSnapshot API (CSI backups). @@ -238,16 +322,48 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li ### RestoreReconciler - **Responsibility:** Drive Restore to completion: suspend GameServer, run restic restore Job, resume. - **Key functions:** + - Both restore strategies participate in shared lifecycle exclusion. A CSI restore reserves its new target with a Restore guard in the GameServer Create, then pins the target UID and adds the full owner. Its `initialStartup` mode permits the first workload to run, including PVCs using WaitForFirstConsumer; backup, wipe, restart, idle transitions and template rollout still wait for restore completion. The original server is unaffected. - Pin snapshot ID at observation (immutable during restore). - Restic restores pin the target GameServer name/UID on the Restore and add a cleanup finalizer before acquiring an annotation guard on that GameServer. The guard records the owning Restore name/UID and is acquired with an optimistic resourceVersion update that also sets spec.suspend=true. Other restores targeting that server wait; restores of independent targets may run concurrently. The game workload remains suspended while the guard exists, even if a power request clears spec.suspend; data-wipe reconciliation also waits for the guard to clear. - A nonempty pending data-wipe request blocks guard acquisition, including after wipe failure. Restore status explains this wait. An uncached read of the fixed-name wipe Job identifies a failed current request only when the current GameServer controls it, its token matches, and it reports Failed=True; DataWipe=False alone is insufficient. Confirmed failure reports that restore is blocked and asks the operator to inspect wipe Job logs and resolve the wipe failure. The restore does not cancel or acknowledge the wipe, since an in-flight wipe reconciliation could still create a writer. The wait message clears when restore acquires the target normally. - Suspend: wait for Suspended/Stopped status plus the actual workload: the owned StatefulSet must exist, request zero replicas, report zero replicas, and have observed its current generation. An absent StatefulSet waits for workload reconciliation; a missing template or broken workload must be repaired before restore can proceed. No live or terminating game pod or pod mounting the data PVC may remain. Nonterminal Jobs mounting that PVC must also finish before a new restore Job starts. These checks use the uncached API reader and run again immediately before creating the Job. - The stopped StatefulSet also receives the owning Restore guard. A successful fence write immediately requeues the Restore to repeat the workload and writer checks; only external drainage waits use the polling delay. Its mutation callback honors that guard even when the GameServer cache snapshot predates acquisition, fencing delayed scale-up writes with the StatefulSet resourceVersion. Cleanup removes both guards after workers drain. - Restore Job: mount snapshot (from VolumeSnapshot or restic), run restic restore. Job carries BackoffLimit and ActiveDeadlineSeconds from the reconciler's JobBackoffLimit / JobActiveDeadlineSeconds fields (operator flags --backup-job-backoff-limit / --backup-job-active-deadline-seconds, Helm operator.backupJobBackoffLimit / operator.backupJobActiveDeadlineSeconds; defaults 2 and 86400, applied when unset; spec 018 OD-027); Job failure is recognized only from a Failed=True condition. Legacy in-flight Jobs without the pinned target annotation fail the Restore and are canceled/drained; operators can start a fresh Restore after cleanup. Pod SecurityContext takes FSGroup from the target GameTemplate's spec.security.fsGroup (fallback 65532 if template missing). - - Resume: once the restore Job succeeds and its live pods have drained, persist Succeeded before releasing the matching guard and clearing suspend in the same pass. This ordering survives interrupted status writes or operator restarts; it does not wait for the GameServer to reach Running again. Failure/deletion cancels active owned Jobs with foreground deletion and retains the guard/finalizer until Jobs and live worker pods drain, then leaves the target suspended. A missing or recreated guard owner can be recovered only after its UID's orphan Jobs and workers drain. Target recreation fails the old restore; cleanup never changes the replacement server. + - Resume: once the restore Job succeeds and its live pods have drained, persist Succeeded before releasing the matching guard and clearing suspend in the same pass. This ordering survives interrupted status writes or operator restarts; it does not wait for the GameServer to reach Running again. Failure/deletion cancels active owned Jobs with foreground deletion, retires CSI initial-startup permission and suspends the target while retaining the guard/finalizer. Release additionally requires an owned StatefulSet with desired/observed zero replicas at its current generation, no live or terminating game/PVC writer pods and no pending PVC writer Jobs. Orphan recovery applies the same stopped-target proof before admitting a successor, preserving the template while forcing zero replicas. Successful CSI restores keep their initial workload running. Target recreation fails the old restore; cleanup never changes the replacement server. - Status: report phase (Pending/Suspending/Running/Succeeded/Failed). `RestorePhaseResuming` is declared on the type but never assigned by this reconciler (F-260). - Volume-snapshot restore reference handling: When a volume-snapshot restore stands up a new GameServer from the original server's spec, any references in spec.env (SecretKeyRef, ConfigMapKeyRef) and spec.networking.tunnel.credentialsSecretRef that are owned by the original server must be copied to new objects owned by the restored server, with rewritten names, because the restored server does not own the originals. Ownership of a source object is an OwnerReference to the original server (Kind, Name, UID) and is re-verified against the source object immediately before each copy, not just once during planning. An object that already exists under a copy name is accepted only if the restored server controls it (a controller OwnerReference, `controller: true`); otherwise the restore fails. The restore fails before the new GameServer is created if the original server does not own every referenced object. The references are rewritten in the new GameServer's spec when it is created, and the copies are made right after creation and re-ensured on every reconcile pass. Only an ownership refusal fails the restore; transient errors (stale cache reads, write conflicts, a referenced object not yet visible in the cache) requeue it. A restore's 10-minute deadline starts when the restore first plans the new server (before it is created) and covers both creating the server with its reference copies and waiting for it to reach Running; it is enforced even while a transient error (such as a referenced object that stays missing) keeps recurring, so the restore cannot run past the limit, and once it expires the restore fails with a deadline message. +### Shared lifecycle exclusion + +`lifecycle_operation.go` is the admission boundary for Backup, Restore and data +wipe. `operations.gameplane.local/owner` on each GameServer stores kind, name, +operation UID, target UID and admitted generation. Wipes use the GameServer UID +plus their existing request token. Acquisition refreshes the target with the +uncached reader and commits the owner and compatibility guard in one checked +resourceVersion Update. Existing owners never expire on time alone. Independent +servers remain concurrent; compatible reader modes are not supported. + +After acquisition, backups fence the owned StatefulSet with a checked metadata +write and wait for its current generation and rollout revision to be observed. +Workload mutation reads ownership inside its callback: backup ownership freezes +replicas and the complete PodTemplate, while restore/wipe may stop the existing +workload but cannot roll it. A missing workload is created stopped so it can be +fenced. Idle sleep/wake and restart acknowledgment use checked writes; a stop RPC +is preceded by a checked durable stop intent which prevents backup admission +until that process drains. Ordinary Services, credentials and status continue +reconciling while operations hold the workload. + +A missing owner is recovered by a successor under exclusive ownership. Recovery +lineage is retained on the GameServer across repeated force deletion. Owned Jobs +are canceled with UID preconditions and foreground propagation; independently +inventoried live or terminating worker pods and pending CSI captures retain the +guard. Saving is restored only after recovery workers drain. The workload remains +frozen during recovery even when the successor is a Restore or wipe. Terminal +state and existing cleanup intent are preserved before releasing the owner; +failed/canceled restores remain suspended. Generation changes to active Backup +or Restore specs fail the operation and enter cleanup rather than authorizing a +different copy under the old owner. See [operator operation recovery](../docs/operator-operations.md). + ### ModuleSourceReconciler - **Responsibility:** Index module sources and surface available modules into status. - **Key functions:** @@ -446,7 +562,7 @@ Forgetting codegen leaves the YAML out of sync with types — CI's `make manifes 6. **Module sources are immutable once pulled.** Digest pinning (spec.digest) defeats tag moves; version pinning (spec.version) tracks a specific semver. Floating (unset version/digest) tracks latest and re-pulls on ModuleSource refresh. -7. **Backup quiesce is best-effort.** If agent unavailable, backup proceeds raw (no pause). If quiesce unsupported (agent returns ErrUnsupported), backup continues degraded (success-with-note). +7. **Backup quiesce fails closed on live transport errors.** With no configured agent client or `quiesce=false`, a copy proceeds raw. With a client enabled, a drained stopped source skips RPC, but an unreachable live agent prevents the copy. A game that explicitly reports quiesce unsupported can continue without claiming it was quiesced. 8. **Remote cluster health checks are non-blocking.** A Cluster with health Unhealthy does not prevent GameServer creation on the local cluster; it surfaces the issue so operators can intervene. diff --git a/test/e2e/api_events_roles_scope_e2e_test.go b/test/e2e/api_events_roles_scope_e2e_test.go index 2ab19b2b4..4c3b388c0 100644 --- a/test/e2e/api_events_roles_scope_e2e_test.go +++ b/test/e2e/api_events_roles_scope_e2e_test.go @@ -12,6 +12,9 @@ import ( "strings" "testing" "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions checks two @@ -64,19 +67,37 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { applyBusyboxTemplate(t, tmpl) createGameServerViaAPI(t, admin, ns, gsName, tmpl) + // /events starts at each collection's current resourceVersion. Drive real + // updates throughout both streams instead of depending on initial replay + // or other parallel tests to happen to mutate a readable resource kind. + changes := func(ctx context.Context) error { + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": map[string]string{ + "gameplane.local/e2e-event-sequence": time.Now().UTC().Format(time.RFC3339Nano), + }}}) + if err != nil { + return fmt.Errorf("encode event trigger: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, tmpl, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger template event: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, gsName, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger server event: %w", err) + } + return nil + } user := envInstance.APIClient(t, userName, userPW) defer user.Close() t.Run("admin event stream includes templates", func(t *testing.T) { - kinds, found := streamEventKinds(t, admin, ns, func(kind, _ string) bool { return kind == "templates" }, 0) + kinds, found := streamEventKinds(t, admin, ns, func(kind, name string) bool { return kind == "templates" && name == tmpl }, 0, changes) if !found { t.Fatalf("admin stream sent no templates event; kinds=%v", kinds) } }) t.Run("event stream sends only kinds the caller may read", func(t *testing.T) { - kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second) + kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second, changes) if !found { t.Fatalf("user stream never sent server %s; kinds=%v", gsName, kinds) } @@ -126,11 +147,41 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { // streamEventKinds reads GET /events as cli and returns the set of event // kinds it received. When match reports true for a frame, it keeps // reading for grace (0 = stop at once); otherwise it stops after 30s. +// changes keeps both permitted and forbidden kinds active while connecting and +// reading; its worker is cancelled and joined before the helper returns. // found reports whether match was ever true. -func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration) (map[string]bool, bool) { +func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration, changes func(context.Context) error) (map[string]bool, bool) { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() + changesDone := make(chan struct{}) + var changesErr error + go func() { + defer close(changesDone) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + if err := changes(ctx); err != nil { + if ctx.Err() == nil { + changesErr = err + } + cancel() + return + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + defer func() { + cancel() + <-changesDone + if changesErr != nil { + t.Errorf("event trigger failed: %v", changesErr) + } + }() req, err := http.NewRequestWithContext(ctx, http.MethodGet, cli.BaseURL+"/events?namespace="+ns, nil) if err != nil { t.Fatalf("new events request: %v", err) diff --git a/test/e2e/backup_stopped_e2e_test.go b/test/e2e/backup_stopped_e2e_test.go new file mode 100644 index 000000000..d469a635d --- /dev/null +++ b/test/e2e/backup_stopped_e2e_test.go @@ -0,0 +1,112 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// Exercise real PVC data and restic for two consecutive protected copies while +// the agent is absent. This test needs no API login and runs in the operator bucket. +func TestBackup_StoppedCopiesRemainRestorable(t *testing.T) { + t.Parallel() + ctx := context.Background() + ns, gs, tmpl := "gameplane-games", "e2e-stopped-backup", "e2e-stopped-backup-template" + ensureResticRepo(t) + applyBusyboxTemplate(t, tmpl) + applyBusyboxGameServer(t, ns, gs, tmpl) + waitPVCBound(t, ns, gs+"-data", 90*time.Second) + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "game" && status.Ready { + return true, "" + } + } + return false, "game container not ready" + }) + if out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "sh", "-c", "echo stopped-backup-marker > /data/marker.txt && sync"); err != nil { + t.Fatalf("write marker: %v\n%s", err, out) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, gs, types.MergePatchType, []byte(`{"spec":{"suspend":true}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + ss, err := envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, gs, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation { + return false, "workload not drained" + } + _, err = envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + return apierrors.IsNotFound(err), fmt.Sprintf("pod absence: %v", err) + }) + for i := 1; i <= 2; i++ { + name := fmt.Sprintf("e2e-stopped-copy-%d", i) + backup := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "Backup", + "metadata": map[string]any{"name": name, "namespace": ns}, + "spec": map[string]any{"serverRef": map[string]any{"name": gs}, "repoRef": map[string]any{"name": "e2e-restic-creds", "key": "repo"}, "strategy": "restic-snapshot", "quiesce": true}, + }} + if _, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Create(ctx, backup, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { deleteBackupCR(t, ns, name) }) + var snapshotID string + envInstance.Eventually(t, 5*time.Minute, func() (bool, string) { + // Check throughout the copy, not only after its terminal result. + _, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("backup woke game or absence could not be confirmed: %v", err) + } + got, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + phase, _, _ := unstructured.NestedString(got.Object, "status", "phase") + if phase == "Failed" { + t.Fatalf("stopped backup failed: %+v", got.Object["status"]) + } + if phase != "Succeeded" { + return false, "phase=" + phase + } + var preparation struct { + Outcome string `json:"outcome"` + } + if err := json.Unmarshal([]byte(got.GetAnnotations()["backup.gameplane.local/preparation"]), &preparation); err != nil || preparation.Outcome != "already-stopped" { + t.Fatalf("missing stopped-source proof: %+v, %v", preparation, err) + } + if got.GetAnnotations()["backup.gameplane.local/quiesce-attempted"] != "" { + t.Fatal("stopped source contacted absent agent") + } + snapshotID, _, _ = unstructured.NestedString(got.Object, "status", "snapshotID") + return snapshotID != "", "snapshot ID pending" + }) + if !regexp.MustCompile(`^[0-9a-f]{8,64}$`).MatchString(snapshotID) { + t.Fatalf("unexpected snapshot ID %q", snapshotID) + } + runResticJob(t, "restic restore "+snapshotID+" --target /tmp/stopped-restore --retry-lock 5m && test \"$(cat /tmp/stopped-restore/data/marker.txt)\" = stopped-backup-marker") + } + got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, gs, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + suspended, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend") + if !suspended { + t.Fatal("backup changed manual suspend intent") + } +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index a32d38dea..3fa48bcd6 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -62,6 +62,7 @@ TestGameServer_TemplateDefaultOverridesInstallTime TestGameServer_ExplicitStorageClassOverridesDefault TestGameServer_NonexistentStorageClassSurfacesError TestGameServer_VersionSwitch +TestGameServer_TemplateOnRequestRetainsData TestGameServer_WipeReportsFailureOnPermissionDenied TestGameServer_CascadingDelete TestGameServer_TunnelCredentialRefusalSurfaced @@ -83,6 +84,7 @@ TestModuleSourceUpload TestModule_ScaffoldAndPackage TestModule_SteamcmdScaffoldKeepsNonRootSecurity TestBackup_OperatorMaterializesJob +TestBackup_StoppedCopiesRemainRestorable TestBackup_FailsOnMissingPVC TestBackup_FailsOnBadCredentials TestBackup_DeleteForgetsSnapshot @@ -393,6 +395,12 @@ TestOptionalComponents_Lifecycle EOF } +# Dedicated release: enables real PSA enforcement and privileged diagnostics. +bucket_restricted_workloads() { cat <<'EOF' +TestRestrictedWorkloads_NodeCaptureAndTokenRotation +EOF +} + # capture-overhead is kept OUT of bot-fast on purpose: bot-fast's matrix legs # are blocking, while this bucket feeds the non-blocking e2e-capture-overhead # job (continue-on-error, never blocks a merge — spec 007 OD-1) and runs only @@ -406,10 +414,15 @@ EOF # Tests that exist in the suite but deliberately run in NO bucket. Every # entry needs a reason; `verify` fails on any unlisted stray so additions # here are a conscious, reviewed act. Currently empty. +bucket_management_placement() { cat <<'EOF' +TestManagementPlacementAndDisruption +EOF +} + unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components restricted-workloads management-placement } list_bucket() { @@ -428,6 +441,8 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + restricted-workloads) bucket_restricted_workloads ;; + management-placement) bucket_management_placement ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index d2d6edd00..02949e334 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", + "telemetry", "optional-components", "restricted-workloads", "management-placement", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. diff --git a/test/e2e/management_placement_e2e_test.go b/test/e2e/management_placement_e2e_test.go new file mode 100644 index 000000000..ec4525dfa --- /dev/null +++ b/test/e2e/management_placement_e2e_test.go @@ -0,0 +1,252 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/yaml" +) + +// This bucket has no API logins and runs alone on the three-node management +// profile. It uses chart-rendered placement/PDBs with sleeping fixture containers +// to exercise the real scheduler and eviction controller without game downloads. +func TestManagementPlacementAndDisruption(t *testing.T) { + e := envInstance + nodes, err := e.K8s.CoreV1().Nodes().List(t.Context(), metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + var candidates []string + for _, node := range nodes.Items { + eligible := !node.Spec.Unschedulable + for _, taint := range node.Spec.Taints { + if taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute { + eligible = false + } + } + ready := false + for _, condition := range node.Status.Conditions { + ready = ready || (condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue) + } + if eligible && ready { + candidates = append(candidates, node.Name) + } + } + if len(candidates) < 2 { + t.Fatal("management placement requires two schedulable workers; use the management e2e profile") + } + candidates = candidates[:2] + ns := fmt.Sprintf("placement-%d", time.Now().UnixNano()) + if _, err := e.K8s.CoreV1().Namespaces().Create(t.Context(), &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: ns}}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = e.K8s.CoreV1().Namespaces().Delete(context.Background(), ns, metav1.DeleteOptions{}) }) + poolLabel := "gameplane.local/" + ns + for _, name := range candidates { + patchNode := func(value any) error { + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"labels": map[string]any{poolLabel: value}}}) + if err != nil { + return err + } + _, err = e.K8s.CoreV1().Nodes().Patch(context.Background(), name, types.MergePatchType, patch, metav1.PatchOptions{}) + return err + } + if err := patchNode("management"); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = patchNode(nil) }) + } + + chart, err := findRepoRel("charts/gameplane") + if err != nil { + t.Fatal(err) + } + selector := map[string]any{"app.kubernetes.io/name": "gameplane-web", "app.kubernetes.io/instance": ns} + values := map[string]any{ + "operator": map[string]any{"enabled": false}, + "api": map[string]any{"pdb": map[string]any{"enabled": true, "maxUnavailable": 1}}, + "web": map[string]any{ + "enabled": true, "replicas": 2, "nodeSelector": map[string]any{poolLabel: "management"}, + "pdb": map[string]any{"enabled": true, "maxUnavailable": 1}, + "affinity": map[string]any{"podAntiAffinity": map[string]any{"requiredDuringSchedulingIgnoredDuringExecution": []any{map[string]any{"topologyKey": "kubernetes.io/hostname", "labelSelector": map[string]any{"matchLabels": selector}}}}}, + "topologySpreadConstraints": []any{map[string]any{"maxSkew": 1, "topologyKey": "kubernetes.io/hostname", "whenUnsatisfiable": "DoNotSchedule", "labelSelector": map[string]any{"matchLabels": selector}}}, + }, + } + data, err := json.Marshal(values) + if err != nil { + t.Fatal(err) + } + valuesPath := filepath.Join(t.TempDir(), "values.json") + if err := os.WriteFile(valuesPath, data, 0o600); err != nil { + t.Fatal(err) + } + rendered, err := exec.CommandContext(t.Context(), "helm", "template", ns, chart, "--namespace", ns, "--values", valuesPath, "--show-only", "templates/web.yaml", "--show-only", "templates/poddisruptionbudgets.yaml").CombinedOutput() + if err != nil { + t.Fatalf("render placement chart: %v: %s", err, rendered) + } + fixture := corev1.Container{Name: "fixture", Image: "busybox:1.38.0", Command: []string{"sleep", "3600"}} + decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewReader(rendered), 4096) + for { + var object unstructured.Unstructured + if err := decoder.Decode(&object); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + switch object.GetKind() { + case "Deployment": + var deployment appsv1.Deployment + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.Object, &deployment); err != nil { + t.Fatal(err) + } + deployment.Spec.Template.Spec.Containers = []corev1.Container{fixture} + if _, err := e.K8s.AppsV1().Deployments(ns).Create(t.Context(), &deployment, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + case "PodDisruptionBudget": + var pdb policyv1.PodDisruptionBudget + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.Object, &pdb); err != nil { + t.Fatal(err) + } + if _, err := e.K8s.PolicyV1().PodDisruptionBudgets(ns).Create(t.Context(), &pdb, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + } + webPods := func() ([]corev1.Pod, error) { + list, err := e.K8s.CoreV1().Pods(ns).List(t.Context(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gameplane-web"}) + if err != nil { + return nil, err + } + return list.Items, nil + } + waitReady := func() { + e.Eventually(t, 3*time.Minute, func() (bool, string) { + pods, err := webPods() + if err != nil || len(pods) != 2 { + return false, fmt.Sprintf("pods=%d error=%v", len(pods), err) + } + seen := map[string]bool{} + for _, pod := range pods { + ready, err := e.PodIsReady(t.Context(), ns, pod.Name) + if err != nil || !ready || !pod.DeletionTimestamp.IsZero() { + return false, "web fixture not ready" + } + if pod.Spec.NodeName != candidates[0] && pod.Spec.NodeName != candidates[1] { + t.Fatalf("pod escaped management selector to %s", pod.Spec.NodeName) + } + seen[pod.Spec.NodeName] = true + } + return len(seen) == 2, "web replicas were not spread across two workers" + }) + } + waitReady() + setSelector := func(value string) { + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{"nodeSelector": map[string]any{poolLabel: value}}}}}) + if err != nil { + t.Fatal(err) + } + if _, err := e.K8s.AppsV1().Deployments(ns).Patch(t.Context(), "gameplane-web", types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + } + setSelector("absent-pool") + e.Eventually(t, time.Minute, func() (bool, string) { + pods, err := webPods() + if err != nil { + return false, err.Error() + } + for _, pod := range pods { + for _, condition := range pod.Status.Conditions { + if pod.Spec.NodeSelector[poolLabel] == "absent-pool" && condition.Type == corev1.PodScheduled && condition.Status == corev1.ConditionFalse && condition.Reason == "Unschedulable" { + return true, "" + } + } + } + return false, "missing unschedulable replacement" + }) + setSelector("management") + waitReady() + cordon := func(unschedulable bool) error { + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"unschedulable": unschedulable}}) + if err != nil { + return err + } + _, err = e.K8s.CoreV1().Nodes().Patch(context.Background(), candidates[0], types.MergePatchType, patch, metav1.PatchOptions{}) + return err + } + if err := cordon(true); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cordon(false) }) + waitBudget := func(name string, allowed int32) { + e.Eventually(t, time.Minute, func() (bool, string) { + pdb, err := e.K8s.PolicyV1().PodDisruptionBudgets(ns).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return pdb.Status.ObservedGeneration >= pdb.Generation && pdb.Status.DisruptionsAllowed == allowed, fmt.Sprintf("budget status: %+v", pdb.Status) + }) + } + waitBudget("gameplane-web", 1) + pods, err := webPods() + if err != nil { + t.Fatal(err) + } + var remaining string + for _, pod := range pods { + if pod.Spec.NodeName == candidates[0] { + if err := e.K8s.CoreV1().Pods(ns).EvictV1(t.Context(), &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Name: pod.Name, Namespace: ns}}); err != nil { + t.Fatal(err) + } + } else { + remaining = pod.Name + } + } + waitBudget("gameplane-web", 0) + if err := e.K8s.CoreV1().Pods(ns).EvictV1(t.Context(), &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Name: remaining, Namespace: ns}}); !apierrors.IsTooManyRequests(err) { + t.Fatalf("second eviction bypassed web availability budget: %v", err) + } + if ready, err := e.PodIsReady(t.Context(), ns, remaining); err != nil || !ready { + t.Fatalf("remaining web replica lost readiness: %v", err) + } + if err := cordon(false); err != nil { + t.Fatal(err) + } + waitReady() + // The chart's API budget must permit the only API pod to be evicted. This + // isolated labeled fixture tests the PDB without stopping the suite's API. + apiLabels := map[string]string{"app.kubernetes.io/name": "gameplane-api", "app.kubernetes.io/instance": ns} + one := int32(1) + apiDeployment := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "gameplane-api", Namespace: ns}, Spec: appsv1.DeploymentSpec{Replicas: &one, Selector: &metav1.LabelSelector{MatchLabels: apiLabels}, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: apiLabels}, Spec: corev1.PodSpec{Containers: []corev1.Container{fixture}}}}} + if _, err := e.K8s.AppsV1().Deployments(ns).Create(t.Context(), apiDeployment, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + waitBudget("gameplane-api", 1) + apiPods, err := e.K8s.CoreV1().Pods(ns).List(t.Context(), metav1.ListOptions{LabelSelector: "app.kubernetes.io/name=gameplane-api"}) + if err != nil || len(apiPods.Items) != 1 { + t.Fatalf("API fixture pods: %v", err) + } + if err := e.K8s.CoreV1().Pods(ns).EvictV1(t.Context(), &policyv1.Eviction{ObjectMeta: metav1.ObjectMeta{Name: apiPods.Items[0].Name, Namespace: ns}}); err != nil { + t.Fatalf("API singleton PDB blocked drain: %v", err) + } +} diff --git a/test/e2e/restricted_workloads_e2e_test.go b/test/e2e/restricted_workloads_e2e_test.go new file mode 100644 index 000000000..823b1d403 --- /dev/null +++ b/test/e2e/restricted_workloads_e2e_test.go @@ -0,0 +1,300 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os/exec" + "strings" + "testing" + "time" + + "github.com/gopacket/gopacket/pcapgo" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// Dedicated kind bucket on both architectures: it upgrades namespace policy, +// checks real PSA admission, and waits for kubelet to rotate an actual token. +// It runs serially because it upgrades the shared Helm release. +func TestRestrictedWorkloads_NodeCaptureAndTokenRotation(t *testing.T) { + ctx := t.Context() + const ns, diagnosticNS = "gameplane-games", "gameplane-diagnostics" + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + name, template, incompatible := "restricted-game-"+suffix, "restricted-template-"+suffix, "restricted-incompatible-"+suffix + applyBusyboxTemplate(t, template) + // The template controller updates status immediately after creation. A + // field-scoped patch preserves that status without racing resourceVersion. + patch := []byte(`{"spec":{"security":{"runAsUser":1000,"runAsGroup":1000,"fsGroup":1000},"configFiles":[{"path":"cfg/profile.txt","template":"restricted-config\n"}]}}`) + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, template, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + applyBusyboxGameServer(t, ns, name, template) + t.Cleanup(func() { + if !t.Failed() { + return + } + logCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + out, err := envInstance.Kubectl(logCtx, "-n", ns, "logs", "pod/"+name+"-0", "-c", "config-init", "--tail=30") + t.Logf("config-init diagnostic: %v\n%s", err, out) + }) + waitGameContainerReady(t, ns, name+"-0", 3*time.Minute) + before, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if out, err := envInstance.KubectlExec(t, ns, "pod/"+name+"-0", "sh", "-c", "test $(stat -c %u /data/cfg/profile.txt) = 0 && test $(stat -c %g /data/cfg/profile.txt) = 1000 && echo before-upgrade >> /data/cfg/profile.txt"); err != nil { + t.Fatalf("legacy root config must be writable by the game through its shared group: %v %s", err, out) + } + if out, err := envInstance.KubectlExec(t, ns, "pod/"+name+"-0", "sh", "-c", "echo upgrade-marker > /data/marker && sync"); err != nil { + t.Fatalf("write data: %v %s", err, out) + } + helmUpgradeReuse(ctx, t, "podSecurity.enforceRestricted=true", "operator.agentTokenExpirationSeconds=600", "capture.enabled=true", "capture.nodeDiagnostics.enabled=true") + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + p, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return p.UID != before.UID && p.Annotations["gameplane.local/workload-security"] == "restricted", "waiting for profile upgrade" + }) + waitGameContainerReady(t, ns, name+"-0", 3*time.Minute) + requireAgentReady(t, ns, name) + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken || len(pod.Spec.InitContainers) != 1 || len(pod.Spec.EphemeralContainers) != 0 { + t.Fatalf("unexpected restricted pod: %+v", pod.Spec) + } + for _, c := range append(pod.Spec.Containers, pod.Spec.InitContainers...) { + sc := c.SecurityContext + if sc == nil || sc.RunAsNonRoot == nil || !*sc.RunAsNonRoot || sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation || sc.Capabilities == nil || len(sc.Capabilities.Add) != 0 || len(sc.Capabilities.Drop) != 1 || sc.Capabilities.Drop[0] != "ALL" || sc.SeccompProfile == nil || sc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Fatalf("%s is not Restricted", c.Name) + } + tokenMounted := false + for _, mount := range c.VolumeMounts { + if mount.Name == "agent-api-token" { + tokenMounted = true + } + } + if tokenMounted != (c.Name == "agent") { + t.Fatalf("%s token mount=%t", c.Name, tokenMounted) + } + } + if out, err := envInstance.KubectlExec(t, ns, "pod/"+name+"-0", "sh", "-c", "test ! -e /var/run/secrets/kubernetes.io/serviceaccount/token && test $(id -u) = 1000 && grep upgrade-marker /data/marker && grep restricted-config /data/cfg/profile.txt && echo writable >> /data/marker"); err != nil { + t.Fatalf("identity/config/data/token: %v %s", err, out) + } + // A root container is rejected by the real admission controller. + zero := int64(0) + _, err = envInstance.K8s.CoreV1().Pods(ns).Create(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "rejected-root-" + suffix}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "root", Image: "busybox:1.36", SecurityContext: &corev1.SecurityContext{RunAsUser: &zero}}}}}, metav1.CreateOptions{}) + if !apierrors.IsForbidden(err) || !strings.Contains(err.Error(), "PodSecurity") { + t.Fatalf("PSA did not reject root: %v", err) + } + // Hash only: never return the bearer token in CI output. This is the kind + // host fixture inspecting kubelet's projection, not an extra game mount. + tokenPath := "/var/lib/kubelet/pods/" + string(pod.UID) + "/volumes/kubernetes.io~projected/agent-api-token/token" + tokenHash := func() (string, error) { + cctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + out, err := exec.CommandContext(cctx, "docker", "exec", pod.Spec.NodeName, "sha256sum", tokenPath).CombinedOutput() + if err != nil { + return "", fmt.Errorf("hash projected token: %w", err) + } + fields := strings.Fields(string(out)) + if len(fields) == 0 { + return "", fmt.Errorf("empty token hash") + } + return fields[0], nil + } + initialHash, err := tokenHash() + if err != nil { + t.Fatal(err) + } + initialTokenObserved := time.Now() + applyBusyboxTemplate(t, incompatible) + applyBusyboxGameServer(t, ns, incompatible, incompatible) + envInstance.Eventually(t, time.Minute, func() (bool, string) { + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, incompatible, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + phase, _, _ := unstructured.NestedString(gs.Object, "status", "phase") + if phase != "Failed" { + return false, "incompatible template not rejected" + } + _, err = envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, incompatible, metav1.GetOptions{}) + return apierrors.IsNotFound(err), "incompatible template reached workload admission" + }) + envInstance.BootstrapAdmin(t, adminUsername, adminPassword) + cli := envInstance.APIClient(t, adminUsername, adminPassword) + defer cli.Close() + post := func(path string, body any, want int) []byte { + resp, raw, err := cli.Post(path, body) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != want { + t.Fatalf("POST %s: %d %s", path, resp.StatusCode, raw) + } + return raw + } + post("/servers/"+name+":capture-enable", nil, http.StatusOK) + post("/servers/"+name+":capture-node-enable", nil, http.StatusOK) + envInstance.Eventually(t, time.Minute, func() (bool, string) { + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + ready, _, _ := unstructured.NestedBool(gs.Object, "status", "capture", "ready") + return ready, "capture not ready" + }) + start := func(duration int) *unstructured.Unstructured { + raw := post("/servers/"+name+":capture-start", map[string]any{"filter": "tcp port 12345", "maxDurationSeconds": duration, "maxSizeBytes": int64(4 << 20), "ttlSecondsAfterFinished": 3600}, http.StatusAccepted) + var response struct { + CaptureID string `json:"captureId"` + } + if err := json.Unmarshal(raw, &response); err != nil || response.CaptureID == "" { + t.Fatalf("capture response: %s %v", raw, err) + } + nc, err := envInstance.Dyn.Resource(networkCaptureGVR).Namespace(ns).Get(ctx, response.CaptureID, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _ = envInstance.Dyn.Resource(networkCaptureGVR).Namespace(ns).Delete(cleanupCtx, nc.GetName(), metav1.DeleteOptions{}) + }) + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + pods, err := envInstance.K8s.CoreV1().Pods(diagnosticNS).List(ctx, metav1.ListOptions{LabelSelector: "gameplane.local/capture-uid=" + string(nc.GetUID()) + ",gameplane.local/capture-component=worker"}) + if err != nil { + return false, err.Error() + } + for _, p := range pods.Items { + if p.Status.Phase == corev1.PodRunning { + if p.Spec.NodeName != pod.Spec.NodeName || !p.Spec.HostPID || p.Spec.AutomountServiceAccountToken == nil || *p.Spec.AutomountServiceAccountToken { + t.Fatal("worker node/privilege/token mismatch") + } + return true, "" + } + } + return false, "waiting for bound worker" + }) + return nc + } + nc := start(60) + traffic := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "restricted-traffic-" + suffix}, Spec: corev1.PodSpec{RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{{Name: "traffic", Image: "busybox:1.36", Command: []string{"sh", "-c", "for i in $(seq 1 20); do nc -w 1 " + pod.Status.PodIP + " 12345 11*time.Minute, "waiting for rotation and original token expiry" + }) + rotatedAt := time.Now() + envInstance.Eventually(t, time.Minute, func() (bool, string) { + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + s, _, _ := unstructured.NestedString(gs.Object, "status", "agent", "lastHeartbeat") + heartbeat, err := time.Parse(time.RFC3339, s) + return err == nil && heartbeat.After(rotatedAt), "no post-rotation heartbeat" + }) + // Deleting the owning server garbage-collects both retained artifacts and + // the canceled capture across namespaces through each capture's finalizer. + if err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + for _, capture := range []*unstructured.Unstructured{nc, second} { + selector := "gameplane.local/capture-uid=" + string(capture.GetUID()) + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + _, err := envInstance.Dyn.Resource(networkCaptureGVR).Namespace(ns).Get(ctx, capture.GetName(), metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + return false, "capture finalizer still present" + } + pods, err := envInstance.K8s.CoreV1().Pods(diagnosticNS).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return false, err.Error() + } + pvcs, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(diagnosticNS).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return false, err.Error() + } + return len(pods.Items) == 0 && len(pvcs.Items) == 0, "diagnostic storage/pods remain" + }) + } +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index abf7cda91..486f3f813 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -6,12 +6,33 @@ ## Purpose +### Restricted workload drill (#714) + +The dedicated `restricted-workloads` bucket runs on amd64 and arm64. It +upgrades a compatible existing game to the real enforced PSA profile, asserts +nonroot config-init/writable data preservation, rejects root admission and an +incompatible template, and checks agent-only credential mounts. It observes a +kubelet-projected token hash change (never logs tokens) and a heartbeat after +the original 600s token expires. Node capture checks exercise audited per-server +opt-in, traffic-bearing PCAP download, unchanged game Pod UID/restart counts, +worker cleanup on completion/revocation, and cross-namespace resource cleanup +after GameServer deletion. Local work compiles test sources only; CI executes +the drill and all unit/chart regressions. + The `test/e2e` module is Gameplane's end-to-end test suite: it stands up a real (kind-based) Kubernetes cluster, installs the Helm chart, and drives the operator/api/agent/sentinel stack as real components rather than mocks — CRDs applied through the typed/dynamic client, HTTP calls through a real port-forward, RCON/wire-protocol joins against real game server containers. Where the other modules' test tiers (unit, envtest) prove a component's own logic in isolation, this module proves the whole product works together on a cluster shaped like the ones users actually run. It was one of three `go.work` modules without CI lint coverage until this feature (`api`, `agent`, and `test/e2e` — see FR-001); all 13 `go.work` members (`netguard`, `gameaction`, `gameproto`, `operator`, `agent`, `api`, `sentinel`, `audit-syslog-bridge`, `telemetry-receiver`, `mcp-server`, `svcutil`, `tunnel`, and `test/e2e`) are golangci-lint gated in CI now — see the `lint` job's matrix in `.github/workflows/ci.yaml`. `test/e2e` lints with `--build-tags=e2e`, same tier as `operator`/`api`'s `--build-tags=envtest`, run as their own step so the catch-all lint step excludes all three and nothing is linted twice. ## Responsibilities +The operator bucket includes `TestGameServer_TemplateOnRequestRetainsData`: a +live busybox workload keeps its Pod and old rendered config after a shared +template update, then admits the exact desired digest and verifies a replacement +Pod, the same PVC UID, preserved save marker and updated config-init output. +Operator regression sources cover lifecycle exclusion, old-template graceful +stop, status reset/leader recovery, exact request binding, snapshot integrity, +complete pinned render inputs and source-template preservation for CSI restore. + - Boot and tear down (or reuse) a kind cluster, install the working-tree Helm chart, and wait for every component to come up healthy. - Exercise the full CRD lifecycle for all 8 kinds (GameServer, GameTemplate, Backup, BackupSchedule, Restore, Module, ModuleSource, Cluster) against a real API server and operator. - Drive the REST + WebSocket API as a real HTTP client would: login, session/CSRF handling, RBAC matrix checks, console/PTY/log streaming, file upload/download. @@ -40,6 +61,17 @@ are removed through test cleanup. CI sets `GAMEPLANE_E2E_REQUIRE_GATEWAY=1`, so missing second cluster fails instead of skipping the scenario. The multicluster Go invocation has a 45-minute deadline within the existing 60-minute job budget. +### Management placement and disruption acceptance + +The `management-placement` bucket uses the three-node `management` Kind profile +and performs no API login. `TestManagementPlacementAndDisruption` applies the +chart's web placement and API/web PDBs in an isolated namespace, with sleeping +fixture containers. It verifies real scheduling across labeled workers, +Unschedulable status for an impossible selector, one-at-a-time eviction during +a cordon, readiness of the surviving replica, and singleton API eviction. Node +labels and cordons are restored in cleanup. This validates scheduler/PDB behavior, +not application HA or storage migration. + ## Non-goals / boundaries - Does **not** replace the unit or envtest tiers for any other module — it exercises integration behavior, not the exhaustive branch coverage those tiers own (this module carries no `.testcoverage.yml` coverage gate). @@ -74,7 +106,7 @@ This is a name-matching contract, not a structural one: ## Conventions (already normative via `CLAUDE.md`, restated here for the module's own record) -- **`t.Parallel()`**: tests call it unless shared infrastructure requires serialization. The `ratelimit` bucket deliberately drains the shared login limiter; gateway parity temporarily changes the shared remote Helm release and trust. These are documented exceptions. +- **`t.Parallel()`**: tests call it unless shared infrastructure requires serialization. The `ratelimit` bucket deliberately drains the shared login limiter; gateway parity temporarily changes the shared remote Helm release and trust; `restricted-workloads` upgrades the shared release and namespace admission policy. These are documented exceptions. - **Per-test unique resource names**: CRs, namespaces, and users a test creates get a name unique to that test run (commonly time-based, e.g. `CreateUser`'s `fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())`) so parallel tests never collide on a shared cluster. - **`ociPushMu`**: a package-level mutex guarding tests that push through the shared, fixed-name `oras-push` Job (`module_e2e_test.go`, `module_verify_e2e_test.go`). Because the Job's name and namespace are fixed fixtures, two tests racing to apply/wait on it would stomp each other; `ociPushMu.Lock()`/`Unlock()` serializes them. - **`ensureResticRepo(t)`**: guards tests that run a backup against the shared restic repository fixture (`backup_e2e_test.go`, `backupschedule_e2e_test.go`, `failure_paths_e2e_test.go`) — restic repo initialization is not safe to race. diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index 017d1f88f..f5297e1ed 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -16,6 +16,8 @@ "test/e2e/suite-components.json" ], "suites": [ + { "tests": "^TestRestrictedWorkloads_", "components": ["operator", "api", "agent", "capture-sidecar"] }, + { "tests": "^TestManagementPlacementAndDisruption$", "components": ["operator", "api", "agent"] }, { "tests": "^TestHelmInstall_", "components": ["*"] }, { "tests": "^TestCRD_Validation_", "components": ["operator"] }, { "tests": "^TestGameServer_", "components": ["operator", "api", "agent"] }, diff --git a/test/e2e/template_rollout_e2e_test.go b/test/e2e/template_rollout_e2e_test.go new file mode 100644 index 000000000..0f6e7c046 --- /dev/null +++ b/test/e2e/template_rollout_e2e_test.go @@ -0,0 +1,121 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" +) + +// A real workload remains unchanged while a new shared template is installed, +// then adopts only on an exact request while retaining its PVC and saved data. +func TestGameServer_TemplateOnRequestRetainsData(t *testing.T) { + t.Parallel() + ctx := context.Background() + const ns, name, template = "gameplane-games", "e2e-template-stage", "e2e-template-stage" + applyBusyboxTemplate(t, template) + setConfigFile := func(contents string) { + t.Helper() + // The template controller updates status concurrently with both + // spec changes. Retry the complete read-modify-write on conflict. + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + tmpl, err := envInstance.Dyn.Resource(gameTemplateGVR).Get(ctx, template, metav1.GetOptions{}) + if err != nil { + return err + } + if err := unstructured.SetNestedSlice(tmpl.Object, []any{map[string]any{"path": "revision.txt", "template": contents}}, "spec", "configFiles"); err != nil { + return err + } + _, err = envInstance.Dyn.Resource(gameTemplateGVR).Update(ctx, tmpl, metav1.UpdateOptions{}) + return err + }); err != nil { + t.Fatal(err) + } + } + setConfigFile("old") + gs := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", "metadata": map[string]any{"name": name, "namespace": ns}, "spec": map[string]any{"templateRef": map[string]any{"name": template}, "templateRollout": map[string]any{"policy": "OnRequest"}}}} + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Create(ctx, gs, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + get := func() *unstructured.Unstructured { + t.Helper() + obj, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return obj + } + var old string + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + obj := get() + running, _, _ := unstructured.NestedString(obj.Object, "status", "templateRevision", "running") + effective, _, _ := unstructured.NestedString(obj.Object, "status", "templateRevision", "effective") + old = effective + return effective != "" && running == effective && obj.GetAnnotations()["operations.gameplane.local/owner"] == "", "waiting for initial pinned workload" + }) + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + oldPod := pod.UID + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name+"-data", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + oldPVC := pvc.UID + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, name+"-0", "-c", "game", "--", "sh", "-c", "printf retained > /data/save-marker; cat /data/revision.txt"); err != nil || strings.TrimSpace(out) != "old" { + t.Fatalf("seed persisted data: %s %v", out, err) + } + setConfigFile("new") + var desired string + envInstance.Eventually(t, time.Minute, func() (bool, string) { + obj := get() + desired, _, _ = unstructured.NestedString(obj.Object, "status", "templateRevision", "desired") + effective, _, _ := unstructured.NestedString(obj.Object, "status", "templateRevision", "effective") + return desired != "" && desired != old && effective == old, "waiting for pending revision" + }) + // Check after the controller reported processing the update, rather than + // relying on an arbitrary sleep before asserting an unchanged Pod. + pod, err = envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil || pod.UID != oldPod { + t.Fatalf("catalog update replaced staged Pod: %v", err) + } + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, name+"-0", "-c", "game", "--", "cat", "/data/revision.txt"); err != nil || strings.TrimSpace(out) != "old" { + t.Fatalf("pending config changed: %s %v", out, err) + } + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"templateRollout": map[string]any{"requestedRevision": desired}}}) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + obj := get() + running, _, _ := unstructured.NestedString(obj.Object, "status", "templateRevision", "running") + current, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return running == desired && current.UID != oldPod && obj.GetAnnotations()["operations.gameplane.local/owner"] == "", fmt.Sprintf("running revision=%s", running) + }) + pvc, err = envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name+"-data", metav1.GetOptions{}) + if err != nil || pvc.UID != oldPVC { + t.Fatalf("adoption replaced PVC: %v", err) + } + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, name+"-0", "-c", "game", "--", "sh", "-c", "cat /data/save-marker; printf ':'; cat /data/revision.txt"); err != nil || strings.TrimSpace(out) != "retained:new" { + t.Fatalf("persisted contents after adoption: %s %v", out, err) + } +} diff --git a/test/e2e/upgrade_e2e_test.go b/test/e2e/upgrade_e2e_test.go index 25a1221c9..65d9007b9 100644 --- a/test/e2e/upgrade_e2e_test.go +++ b/test/e2e/upgrade_e2e_test.go @@ -115,6 +115,12 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { } // ---- 3. upgrade to the working tree ---------------------------------- + // Exercise the SSA migration even when the published fixture already uses + // Recreate: Kubernetes defaults rollingUpdate on this merge patch. + if out, err := envInstance.Kubectl(ctx, "-n", "gameplane-system", "patch", "deployment", "gameplane-operator", + "--type=merge", "-p", `{"spec":{"strategy":{"type":"RollingUpdate"}}}`); err != nil { + t.Fatalf("seed operator RollingUpdate strategy: %v\n%s", err, out) + } upgrade := exec.CommandContext(ctx, "helm", "upgrade", "gameplane", filepath.Join(repoRoot, "charts", "gameplane"), @@ -133,6 +139,13 @@ func TestUpgrade_FromPreviousRelease(t *testing.T) { if out, err := upgrade.CombinedOutput(); err != nil { t.Fatalf("helm upgrade to the working-tree chart failed: %v\n%s", err, out) } + operator, err := envInstance.K8s.AppsV1().Deployments("gameplane-system").Get(ctx, "gameplane-operator", metav1.GetOptions{}) + if err != nil { + t.Fatalf("read upgraded operator: %v", err) + } + if operator.Spec.Strategy.Type != "Recreate" || operator.Spec.Strategy.RollingUpdate != nil { + t.Fatalf("operator strategy migration retained RollingUpdate fields: %+v", operator.Spec.Strategy) + } // ---- 4. the CRDs were actually updated ------------------------------- diff --git a/web/specs.md b/web/specs.md index d24b3f964..2eaa729be 100644 --- a/web/specs.md +++ b/web/specs.md @@ -10,6 +10,10 @@ The Gameplane dashboard is a React SPA providing a UI layer over the Gameplane A ## Responsibilities +GameServer domain types include optional `spec.templateRollout` policy/exact +revision request and `status.templateRevision` desired/effective/running digests. +These are CRD/API contracts; no new rollout controls or visual flow are defined. + - Render all authenticated Gameplane CRDs and operations as React pages and interactive forms - Handle user authentication (local + OIDC) and session management (cookies + CSRF tokens) - Fetch and cache data via TanStack Query; invalidate caches on Kubernetes watch events (SSE) diff --git a/web/src/components/backups/BackupDetailDrawer.test.tsx b/web/src/components/backups/BackupDetailDrawer.test.tsx index 8e5056db3..835470dd7 100644 --- a/web/src/components/backups/BackupDetailDrawer.test.tsx +++ b/web/src/components/backups/BackupDetailDrawer.test.tsx @@ -8,6 +8,34 @@ import { makeBackup } from "@/test/factories"; import { BackupDetailDrawer } from "./BackupDetailDrawer"; describe("BackupDetailDrawer", () => { + it("shows captured and excluded claims independently of copy success", async () => { + server.use(http.get("/backups/coverage", () => HttpResponse.json(makeBackup({ + metadata: { name: "coverage" }, + status: { + phase: "Succeeded", snapshotID: "copy", + conditions: [{ type: "Consistency", status: "False", reason: "UnsupportedRawCopy", message: "The game does not support quiesce." }], + captureManifest: { state: "Recorded", recordedAt: "2026-10-21T12:00:00Z", volumes: [ + { claimName: "game-data", claimUID: "world-uid", mountPaths: ["/world"], included: true, classification: "data", reason: "PrimaryDataClaim" }, + { claimName: "game-extra-install", mountPaths: ["/serverfiles"], included: false, classification: "rebuildable", reason: "NotCovered" }, + { claimName: "game-mods", mountPaths: ["/world/mods"], included: false, classification: "unclassified", reason: "NotCovered" }, + ] }, + }, + })))); + renderWithQuery( {}} onRestore={() => {}} />); + expect(await screen.findByText("The game does not support quiesce.")).toBeInTheDocument(); + expect(screen.getByText("game-data")).toBeInTheDocument(); + expect(screen.getByText("/world/mods")).toBeInTheDocument(); + expect(screen.getAllByText("Not covered")).toHaveLength(2); + expect(screen.getByText("Rebuildable")).toBeInTheDocument(); + }); + + it("shows legacy coverage as unknown without inventing a current layout", async () => { + server.use(http.get("/backups/legacy", () => HttpResponse.json(makeBackup({ metadata: { name: "legacy" }, status: { phase: "Succeeded" } })))); + renderWithQuery( {}} onRestore={() => {}} />); + expect(await screen.findByText(/Capture-time volume metadata is unavailable/)).toBeInTheDocument(); + expect(screen.queryByText("Included")).not.toBeInTheDocument(); + }); + it("does not render content when name is null", () => { renderWithQuery( {}} onRestore={() => {}} />); expect(screen.queryByText(/Backup details/i)).not.toBeInTheDocument(); diff --git a/web/src/components/backups/BackupDetailDrawer.tsx b/web/src/components/backups/BackupDetailDrawer.tsx index 7e9422231..eff9875e1 100644 --- a/web/src/components/backups/BackupDetailDrawer.tsx +++ b/web/src/components/backups/BackupDetailDrawer.tsx @@ -9,6 +9,7 @@ import type { Backup } from "@/types"; import { PhaseChip } from "@/components/ui/PhaseChip"; import { ErrorBanner } from "./ErrorBanner"; import { useOptionalIntl } from "@/lib/i18n/format"; +import { useConditionMessage } from "@/lib/i18n/conditionMessage"; interface Props { target?: ResourceTarget; @@ -44,6 +45,8 @@ export function BackupDetailDrawer({ name, ns, onClose, onRestore, target: expli const restorable = backup?.status?.phase === "Succeeded" && Boolean(backup.status.snapshotID); + const consistency = useConditionMessage(backup?.status?.conditions?.find((c) => c.type === "Consistency")); + const manifest = backup?.status?.captureManifest; return ( {backup.spec.serverRef.name} + + {consistency ?? intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.consistency_unknown", defaultMessage: "Consistency has not been reported.", description: "Backup details: no achieved-consistency condition is available" })} + + + {manifest?.state === "Recorded" ? ( +
+ {manifest.recordedAt &&
{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.coverage_recorded", defaultMessage: "Recorded: {time}", description: "Backup details: timestamp of the immutable capture manifest" }, { time: manifest.recordedAt })}
} +

{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.coverage_scope", defaultMessage: "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", description: "Backup details: explains the current single-volume backup scope and rebuildable declarations" })}

+
    + {manifest.volumes?.map((volume) => ( +
  • +
    {volume.claimName}
    +
    {volume.included + ? intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.included", defaultMessage: "Included", description: "Backup volume: this claim is selected for copying; success is reported separately" }) + : intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.not_covered", defaultMessage: "Not covered", description: "Backup volume: contents of this claim are not included" })}
    +
    {intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.classification", defaultMessage: "{role, select, data {Data} rebuildable {Rebuildable} other {Unclassified}}", description: "Backup volume: optional declaration from the module or server at capture time" }, { role: volume.classification })}
    + {volume.mountPaths?.map((path) =>
    {path}
    )} + {volume.reason === "RetainedUnmountedClaim" &&
    {intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.retained_unmounted", defaultMessage: "Retained claim; not mounted in the captured workload.", description: "Backup volume: an older server-owned claim was retained but was not part of the observed pod layout" })}
    } +
    • + ))} +
+
+ ) : intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.legacy_coverage", defaultMessage: "Capture-time volume metadata is unavailable. Inspect the original backup and module configuration before recovery.", description: "Backup details: older backups have no reliable volume manifest, so the current template is not used to invent coverage" })} + {backup.status?.snapshotID ?? intl.formatMessage({ id: "web.common.empty_value", defaultMessage: "—", description: "Shown in place of a value that is not set" })} diff --git a/web/src/types.ts b/web/src/types.ts index 880c2b72c..47b71ff99 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -541,6 +541,7 @@ export interface GameServer { metadata: ObjectMeta & { resourceVersion?: string }; spec: { templateRef: { name: string }; + templateRollout?: { policy?: "Automatic" | "OnRequest"; requestedRevision?: string }; suspend?: boolean; // Seconds the operator waits for the template's stop sequence to run // over RCON before scaling the pod down (soft-stop). Range 0–600, @@ -576,6 +577,7 @@ export interface GameServer { }; status?: { phase?: GameServerPhase; + templateRevision?: { desired?: string; effective?: string; running?: string }; endpoints?: GameServerEndpoint[]; agent?: { lastHeartbeat?: string; @@ -644,6 +646,23 @@ export interface ServerEvent { count: number; } +export interface BackupCaptureManifest { + state: "Recorded" | "LegacyUnknown"; + recordedAt?: string; + serverUID?: string; + strategy?: string; + layoutSource?: string; + quiesceOutcome?: string; + volumes?: Array<{ + claimName: string; + claimUID?: string; + mountPaths?: string[]; + included: boolean; + classification: "data" | "rebuildable" | "unclassified"; + reason: string; + }>; +} + export interface Backup { metadata: ObjectMeta; // strategy: restic-snapshot writes to a restic repo; volume-snapshot takes a @@ -655,6 +674,8 @@ export interface Backup { completionTime?: string; snapshotID?: string; size?: string; + captureManifest?: BackupCaptureManifest; + conditions?: Array<{ type: string; status: string; reason?: string; message?: string }>; }; }