diff --git a/agent/cmd/main.go b/agent/cmd/main.go index 8ce7f15f8..0b5c86d44 100644 --- a/agent/cmd/main.go +++ b/agent/cmd/main.go @@ -209,6 +209,13 @@ func main() { protected.Use(authCheck.Middleware, target.Guard(serverUID)) mountProtected(protected) }) + // Backups bind save-off/save-on to the process they prepared. The service + // may already route to a replacement pod, which must reject the old UID. + r.Route("/v1/pods/{uid}", func(protected chi.Router) { + protected.Use(authCheck.Middleware, target.Guard(envOr("GAMEPLANE_POD_UID", "")), middleware.Timeout(30*time.Second)) + protected.Get("/identity", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) }) + quiesce.Mount(protected, rconClient, gameName, quiesceSpec) + }) srv := &http.Server{ Addr: addr, diff --git a/agent/internal/quiesce/quiesce_test.go b/agent/internal/quiesce/quiesce_test.go index 777ba9ade..62aca43a4 100644 --- a/agent/internal/quiesce/quiesce_test.go +++ b/agent/internal/quiesce/quiesce_test.go @@ -12,6 +12,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/GameplanePanel/gameplane/agent/internal/caps" + "github.com/GameplanePanel/gameplane/agent/internal/target" ) type fakeRcon struct { @@ -86,6 +87,33 @@ func newTestRouter(rc Rcon, spec *caps.Quiesce) *chi.Mux { return r } +func TestPodBoundRoutesRejectReplacementBeforeRCON(t *testing.T) { + for _, actual := range []string{"original", "replacement", ""} { + t.Run("pod="+actual, func(t *testing.T) { + rc := &fakeRcon{} + r := chi.NewRouter() + r.Route("/v1/pods/{uid}", func(bound chi.Router) { + bound.Use(target.Guard(actual)) + Mount(bound, rc, "test", minecraftQuiesceSpec()) + }) + for _, path := range []string{"quiesce", "unquiesce"} { + response := httptest.NewRecorder() + r.ServeHTTP(response, httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/v1/pods/original/"+path, nil)) + if actual == "original" { + if response.Code != http.StatusOK { + t.Fatalf("matching process rejected: %d", response.Code) + } + } else if response.Code != http.StatusNotFound { + t.Fatalf("wrong process admitted: %d", response.Code) + } + } + if (len(rc.calls()) > 0) != (actual == "original") { + t.Fatalf("commands reached wrong process: %v", rc.calls()) + } + }) + } +} + func decodeResponse(t *testing.T, body []byte) response { t.Helper() var got response diff --git a/agent/internal/target/target.go b/agent/internal/target/target.go index 6a1300631..3e79a3944 100644 --- a/agent/internal/target/target.go +++ b/agent/internal/target/target.go @@ -1,4 +1,4 @@ -// Package target binds versioned agent routes to an immutable GameServer UID. +// Package target binds versioned agent routes to an immutable resource UID. package target import ( @@ -7,12 +7,13 @@ import ( "github.com/go-chi/chi/v5" ) -// Guard rejects a route for any server other than the one that started this -// agent. Legacy routes stay available for local callers during rolling upgrades. -func Guard(serverUID string) func(http.Handler) http.Handler { +// Guard rejects a route for any resource other than the one that started this +// agent (GameServer or Pod, according to the route). Legacy routes stay available +// for local callers during rolling upgrades. +func Guard(resourceUID string) func(http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - if serverUID == "" || chi.URLParam(req, "uid") != serverUID { + if resourceUID == "" || chi.URLParam(req, "uid") != resourceUID { http.NotFound(w, req) return } diff --git a/agent/specs.md b/agent/specs.md index af1f57529..8c716bb9c 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -177,6 +177,8 @@ Resource usage env vars (set by the operator): | `/players/banned` | GET | Currently banned players; response: `[ { name, reason?, source? } ]`; 502 if RCON unavailable | | `/quiesce` | POST | Pause auto-saves before snapshot; response: `{ quiesced, reason? }` (boolean, reason optional); unsupported games return `quiesced: false` with a reason | | `/unquiesce` | POST | Resume auto-saves after snapshot; response: `{ quiesced, reason? }` | +| `/v1/pods/{uid}/identity` | GET | Operator preflight: 204 only when `{uid}` matches `GAMEPLANE_POD_UID`; never mutates game state | +| `/v1/pods/{uid}/quiesce`, `/v1/pods/{uid}/unquiesce` | POST | Pod-bound backup RPCs with the same response contract as the legacy routes; reject missing/mismatched Pod UID with 404 before RCON | | `/lifecycle/stop` | POST | Run stop sequence before scale-to-zero; RCON connection drop is the expected outcome | | `/actions/run` | POST | Execute module-declared operator action; request: `{ id, params }`; response: `{ ok, raw? }` | | `/status` | GET | Live game metrics (module-declared); response: bare JSON array `[ { id, displayName?, value, unit? } ]` | @@ -209,6 +211,7 @@ All endpoints on the `--addr` control mux, except `/healthz`, return `401 Unauth - **Extracted mod files stay world-readable**: `mods.moduleFileMode` is `0o644`, not a tighter `0o600`, because the mods volume is shared with the game container, which runs as whatever uid its image needs — a different uid than the agent's own. At `0o600` the game process couldn't read its own mods. This is the rationale behind the `.golangci.yml` gosec G302 exclusion scoped to `agent/internal/mods/mods.go`. - **Resource usage is in-pod**: The `usage` package reads from `/proc` or cgroups; no external metrics pipeline required. Cgroup mode is a fallback for older clusters; proc mode (default in production) requires the operator to set `ShareProcessNamespace: true`. - **Module capabilities drive behavior**: Every game-specific handler (players, quiesce, lifecycle, status, actions) reads its config from `--capabilities` (JSON unmarshaled into `caps.Spec`). New games require no agent code change. +- **Backup process identity**: The operator injects `GAMEPLANE_POD_UID` from `metadata.uid` through the DownwardAPI. Pod-bound backup routes require the existing mTLS authentication and check this UID at the final network hop, including when the Service resolves to a replacement pod. Missing UID fails closed. Deploy the updated agent image and allow pod rollout when upgrading the operator; older agents return 404 and the operator does not fall back to unbound routes. Legacy endpoints remain available to existing callers during rollout. - **RCON connection errors are graceful**: A lost RCON connection does not crash the agent. `console`, `players`, `quiesce`, and `lifecycle` handlers catch connection errors and return appropriate HTTP status (e.g., `502 Bad Gateway`). - **Source RCON sends complete frames**: Each AUTH or command frame is submitted in one socket write, with its existing byte-size limit, request ID, type, body, and two NUL terminators preserved. This avoids separate header-only writes rejected by vanilla Minecraft's RCON reader. A short write is an error and drops the connection without replaying the command; response grace and healthy connection reuse remain unchanged. - **Log streams are tail-only**: The `logs` package does not support random-access reads. It streams from the current end (live mode) or from file start (backlog mode); clients must handle partial output and reconnection. diff --git a/api/internal/handlers/resources_annotations_test.go b/api/internal/handlers/resources_annotations_test.go index 457d5ebfb..a615dc2fe 100644 --- a/api/internal/handlers/resources_annotations_test.go +++ b/api/internal/handlers/resources_annotations_test.go @@ -39,6 +39,7 @@ func TestGenericServerWritesCannotInjectLifecycleAnnotations(t *testing.T) { wipeRequestedAnnotation: "injected", "gameplane.local/wipe-data-completed": "forged", "gameplane.local/restore-guard": "forged", + "operations.gameplane.local/owner": "forged", "gameserver.gameplane.local/stop-requested-at": "forged", "restore.gameplane.local/created-by": "forged", ownerIDAnnotation: "999", @@ -51,7 +52,7 @@ func TestGenericServerWritesCannotInjectLifecycleAnnotations(t *testing.T) { t.Fatalf("write failed: %d %s", response.Code, response.Body) } got := serverAnnotations(t, k, "alpha") - for _, key := range []string{wipeRequestedAnnotation, "gameplane.local/wipe-data-completed", "gameplane.local/restore-guard", "gameserver.gameplane.local/stop-requested-at", "restore.gameplane.local/created-by"} { + for _, key := range []string{wipeRequestedAnnotation, "gameplane.local/wipe-data-completed", "gameplane.local/restore-guard", "operations.gameplane.local/owner", "gameserver.gameplane.local/stop-requested-at", "restore.gameplane.local/created-by"} { if _, found := got[key]; found { t.Errorf("generic write injected %s", key) } @@ -98,6 +99,7 @@ func TestGenericServerUpdatePreservesControllerState(t *testing.T) { wipeRequestedAnnotation: "pending", "gameplane.local/wipe-data-completed": "previous", "gameplane.local/backup-guard": "active-backup", + "operations.gameplane.local/owner": `{"kind":"Backup","name":"copy","uid":"copy-uid","targetUID":"server-uid","generation":1}`, "gameserver.gameplane.local/stop-requested-at": "original-time", "restore.gameplane.local/created-by": "original-restore", } diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index d902ea995..2b357a6ce 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: @@ -129,6 +129,75 @@ spec: status: description: BackupStatus is the observed state of a Backup. properties: + captureManifest: + description: |- + CaptureManifest is recorded once, before the first capture worker starts. + It describes persistent-volume coverage independently of copy success. + Missing on pre-upgrade backups; never reconstruct it from current layout. + properties: + layoutSource: + description: LayoutSource identifies the frozen workload used + for mount paths. + type: string + quiesceOutcome: + description: |- + QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + agent-unavailable, or unknown, independent of subsequent copy success. + type: string + recordedAt: + format: date-time + type: string + serverUID: + type: string + state: + description: |- + State is Recorded for new captures, LegacyUnknown for an existing worker + whose original layout was never recorded. + enum: + - Recorded + - LegacyUnknown + type: string + strategy: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + mountPaths: + items: + type: string + type: array + reason: + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - state + type: object completionTime: format: date-time type: string diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index d85406090..d0f930d29 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 4d6aadcbb..fa735d0c8 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index 95178e56e..0604421b6 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -2215,6 +2215,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index e5e3e1e3b..36dff4f75 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: @@ -2166,6 +2166,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 5ade28cee..bfeb260db 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 6c5dcbf36..deca18b51 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index cc113e7e8..30daf3e9f 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index b77cd5c25..480794bab 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index d902ea995..2b357a6ce 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: @@ -129,6 +129,75 @@ spec: status: description: BackupStatus is the observed state of a Backup. properties: + captureManifest: + description: |- + CaptureManifest is recorded once, before the first capture worker starts. + It describes persistent-volume coverage independently of copy success. + Missing on pre-upgrade backups; never reconstruct it from current layout. + properties: + layoutSource: + description: LayoutSource identifies the frozen workload used + for mount paths. + type: string + quiesceOutcome: + description: |- + QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + agent-unavailable, or unknown, independent of subsequent copy success. + type: string + recordedAt: + format: date-time + type: string + serverUID: + type: string + state: + description: |- + State is Recorded for new captures, LegacyUnknown for an existing worker + whose original layout was never recorded. + enum: + - Recorded + - LegacyUnknown + type: string + strategy: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + mountPaths: + items: + type: string + type: array + reason: + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - state + type: object completionTime: format: date-time type: string diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index d85406090..d0f930d29 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 4d6aadcbb..fa735d0c8 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index 95178e56e..0604421b6 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -2215,6 +2215,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index e5e3e1e3b..36dff4f75 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: @@ -2166,6 +2166,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 5ade28cee..bfeb260db 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 6c5dcbf36..deca18b51 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index cc113e7e8..30daf3e9f 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index b77cd5c25..480794bab 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: fc63992190029342d40e8b7cc96ce57b24527f03275df038524b0a3a3e22ae6e controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/docs/module-authoring.md b/docs/module-authoring.md index 1141965bb..dbd6a28fb 100644 --- a/docs/module-authoring.md +++ b/docs/module-authoring.md @@ -1636,7 +1636,35 @@ useful, so readiness falls back to TCP on the game port. `spec.storage.mountPath` is where the PVC is mounted inside the game container. Pick the path the game writes its world/config files to — -backups snapshot this whole directory. +backups copy the primary claim mounted here. Separate PVCs mounted beneath +that directory are not included. + +Extra persistent volumes can declare their recovery role: + +```yaml +spec: + storage: + extra: + - name: serverfiles + mountPath: /serverfiles + size: 10Gi + backupRole: rebuildable # or data +``` + +`backupRole: data` identifies recovery data; `rebuildable` declares that the +contents can be downloaded or reconstructed. Neither includes the extra claim +in today's single-volume backup. Omission remains valid and appears as +unclassified/not covered. Do not mark an installation volume rebuildable if it +also contains unique configuration or mods that cannot be reconstructed. +The GameServer's existing `spec.storage.extra` override can supply the same field. + +The Backup pins its volume list, claim UIDs, observed game-container mount paths +and these declarations before copying. Template changes do not rewrite that +record. Nested loader/mod claims and retained server-owned claims remain +explicitly uncovered. See [backup coverage](operator-operations.md#backup-consistency-and-volume-coverage). +The module editor schema is derived from the GameTemplate CRD by +`make module-schema`; publish its regenerated copy in the module repository +when opting modules into this field. ## Testing a new module locally diff --git a/docs/operator-operations.md b/docs/operator-operations.md new file mode 100644 index 000000000..88366705f --- /dev/null +++ b/docs/operator-operations.md @@ -0,0 +1,79 @@ +# Backup, restore and wipe ownership + +## Backup consistency and volume coverage + +A successful copy is not necessarily a complete server recovery point. +`status.conditions[type=Consistency]` reports the achieved result separately +from `Completed`: Quiesced means the agent confirmed the module's quiesce +commands before the completed copy; it does not prove those commands flush all +application state. UnsupportedRawCopy, RawCopy and AgentUnavailableRawCopy +describe restic copies without confirmed quiesce. CSI without confirmed +quiesce reports CrashConsistentSnapshot. StoppedSource means the source was +already stopped and its writers drained before capture; the backup did not stop +it. Failed or unfinished copies, including completed Jobs still awaiting a +snapshot ID, cannot report achieved quiesce or stopped-source consistency. + +`status.captureManifest` is recorded once before the first copy worker is +created, after lifecycle ownership and quiesce preparation. It lists the primary +`-data` claim as included and every other observed persistent claim as +uncovered, including extra volumes, nested mod/loader mounts and retained +server-owned claims. It records claim UIDs, observed mount paths, source layout +and optional `data`/`rebuildable` declarations. Included describes copy scope; +check the Backup's phase and snapshot ID for copy success. A declaration that +an uncovered claim is rebuildable is not a recovery guarantee. + +The API returns this status and the backup details drawer displays consistency +and coverage. Capture metadata is immutable through ordinary reconciliation: +editing a template later never changes the historical list. This release does +not add multi-volume restic, automatic game stopping or a stricter consistency +request policy. + +Pre-upgrade captures have missing metadata or `state: LegacyUnknown`; their +Consistency condition is Unknown. The operator never reconstructs their history +from the current template. Before recovery, inspect the original restic snapshot +or CSI source claim, recover the original module/server configuration and arrange +separate recovery of extra/mod volumes. Use an isolated recovery server to +verify those prerequisites before replacing a live world. + +## Shared operation ownership + +Gameplane serializes these operations per GameServer. All backups participate, +including backups with `quiesce: false` and CSI VolumeSnapshots. A held operation +also defers idle sleep/wake, restart, and pod-template rollout. Requests remain +pending and reconciliation resumes after the operation's cleanup completes. + +The GameServer annotation `operations.gameplane.local/owner` identifies the +operation kind, name, UID, admitted generation and target UID. A wipe is identified +by the GameServer UID and its request token. The existing backup, restore and wipe +owner annotations remain compatibility markers. A `recovery` array identifies +older owners whose workers or quiesce obligations must be cleaned up first. + +Deleting an operation requests cleanup. Restic Jobs are canceled with foreground +deletion; live and terminating worker pods must drain before the owner is released. +Quiesced backups also owe a successful save-on call. The existing bounded Backup +deletion retry may release its finalizer, but an unconfirmed save-on leaves the +durable owner behind for recovery by the next operation. Time alone never releases +ownership. A pending CSI capture retains the current capture-drain behavior; a +separate CSI timeout change is tracked in issue #719. + +If an operation waits unexpectedly, inspect the referenced Backup/Restore, its +conditions, Jobs and worker pods. Resolve terminating workers, repository failures +or agent connectivity before retrying. Removing owner annotations or force-removing +finalizers cannot establish that a worker has stopped. A failed or canceled restore +leaves its target suspended for inspection. + +Upgrades recognize existing owner annotations and import backup recovery lineage. +Already-running legacy quiesced copies continue being observed until their workers +finish and cleanup succeeds. Legacy raw workers also block new admission. Conflicting +legacy guards fail closed until their existing operations finish cleanup. Keep a +single active operator version during upgrade; older binaries do not honor the +shared owner. No new CRD fields or Kubernetes permissions are required. + +CSI restores create a new target and reserve it for the Restore. Initial startup +is allowed under that reservation so storage using WaitForFirstConsumer can bind. +Other operations wait until the restore completes. Existing data-PVC restores +continue to require an observed stopped workload and drained writers. + +Stopped-source preparation (#716), Restic preparation and quiesce deadlines (#717), +CSI capture deadlines (#719), and staged template adoption (#731) are separate +changes building on this ownership boundary. diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index 059470fd2..466a55d4a 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -175,6 +175,42 @@ "defaultMessage": "The backup or restore completed successfully.", "description": "Completed condition on a Backup or Restore that finished." }, + "condition.consistency.agentunavailablerawcopy": { + "defaultMessage": "No agent quiesce capability was configured. The primary volume was copied without pausing game writes.", + "description": "Backup achieved consistency: AgentUnavailableRawCopy." + }, + "condition.consistency.capturefailed": { + "defaultMessage": "The capture failed; no usable consistency guarantee was achieved.", + "description": "Backup achieved consistency: CaptureFailed." + }, + "condition.consistency.capturepending": { + "defaultMessage": "The capture has not completed; its achieved consistency is not yet known.", + "description": "Backup achieved consistency: CapturePending." + }, + "condition.consistency.crashconsistentsnapshot": { + "defaultMessage": "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed.", + "description": "Backup achieved consistency: CrashConsistentSnapshot." + }, + "condition.consistency.legacymetadataunknown": { + "defaultMessage": "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template.", + "description": "Backup achieved consistency: LegacyMetadataUnknown." + }, + "condition.consistency.quiesced": { + "defaultMessage": "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup.", + "description": "Backup achieved consistency: Quiesced." + }, + "condition.consistency.rawcopy": { + "defaultMessage": "The primary volume was copied without confirmed application quiesce.", + "description": "Backup achieved consistency: RawCopy." + }, + "condition.consistency.stoppedsource": { + "defaultMessage": "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup.", + "description": "Backup achieved consistency: StoppedSource." + }, + "condition.consistency.unsupportedrawcopy": { + "defaultMessage": "The game does not support quiesce. The primary volume was copied while it could still change.", + "description": "Backup achieved consistency: UnsupportedRawCopy." + }, "condition.datawipe.jobfailed": { "defaultMessage": "The data wipe job did not complete. The volume may not be fully cleared.", "description": "DataWipe condition when the wipe job fails." @@ -599,6 +635,10 @@ "defaultMessage": "The server could not be resumed after the backup. Its world may still have auto-save turned off.", "description": "Unquiesced condition when the unfreeze failed." }, + "condition.unquiesced.sourceprocessgone": { + "defaultMessage": "The original game process has exited. No resume command was needed.", + "description": "Unquiesced condition when the original source pod no longer runs." + }, "condition.unquiesced.unquiesced": { "defaultMessage": "The server was resumed after the backup.", "description": "Unquiesced condition when the world was unfrozen." @@ -1131,10 +1171,34 @@ "defaultMessage": "Backup details", "description": "Backup details drawer: heading at the top of the side panel for one backup; the backup name is shown below it" }, + "web.components.backups.backup_detail_drawer.classification": { + "defaultMessage": "{role, select, data {Data} rebuildable {Rebuildable} other {Unclassified}}", + "description": "Backup volume: optional declaration from the module or server at capture time" + }, "web.components.backups.backup_detail_drawer.completed": { "defaultMessage": "Completed", "description": "Backup details drawer: field label above the time the backup completed, shown in relative time" }, + "web.components.backups.backup_detail_drawer.consistency": { + "defaultMessage": "Achieved consistency", + "description": "Backup details: actual consistency reported by the controller, independent of successful copying" + }, + "web.components.backups.backup_detail_drawer.consistency_unknown": { + "defaultMessage": "Consistency has not been reported.", + "description": "Backup details: no achieved-consistency condition is available" + }, + "web.components.backups.backup_detail_drawer.coverage": { + "defaultMessage": "Volume coverage", + "description": "Backup details: persistent claims recorded before the backup copy began" + }, + "web.components.backups.backup_detail_drawer.coverage_recorded": { + "defaultMessage": "Recorded: {time}", + "description": "Backup details: timestamp of the immutable capture manifest" + }, + "web.components.backups.backup_detail_drawer.coverage_scope": { + "defaultMessage": "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", + "description": "Backup details: explains the current single-volume backup scope and rebuildable declarations" + }, "web.components.backups.backup_detail_drawer.delete": { "defaultMessage": "Delete", "description": "Backup detail drawer: delete button label" @@ -1143,6 +1207,18 @@ "defaultMessage": "Deleting…", "description": "Backup detail drawer: delete button label while the delete request is pending" }, + "web.components.backups.backup_detail_drawer.included": { + "defaultMessage": "Included", + "description": "Backup volume: this claim is selected for copying; success is reported separately" + }, + "web.components.backups.backup_detail_drawer.legacy_coverage": { + "defaultMessage": "Capture-time volume metadata is unavailable. Inspect the original backup and module configuration before recovery.", + "description": "Backup details: older backups have no reliable volume manifest, so the current template is not used to invent coverage" + }, + "web.components.backups.backup_detail_drawer.not_covered": { + "defaultMessage": "Not covered", + "description": "Backup volume: contents of this claim are not included" + }, "web.components.backups.backup_detail_drawer.phase": { "defaultMessage": "Phase", "description": "Backup details drawer: field label above the backup's current phase, shown as a status chip" @@ -1155,6 +1231,10 @@ "defaultMessage": "Restore backup", "description": "Backup details drawer: screen-reader name of the restore button in the drawer header, read instead of its visible word Restore; not visible on screen" }, + "web.components.backups.backup_detail_drawer.retained_unmounted": { + "defaultMessage": "Retained claim; not mounted in the captured workload.", + "description": "Backup volume: an older server-owned claim was retained but was not part of the observed pod layout" + }, "web.components.backups.backup_detail_drawer.server": { "defaultMessage": "Server", "description": "Backup details drawer: field label above the name of the server the backup belongs to" diff --git a/operator/api/v1alpha1/backup_types.go b/operator/api/v1alpha1/backup_types.go index d093916f4..c2ec70c00 100644 --- a/operator/api/v1alpha1/backup_types.go +++ b/operator/api/v1alpha1/backup_types.go @@ -17,11 +17,10 @@ const ( BackupPhaseFailed BackupPhase = "Failed" ) -// BackupFinalizer blocks a quiesced Backup's deletion until the operator has -// sent the matching unquiesce. Without it, deleting a Backup while -// quiesce-attempted=true drops the unquiesce entirely and the game is left -// with auto-save off indefinitely (F-048). Added only for Backups with -// spec.quiesce=true; see backup_forget.go's finalizeDelete. +// BackupFinalizer blocks deletion until copy workers have drained and any +// matching unquiesce has completed. Every Backup carries it, including raw +// copies and CSI snapshots, so deletion cannot release shared lifecycle +// ownership while a copy is still using the target. See finalizeDelete. const BackupFinalizer = "gameplane.local/backup-finalizer" // BackupSnapshotFinalizer blocks the deletion of a restic-strategy Backup until @@ -89,6 +88,12 @@ type LocalObjectRef struct { // BackupStatus is the observed state of a Backup. type BackupStatus struct { + // CaptureManifest is recorded once, before the first capture worker starts. + // It describes persistent-volume coverage independently of copy success. + // Missing on pre-upgrade backups; never reconstruct it from current layout. + // +optional + CaptureManifest *BackupCaptureManifest `json:"captureManifest,omitempty"` + // +optional Phase BackupPhase `json:"phase,omitempty"` @@ -129,6 +134,47 @@ type BackupStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } +// BackupCaptureManifest pins the layout and quiesce result at capture admission. +// Recorded coverage does not imply a successful or complete server backup. +type BackupCaptureManifest struct { + // State is Recorded for new captures, LegacyUnknown for an existing worker + // whose original layout was never recorded. + // +kubebuilder:validation:Enum=Recorded;LegacyUnknown + State string `json:"state"` + // +optional + RecordedAt *metav1.Time `json:"recordedAt,omitempty"` + // +optional + ServerUID string `json:"serverUID,omitempty"` + // +optional + Strategy string `json:"strategy,omitempty"` + // LayoutSource identifies the frozen workload used for mount paths. + // +optional + LayoutSource string `json:"layoutSource,omitempty"` + // QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + // agent-unavailable, or unknown, independent of subsequent copy success. + // +optional + QuiesceOutcome string `json:"quiesceOutcome,omitempty"` + // +optional + Volumes []BackupVolumeCoverage `json:"volumes,omitempty"` +} + +// BackupVolumeCoverage describes one persistent claim. Extra and loader/mod +// claims remain uncovered even when mounted beneath the primary data directory. +type BackupVolumeCoverage struct { + ClaimName string `json:"claimName"` + // ClaimUID is empty if the claim did not yet exist at admission. + // +optional + ClaimUID string `json:"claimUID,omitempty"` + // +optional + MountPaths []string `json:"mountPaths,omitempty"` + Included bool `json:"included"` + // Classification records the module/server declaration at admission; + // it does not prove that the volume can actually be reconstructed. + // +kubebuilder:validation:Enum=data;rebuildable;unclassified + Classification string `json:"classification"` + CoverageReason string `json:"reason"` +} + // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=bk // +kubebuilder:printcolumn:name="Server",type=string,JSONPath=`.spec.serverRef.name` diff --git a/operator/api/v1alpha1/gametemplate_types.go b/operator/api/v1alpha1/gametemplate_types.go index bea234aea..8001a2b27 100644 --- a/operator/api/v1alpha1/gametemplate_types.go +++ b/operator/api/v1alpha1/gametemplate_types.go @@ -946,6 +946,14 @@ type GameStorageSpec struct { // primary Storage.MountPath, since these exist precisely for directories // that share no safe common parent with it. type ExtraVolumeSpec struct { + // BackupRole describes recovery requirements without adding this volume + // to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + // Unset remains unclassified and not covered. Both backup strategies still + // capture only the primary data claim. + // +kubebuilder:validation:Enum=data;rebuildable + // +optional + BackupRole string `json:"backupRole,omitempty"` + // Name identifies the volume and forms part of its PVC name. Must be a // DNS-1123 label. // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 05926fd58..f7a291f7d 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -178,6 +178,32 @@ func (in *Backup) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupCaptureManifest) DeepCopyInto(out *BackupCaptureManifest) { + *out = *in + if in.RecordedAt != nil { + in, out := &in.RecordedAt, &out.RecordedAt + *out = (*in).DeepCopy() + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]BackupVolumeCoverage, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupCaptureManifest. +func (in *BackupCaptureManifest) DeepCopy() *BackupCaptureManifest { + if in == nil { + return nil + } + out := new(BackupCaptureManifest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupList) DeepCopyInto(out *BackupList) { *out = *in @@ -388,6 +414,11 @@ func (in *BackupSpec) DeepCopy() *BackupSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { *out = *in + if in.CaptureManifest != nil { + in, out := &in.CaptureManifest, &out.CaptureManifest + *out = new(BackupCaptureManifest) + (*in).DeepCopyInto(*out) + } if in.StartTime != nil { in, out := &in.StartTime, &out.StartTime *out = (*in).DeepCopy() @@ -420,6 +451,26 @@ func (in *BackupStatus) DeepCopy() *BackupStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupVolumeCoverage) DeepCopyInto(out *BackupVolumeCoverage) { + *out = *in + if in.MountPaths != nil { + in, out := &in.MountPaths, &out.MountPaths + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupVolumeCoverage. +func (in *BackupVolumeCoverage) DeepCopy() *BackupVolumeCoverage { + if in == nil { + return nil + } + out := new(BackupVolumeCoverage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BanListSpec) DeepCopyInto(out *BanListSpec) { *out = *in diff --git a/operator/cmd/main.go b/operator/cmd/main.go index 85263e901..48872122e 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -390,6 +390,7 @@ func main() { AgentCASecretName: agentCASecretName, AgentCASecretNamespace: agentCASecretNamespace, AgentClient: agentClient, + OperationQuiescer: agentClient, PodAttacher: &controller.StopAttachClient{ Config: mgr.GetConfig(), Clientset: kubernetes.NewForConfigOrDie(mgr.GetConfig()), @@ -437,6 +438,7 @@ func main() { Client: mgr.GetClient(), APIReader: mgr.GetAPIReader(), Scheme: mgr.GetScheme(), + OperationQuiescer: agentClient, ResticImage: resticImage, JobBackoffLimit: &backupJobBackoff32, JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds, diff --git a/operator/config/crd/gameplane.local_backups.yaml b/operator/config/crd/gameplane.local_backups.yaml index f96731731..649a29a0d 100644 --- a/operator/config/crd/gameplane.local_backups.yaml +++ b/operator/config/crd/gameplane.local_backups.yaml @@ -128,6 +128,75 @@ spec: status: description: BackupStatus is the observed state of a Backup. properties: + captureManifest: + description: |- + CaptureManifest is recorded once, before the first capture worker starts. + It describes persistent-volume coverage independently of copy success. + Missing on pre-upgrade backups; never reconstruct it from current layout. + properties: + layoutSource: + description: LayoutSource identifies the frozen workload used + for mount paths. + type: string + quiesceOutcome: + description: |- + QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + agent-unavailable, or unknown, independent of subsequent copy success. + type: string + recordedAt: + format: date-time + type: string + serverUID: + type: string + state: + description: |- + State is Recorded for new captures, LegacyUnknown for an existing worker + whose original layout was never recorded. + enum: + - Recorded + - LegacyUnknown + type: string + strategy: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + mountPaths: + items: + type: string + type: array + reason: + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - state + type: object completionTime: format: date-time type: string diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index 4be670ebc..f4ab3ceb1 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -2214,6 +2214,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index 13d8a1fa5..ce25ffca9 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -2165,6 +2165,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index e644cf041..b361725db 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -140,10 +140,22 @@ var rows = []row{ {types: []string{"Completed"}, reason: "Pending", english: "The backup is waiting to start.", description: "Completed condition on a Backup that has not started (dynamic: reason is the phase).", dynamic: true}, {types: []string{"Completed"}, reason: "Running", english: "The backup is running.", description: "Completed condition on a Backup in progress (dynamic: reason is the phase).", dynamic: true}, {types: []string{"Unquiesced"}, reason: "Unquiesced", english: "The server was resumed after the backup.", description: "Unquiesced condition when the world was unfrozen."}, + {types: []string{"Unquiesced"}, reason: "SourceProcessGone", english: "The original game process has exited. No resume command was needed.", description: "Unquiesced condition when the original source pod no longer runs."}, {types: []string{"Unquiesced"}, reason: "Failed", english: "The server could not be resumed after the backup. Its world may still have auto-save turned off.", description: "Unquiesced condition when the unfreeze failed."}, {types: []string{"SnapshotForgotten"}, reason: "Forgotten", english: "The snapshot was removed from the backup repository.", description: "SnapshotForgotten condition after a deleted Backup's snapshot was forgotten."}, {types: []string{"SnapshotForgotten"}, reason: "ForgetAbandoned", english: "The snapshot was not removed from the backup repository when the backup was deleted.", description: "SnapshotForgotten condition when the forget step was abandoned."}, + // Consistency is separate from copy completion and volume coverage. + {types: []string{"Consistency"}, reason: "CaptureFailed", english: "The capture failed; no usable consistency guarantee was achieved.", description: "Backup achieved consistency: CaptureFailed."}, + {types: []string{"Consistency"}, reason: "LegacyMetadataUnknown", english: "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template.", description: "Backup achieved consistency: LegacyMetadataUnknown."}, + {types: []string{"Consistency"}, reason: "CapturePending", english: "The capture has not completed; its achieved consistency is not yet known.", description: "Backup achieved consistency: CapturePending."}, + {types: []string{"Consistency"}, reason: "Quiesced", english: "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup.", description: "Backup achieved consistency: Quiesced."}, + {types: []string{"Consistency"}, reason: "StoppedSource", english: "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup.", description: "Backup achieved consistency: StoppedSource."}, + {types: []string{"Consistency"}, reason: "CrashConsistentSnapshot", english: "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed.", description: "Backup achieved consistency: CrashConsistentSnapshot."}, + {types: []string{"Consistency"}, reason: "UnsupportedRawCopy", english: "The game does not support quiesce. The primary volume was copied while it could still change.", description: "Backup achieved consistency: UnsupportedRawCopy."}, + {types: []string{"Consistency"}, reason: "AgentUnavailableRawCopy", english: "No agent quiesce capability was configured. The primary volume was copied without pausing game writes.", description: "Backup achieved consistency: AgentUnavailableRawCopy."}, + {types: []string{"Consistency"}, reason: "RawCopy", english: "The primary volume was copied without confirmed application quiesce.", description: "Backup achieved consistency: RawCopy."}, + // BackupSchedule {types: []string{"ScheduleValid"}, reason: "CronParseError", english: "The schedule is not a valid cron expression.", description: "ScheduleValid condition for an unparseable schedule."}, {types: []string{"ScheduleValid"}, reason: "CronValid", english: "The schedule is a valid cron expression.", description: "ScheduleValid condition for a good schedule."}, diff --git a/operator/internal/agent/client.go b/operator/internal/agent/client.go index d606c3c75..2b1c06923 100644 --- a/operator/internal/agent/client.go +++ b/operator/internal/agent/client.go @@ -14,6 +14,7 @@ import ( "fmt" "io" "net/http" + "net/url" "os" "time" ) @@ -119,6 +120,47 @@ func (c *Client) Unquiesce(ctx context.Context, namespace, server string) error return c.callQuiesce(ctx, namespace, server, "/unquiesce", false) } +// QuiesceEnabled reports whether this client can perform quiesce operations. +func (c *Client) QuiesceEnabled() bool { return c != nil && !c.Disabled } + +// CheckPod verifies that the responding agent belongs to the selected Pod. +// Older agents return 404; callers must not fall back to unbound mutations. +func (c *Client) CheckPod(ctx context.Context, namespace, server, podUID string) error { + if c.Disabled || podUID == "" { + return fmt.Errorf("agent: pod identity check requires an enabled client and pod UID") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, agentURL(namespace, server, "/v1/pods/"+url.PathEscape(podUID)+"/identity"), nil) + if err != nil { + return err + } + resp, err := c.http.Do(req) + if err != nil { + return fmt.Errorf("agent: pod identity check: %w", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("agent: pod identity check: status %d", resp.StatusCode) + } + return nil +} + +// QuiescePod disables saving only on the selected Pod's agent. +func (c *Client) QuiescePod(ctx context.Context, namespace, server, podUID string) error { + return c.callPodQuiesce(ctx, namespace, server, podUID, "/quiesce", true) +} + +// UnquiescePod restores saving only on the selected Pod's agent. +func (c *Client) UnquiescePod(ctx context.Context, namespace, server, podUID string) error { + return c.callPodQuiesce(ctx, namespace, server, podUID, "/unquiesce", false) +} + +func (c *Client) callPodQuiesce(ctx context.Context, namespace, server, podUID, path string, expectQuiesced bool) error { + if c.Disabled || podUID == "" { + return fmt.Errorf("agent: pod-bound quiesce requires an enabled client and pod UID") + } + return c.callQuiesce(ctx, namespace, server, "/v1/pods/"+url.PathEscape(podUID)+path, expectQuiesced) +} + // expectQuiesced is what the response should report on a successful // completion: true after /quiesce, false after /unquiesce. A mismatch // is a protocol bug worth surfacing. diff --git a/operator/internal/agent/client_test.go b/operator/internal/agent/client_test.go index c3866488e..7d04dc9d1 100644 --- a/operator/internal/agent/client_test.go +++ b/operator/internal/agent/client_test.go @@ -122,6 +122,46 @@ func TestAgentURL(t *testing.T) { } } +func TestPodBoundQuiesceNeverFallsBackToLegacyRoute(t *testing.T) { + for _, status := range []int{http.StatusOK, http.StatusNotFound} { + t.Run(http.StatusText(status), func(t *testing.T) { + paths := []string{} + c, closeServer := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + paths = append(paths, r.URL.Path) + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"quiesced":true}`)) + }) + defer closeServer() + err := c.QuiescePod(t.Context(), "ns", "server", "original-pod") + if (err == nil) != (status == http.StatusOK) { + t.Fatalf("status=%d error=%v", status, err) + } + if len(paths) != 1 || paths[0] != "/v1/pods/original-pod/quiesce" { + t.Fatalf("unbound mutation/fallback: %v", paths) + } + }) + } +} + +func TestPodBoundUnquiesceAndDisabledClient(t *testing.T) { + c, closeServer := newMockClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/pods/original-pod/unquiesce" { + t.Errorf("wrong cleanup target: %s", r.URL.Path) + } + _, _ = w.Write([]byte(`{"quiesced":false}`)) + }) + defer closeServer() + if err := c.UnquiescePod(t.Context(), "ns", "server", "original-pod"); err != nil { + t.Fatal(err) + } + if err := c.QuiescePod(t.Context(), "ns", "server", ""); err == nil { + t.Fatal("empty pod UID admitted") + } + if err := (&Client{Disabled: true}).QuiescePod(t.Context(), "ns", "server", "pod"); err == nil { + t.Fatal("disabled client reported identity-bound quiesce") + } +} + func TestQuiesce_DisabledNoOp(t *testing.T) { c := &Client{Disabled: true} if err := c.Quiesce(context.Background(), "ns", "srv"); err != nil { @@ -257,6 +297,32 @@ func TestUnquiesce_HappyPath(t *testing.T) { } } +func TestCheckPodRequiresIdentityEndpoint(t *testing.T) { + for _, status := range []int{http.StatusNoContent, http.StatusNotFound, http.StatusOK, http.StatusServiceUnavailable} { + t.Run(http.StatusText(status), func(t *testing.T) { + calls := 0 + c, cleanup := newMockClient(t, func(w http.ResponseWriter, req *http.Request) { + calls++ + if req.Method != http.MethodGet || req.URL.Path != "/v1/pods/original/identity" { + t.Errorf("unexpected preflight request: %s %s", req.Method, req.URL.Path) + } + w.WriteHeader(status) + }) + defer cleanup() + err := c.CheckPod(t.Context(), "ns", "server", "original") + if (err == nil) != (status == http.StatusNoContent) || calls != 1 { + t.Fatalf("identity result = %v, calls = %d", err, calls) + } + }) + } + if err := (&Client{Disabled: true}).CheckPod(t.Context(), "ns", "server", "original"); err == nil { + t.Fatal("disabled client confirmed identity") + } + if err := (&Client{}).CheckPod(t.Context(), "ns", "server", ""); err == nil { + t.Fatal("empty pod UID confirmed identity") + } +} + func TestQuiesce_TransportError(t *testing.T) { // Use a base transport that always errors. c := &Client{ diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index b350a5448..cb7cc4dae 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -247,14 +247,12 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr if !b.DeletionTimestamp.IsZero() { return r.finalizeDelete(ctx, &b) } - // Only Backups that might quiesce the world need to block on releasing - // it before deletion; everything else deletes immediately. - if b.Spec.Quiesce && !controllerutil.ContainsFinalizer(&b, gameplanev1alpha1.BackupFinalizer) { + // Every copy owns the source until its workers drain, even without quiesce. + if !controllerutil.ContainsFinalizer(&b, gameplanev1alpha1.BackupFinalizer) { controllerutil.AddFinalizer(&b, gameplanev1alpha1.BackupFinalizer) if err := r.Update(ctx, &b); err != nil { return ctrl.Result{}, err } - return ctrl.Result{}, nil } // Restic Backups own a snapshot in the repository, so hold their deletion @@ -275,6 +273,11 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr // Re-entering the full pipeline would re-validate the GameServer and repo // Secret (flipping an already-Succeeded Backup to Failed if either was since // deleted) and would re-create a cleaned-up Job, running a second backup. + if b.Status.Phase == gameplanev1alpha1.BackupPhaseSucceeded || b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + if err := r.syncBackupConsistency(ctx, &b); err != nil { + return ctrl.Result{}, err + } + } if b.Status.Phase == gameplanev1alpha1.BackupPhaseSucceeded && b.Status.SnapshotID == "" { return r.retrySnapshotScrape(ctx, &b) } @@ -288,6 +291,19 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { return r.runUnquiesce(ctx, &b) } + // Observe pre-upgrade workers even when multiple legacy quiesced copies + // already overlap. They must finish and unquiesce before new admission; + // trying to acquire exclusive ownership first would deadlock their cleanup. + if legacyBackupOwesUnquiesce(&b) && b.Spec.Strategy != "volume-snapshot" { + var existing batchv1.Job + err := r.backupAPIReader().Get(ctx, client.ObjectKeyFromObject(&b), &existing) + if err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + if err == nil && metav1.IsControlledBy(&existing, &b) { + return r.mirrorJobStatus(ctx, &b, &existing) + } + } // Resolve the target before building the Job: a Backup against a // GameServer that doesn't exist would produce a pod waiting forever @@ -372,23 +388,62 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr // requeues; ErrUnsupported is recorded as "attempted but unsupported" // so the matching unquiesce step is skipped on terminal phase. func (r *BackupReconciler) maybeQuiesce(ctx context.Context, b *gameplanev1alpha1.Backup) error { + preparation, err := parseBackupPreparation(b) + if err != nil { + return err + } + if preparation != nil && preparation.Outcome == "already-stopped" { + return nil + } if !b.Spec.Quiesce { return nil } - if r.AgentClient == nil { + if !agentQuiesceAvailable(r.AgentClient) { + if preparation != nil && preparation.Outcome == "live" && b.Annotations[annoQuiesceAttempted] != "true" && b.Annotations[annoQuiesceAttempted] != "unsupported" { + err := fmt.Errorf("agent quiesce client was disabled after live preparation") + if _, ferr := r.fail(ctx, b, err.Error()); ferr != nil { + return ferr + } + return err + } return nil } if state := b.Annotations[annoQuiesceAttempted]; state != "" && state != "pending" { return nil } + // Verify the versioned endpoint before creating the first cleanup debt. + // Old agents cannot mutate through it and must not pin an owner that + // would prevent their own upgrade. The POST still checks the UID again. + if preparation != nil && b.Annotations[annoQuiesceAttempted] == "" { + bound, ok := r.AgentClient.(podAgentQuiescer) + if !ok { + err = fmt.Errorf("agent client does not support pod-bound quiesce") + } else { + err = bound.CheckPod(ctx, b.Namespace, b.Spec.ServerRef.Name, string(preparation.PodUID)) + } + if err != nil { + _, ferr := r.fail(ctx, b, fmt.Sprintf("quiesce identity check failed: %v", err)) + if ferr != nil { + return ferr + } + return err + } + } // Persist cleanup intent BEFORE save-off: a crash or a partial agent failure // must still owe save-on even when quiesced-at was never recorded. + if err := r.setLifecycleUnquiesceIntent(ctx, b, true); err != nil { + return err + } patchBackupAnnotations(b, map[string]string{annoQuiesceAttempted: "pending"}) if err := r.Update(ctx, b); err != nil { return err } - err := r.AgentClient.Quiesce(ctx, b.Namespace, b.Spec.ServerRef.Name) + var podUID types.UID + if preparation != nil { + podUID = preparation.PodUID + } + err = callBackupQuiesce(ctx, r.AgentClient, b.Namespace, b.Spec.ServerRef.Name, podUID, false) switch { case err == nil: now := time.Now().UTC().Format(time.RFC3339) @@ -402,6 +457,9 @@ func (r *BackupReconciler) maybeQuiesce(ctx context.Context, b *gameplanev1alpha patchBackupAnnotations(b, map[string]string{ annoQuiesceAttempted: "unsupported", }) + if err := r.setLifecycleUnquiesceIntent(ctx, b, false); err != nil { + return err + } default: // Hard failure (network, RCON down). Surface as Failed: a // backup taken against an unflushed Minecraft world can be @@ -520,6 +578,7 @@ func (r *BackupReconciler) mirrorJobStatus( cond.Reason = string(phase) } newConds := upsertCondition(b.Status.Conditions, cond) + newConds = upsertCondition(newConds, backupConsistencyCondition(b)) if !sameConditions(b.Status.Conditions, newConds) { b.Status.Conditions = newConds dirty = true @@ -609,8 +668,8 @@ func (r *BackupReconciler) runUnquiesce(ctx context.Context, b *gameplanev1alpha } return ctrl.Result{RequeueAfter: 30 * time.Second}, nil } - // When an unquiesce was actually performed, record success so the - // "world might be frozen" signal clears. + // Confirmed save-on or confirmed original-process exit clears the + // "world might be frozen" signal, with distinct condition reasons. if _, ok := b.Annotations[annoUnquiescedAt]; ok { if cerr := r.setUnquiescedCondition(ctx, b, true, ""); cerr != nil { return ctrl.Result{}, cerr @@ -629,6 +688,10 @@ func (r *BackupReconciler) setUnquiescedCondition( if ok { cond.Status = metav1.ConditionTrue cond.Reason = "Unquiesced" + if b.Annotations[backupCleanupOutcomeAnnotation] == "source-gone" { + cond.Reason = "SourceProcessGone" + cond.Message = "original process no longer exists; no unquiesce RPC was sent" + } } else { cond.Status = metav1.ConditionFalse cond.Reason = "Failed" @@ -720,49 +783,29 @@ func (r *BackupReconciler) finalizeUnquiesce(ctx context.Context, b *gameplanev1 return ctrl.Result{}, nil } -// unquiesceTargetGone reports whether there is nothing left to unquiesce: -// the target GameServer no longer exists or is itself being deleted, or its -// pod no longer exists. A GameServer can be gone-together-with-its-Backup -// under an auto BackupSchedule's owner chain, and a pod alone can be gone if -// it was deleted independently of the GameServer; either way a fresh pod -// starts with auto-save on, so retrying the unquiesce would never succeed -// and would only block this Backup's deletion. +// unquiesceTargetGone requires evidence that the original process has exited. +// A deleted/deleting GameServer alone is insufficient: its pod can still run. func (r *BackupReconciler) unquiesceTargetGone(ctx context.Context, b *gameplanev1alpha1.Backup) (bool, error) { targetName := b.Spec.ServerRef.Name - var targetUID types.UID if value := b.Annotations[backupTargetAnnotation]; value != "" { identity, err := parseBackupIdentity(value) if err != nil { return false, err } - targetName, targetUID = identity.Name, identity.UID + targetName = identity.Name } if targetName == "" { return true, nil } - gs := &gameplanev1alpha1.GameServer{} - key := types.NamespacedName{Namespace: b.Namespace, Name: targetName} - switch err := r.Get(ctx, key, gs); { - case apierrors.IsNotFound(err): - return true, nil - case err != nil: - return false, fmt.Errorf("get gameserver %s to check unquiesce target: %w", key.Name, err) - case gs.DeletionTimestamp != nil: - return true, nil - } - if targetUID != "" && gs.UID != targetUID { - return true, nil + preparation, err := parseBackupPreparation(b) + if err != nil { + return false, err } - - var pod corev1.Pod - podKey := types.NamespacedName{Namespace: b.Namespace, Name: gs.Name + "-0"} - switch err := r.Get(ctx, podKey, &pod); { - case apierrors.IsNotFound(err): - return true, nil - case err != nil: - return false, fmt.Errorf("get pod %s to check unquiesce target: %w", podKey.Name, err) + var podUID types.UID + if preparation != nil { + podUID = preparation.PodUID } - return false, nil + return backupQuiesceProcessGone(ctx, r.backupAPIReader(), b.Namespace, targetName, podUID) } // unquiesceRetryExpired records (on first call) when finalizeDelete's @@ -850,10 +893,35 @@ func (r *BackupReconciler) maybeUnquiesce(ctx context.Context, b *gameplanev1alp return fmt.Errorf("legacy quiesce cleanup is blocked by a protected backup") } } - if r.AgentClient == nil { + if targetName != "" { + gone, err := r.unquiesceTargetGone(ctx, b) + if err != nil { + return err + } + if gone { + patchBackupAnnotations(b, map[string]string{backupCleanupOutcomeAnnotation: "source-gone"}) + return r.recordBackupUnquiesced(ctx, b) + } + } + if !agentQuiesceAvailable(r.AgentClient) { return fmt.Errorf("cannot release quiesced world without an agent client") } - if err := r.AgentClient.Unquiesce(ctx, b.Namespace, targetName); err != nil { + preparation, err := parseBackupPreparation(b) + if err != nil { + return err + } + var podUID types.UID + if preparation != nil { + podUID = preparation.PodUID + } + if err := callBackupQuiesce(ctx, r.AgentClient, b.Namespace, targetName, podUID, true); err != nil { + return err + } + return r.recordBackupUnquiesced(ctx, b) +} + +func (r *BackupReconciler) recordBackupUnquiesced(ctx context.Context, b *gameplanev1alpha1.Backup) error { + if err := r.setLifecycleUnquiesceIntent(ctx, b, false); err != nil { return err } patchBackupAnnotations(b, map[string]string{ @@ -901,6 +969,7 @@ func (r *BackupReconciler) fail(ctx context.Context, b *gameplanev1alpha1.Backup Message: msg, ObservedGeneration: b.Generation, }) + b.Status.Conditions = upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) if err := r.Status().Update(ctx, b); err != nil { return ctrl.Result{}, err } diff --git a/operator/internal/controller/backup_envtest_test.go b/operator/internal/controller/backup_envtest_test.go index 87ce6cc7e..f577563a6 100644 --- a/operator/internal/controller/backup_envtest_test.go +++ b/operator/internal/controller/backup_envtest_test.go @@ -8,9 +8,11 @@ import ( "testing" "time" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -379,6 +381,20 @@ func TestBackup_QuiesceAnnotationSetBeforeJob(t *testing.T) { fa := &fakeAgent{} startMgr(t, ns, withBackupReconciler(backupReconcilerOpts{agent: fa})) seedGameServer(t, ns, "smp") + // envtest has no StatefulSet controller or kubelet: provide the observed + // running source whose immutable identity is bound before the agent RPC. + var workload appsv1.StatefulSet + if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "smp"}, &workload); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "smp-0", Namespace: ns, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: workload.Name, UID: workload.UID, Controller: ownerBoolPtr(true)}}}, Spec: workload.Spec.Template.Spec} + if err := k8sClient.Create(context.Background(), pod); err != nil { + t.Fatal(err) + } + pod.Status.Phase = corev1.PodRunning + if err := k8sClient.Status().Update(context.Background(), pod); err != nil { + t.Fatal(err) + } if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo")); err != nil { t.Fatalf("create secret: %v", err) @@ -577,6 +593,7 @@ func TestBackup_UsesTemplateSecurityContext(t *testing.T) { if err := k8sClient.Create(context.Background(), gs); err != nil { t.Fatalf("create gameserver: %v", err) } + seedBackupWorkload(t, gs) if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo-secret")); err != nil { t.Fatalf("create secret: %v", err) } diff --git a/operator/internal/controller/backup_forget_envtest_test.go b/operator/internal/controller/backup_forget_envtest_test.go index 3a6c642b4..9dbf6bf8e 100644 --- a/operator/internal/controller/backup_forget_envtest_test.go +++ b/operator/internal/controller/backup_forget_envtest_test.go @@ -16,6 +16,7 @@ import ( "k8s.io/client-go/tools/events" "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/manager" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" @@ -149,17 +150,17 @@ func TestBackupForget_FinalizerOnlyOnResticBackups(t *testing.T) { return false, "restic backup has no snapshot finalizer yet" }) - // Once the volume-snapshot Backup is Running the reconciler has been through - // the point where it would have added a finalizer. + // Every strategy gets an operation cleanup finalizer before admission. Do + // not wait for Running: the restic backup may hold the shared server owner. eventually(t, func() (bool, string) { b := getBackup(t, ns, "smp-vs") - if b.Status.Phase != gameplanev1alpha1.BackupPhaseRunning { + if !controllerutil.ContainsFinalizer(b, gameplanev1alpha1.BackupFinalizer) { return false, describeBackupStatus(b) } return true, "" }) - if got := getBackup(t, ns, "smp-vs").Finalizers; len(got) != 0 { - t.Errorf("volume-snapshot backup finalizers = %v, want none", got) + if got := getBackup(t, ns, "smp-vs"); controllerutil.ContainsFinalizer(got, gameplanev1alpha1.BackupSnapshotFinalizer) { + t.Errorf("volume-snapshot backup finalizers = %v, want no restic snapshot finalizer", got.Finalizers) } } diff --git a/operator/internal/controller/backup_forget_test.go b/operator/internal/controller/backup_forget_test.go index 12dd1aa51..ef917dd7f 100644 --- a/operator/internal/controller/backup_forget_test.go +++ b/operator/internal/controller/backup_forget_test.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -180,7 +181,7 @@ func TestReconcile_AddsSnapshotFinalizerForResticBackup(t *testing.T) { } // Volume-snapshot and Failed Backups never write a restic snapshot, so they -// must not be held on deletion. +// need operation cleanup but must not be held for restic snapshot deletion. func TestReconcile_NoSnapshotFinalizerWhenNothingToForget(t *testing.T) { volume := forgetBackup() volume.DeletionTimestamp = nil @@ -205,8 +206,11 @@ func TestReconcile_NoSnapshotFinalizerWhenNothingToForget(t *testing.T) { if !ok { t.Fatal("backup vanished") } - if len(got.Finalizers) != 0 { - t.Errorf("finalizers = %v, want none", got.Finalizers) + if !controllerutil.ContainsFinalizer(got, gameplanev1alpha1.BackupFinalizer) { + t.Errorf("finalizers = %v, want operation cleanup finalizer", got.Finalizers) + } + if controllerutil.ContainsFinalizer(got, gameplanev1alpha1.BackupSnapshotFinalizer) { + t.Errorf("finalizers = %v, want no restic snapshot finalizer", got.Finalizers) } }) } diff --git a/operator/internal/controller/backup_guard.go b/operator/internal/controller/backup_guard.go index 4cc09e2f9..ea0f8d84b 100644 --- a/operator/internal/controller/backup_guard.go +++ b/operator/internal/controller/backup_guard.go @@ -3,6 +3,7 @@ package controller import ( "context" "encoding/json" + "errors" "fmt" "time" @@ -61,9 +62,6 @@ func (r *BackupReconciler) backupAPIReader() client.Reader { // the target before claiming it so a crash or a same-name replacement cannot // redirect cleanup to a different game. func (r *BackupReconciler) claimBackupTarget(ctx context.Context, b *gameplanev1alpha1.Backup) (bool, error) { - if !b.Spec.Quiesce { - return true, nil - } legacy, err := r.legacyQuiescedBackups(ctx, b) if err != nil { return false, err @@ -72,10 +70,9 @@ func (r *BackupReconciler) claimBackupTarget(ctx context.Context, b *gameplanev1 // Already-running pre-upgrade copies must be allowed to finish, but // nobody may start a new protected copy until all of their cleanup is // confirmed. Each legacy cleanup also waits for all legacy workers. - if legacyBackupOwesUnquiesce(b) { - return true, nil + if !legacyBackupOwesUnquiesce(b) { + return false, nil } - return false, nil } var gs gameplanev1alpha1.GameServer if err := r.backupAPIReader().Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: b.Spec.ServerRef.Name}, &gs); err != nil { @@ -98,55 +95,11 @@ func (r *BackupReconciler) claimBackupTarget(ctx context.Context, b *gameplanev1 return false, err } } - owner := backupIdentityJSON(b.Name, b.UID) - guard := gs.Annotations[backupGuardAnnotation] - if guard == owner { - return r.recoverBackupTarget(ctx, b) - } - if guard != "" { - id, err := parseBackupIdentity(guard) - if err != nil { - return false, err - } // Malformed guards fail closed. - var holder gameplanev1alpha1.Backup - err = r.backupAPIReader().Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: id.Name}, &holder) - if err == nil && holder.UID == id.UID { - return false, nil - } - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - // Persist recovery intent, then transfer ownership with CAS BEFORE - // touching the agent. Concurrent contenders cannot both call save-on - // while one of them has already started a new protected copy. - patchBackupAnnotations(b, map[string]string{backupRecoveryAnnotation: guard, annoQuiesceAttempted: "pending"}) - if err := r.Update(ctx, b); err != nil { - return false, err - } - // Preserve every predecessor on the surviving GameServer, atomically - // with the owner transfer. A recovering Backup can itself disappear - // before an earlier owner's copy worker has stopped. - lineage, err := parseBackupRecoveryLineage(gs.Annotations[backupRecoveryLineageAnnotation]) - if err != nil { - return false, err - } - found := false - for _, predecessor := range lineage { - if predecessor == id { - found = true - } - } - if !found { - lineage = append(lineage, id) - } - data, _ := json.Marshal(lineage) - gs.Annotations[backupRecoveryLineageAnnotation] = string(data) - } - if gs.Annotations == nil { - gs.Annotations = make(map[string]string) - } - gs.Annotations[backupGuardAnnotation] = owner - if err := r.Update(ctx, &gs); err != nil { + state := b.Annotations[annoQuiesceAttempted] + owesUnquiesce := state != "" && state != "unsupported" && b.Annotations[annoUnquiescedAt] == "" + claimed, err := claimLifecycleOperation(ctx, r.Client, r.backupAPIReader(), &gs, + backupLifecycleIdentity(&gs, b, owesUnquiesce), r.AgentClient) + if err != nil || !claimed { return false, err } return r.recoverBackupTarget(ctx, b) @@ -166,11 +119,19 @@ func (r *BackupReconciler) recoverBackupTarget(ctx context.Context, b *gameplane return false, err } } - if r.AgentClient == nil { - return false, fmt.Errorf("cannot recover orphaned quiesce without an agent client") + // Legacy owners have no pod UID. Retire them only on positive absence or + // terminal-pod proof; never bind an old RPC to a newly observed identity. + gone, err := backupQuiesceProcessGone(ctx, r.backupAPIReader(), b.Namespace, b.Spec.ServerRef.Name, "") + if err != nil { + return false, err } - if err := r.AgentClient.Unquiesce(ctx, b.Namespace, b.Spec.ServerRef.Name); err != nil { - return false, fmt.Errorf("recover orphaned quiesce: %w", err) + if !gone { + if !agentQuiesceAvailable(r.AgentClient) { + return false, fmt.Errorf("cannot recover orphaned quiesce without an agent client") + } + if err := r.AgentClient.Unquiesce(ctx, b.Namespace, b.Spec.ServerRef.Name); err != nil { + return false, fmt.Errorf("recover orphaned quiesce: %w", err) + } } delete(b.Annotations, backupRecoveryAnnotation) delete(b.Annotations, annoQuiesceAttempted) @@ -227,8 +188,19 @@ func (r *BackupReconciler) backupRecoveryIdentities(ctx context.Context, b *game } func (r *BackupReconciler) backupCleanupWorkersLive(ctx context.Context, b *gameplanev1alpha1.Backup, cancel bool) (bool, error) { - if !b.Spec.Quiesce && b.Annotations[annoQuiesceAttempted] == "" { - return false, nil + gs, err := r.ownedBackupTarget(ctx, b) + if err != nil { + return false, err + } + if gs != nil && gs.Annotations[lifecycleOperationAnnotation] != "" { + op, err := parseLifecycleOperation(gs.Annotations[lifecycleOperationAnnotation]) + if err != nil { + return false, err + } + ready, err := recoverLifecycleOperation(ctx, r.Client, r.backupAPIReader(), gs, op, r.AgentClient) + if err != nil || !ready { + return !ready, err + } } busy, err := r.backupWorkersLive(ctx, b.Namespace, backupIdentity{Name: b.Name, UID: b.UID}, cancel) if err != nil || busy { @@ -283,12 +255,54 @@ func (r *BackupReconciler) legacyQuiescedBackups(ctx context.Context, b *gamepla func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { claimed, err := r.claimBackupTarget(ctx, b) if err != nil { + if errors.Is(err, errLifecycleGenerationChanged) { + _, err = r.fail(ctx, b, err.Error()) + return ctrl.Result{RequeueAfter: time.Nanosecond}, err + } return ctrl.Result{}, err } if !claimed { return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } - return ctrl.Result{}, r.maybeQuiesce(ctx, b) + gs, err := r.ownedBackupTarget(ctx, b) + if err != nil { + return ctrl.Result{}, err + } + if gs == nil { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + fenced, err := fenceBackupWorkload(ctx, r.Client, r.backupAPIReader(), gs) + if err != nil || !fenced { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + ready, err := r.prepareBackupSource(ctx, b, gs) + if errors.Is(err, errBackupSourceChanged) { + _, err = r.fail(ctx, b, err.Error()) + return ctrl.Result{RequeueAfter: time.Nanosecond}, err + } + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + if err := r.maybeQuiesce(ctx, b); err != nil { + return ctrl.Result{}, err + } + // The Service RPC may overlap a pod replacement. Never start a copy on a + // different process from the one pinned before the request. + ready, err = r.prepareBackupSource(ctx, b, gs) + if errors.Is(err, errBackupSourceChanged) { + _, err = r.fail(ctx, b, err.Error()) + return ctrl.Result{RequeueAfter: time.Nanosecond}, err + } + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + if err := r.recordBackupManifest(ctx, b, gs); err != nil { + // Metadata must be durable before capture. Do not leave saving paused + // indefinitely when a lookup/status write prevents the copy starting. + _, failErr := r.fail(ctx, b, "cannot record capture metadata: "+err.Error()) + return ctrl.Result{RequeueAfter: time.Second}, failErr + } + return ctrl.Result{}, nil } // Return only the originally bound GameServer. A foreign owner means this @@ -313,6 +327,15 @@ func (r *BackupReconciler) ownedBackupTarget(ctx context.Context, b *gameplanev1 if gs.UID != id.UID || gs.Annotations[backupGuardAnnotation] != backupIdentityJSON(b.Name, b.UID) { return nil, nil } + if value := gs.Annotations[lifecycleOperationAnnotation]; value != "" { + op, err := parseLifecycleOperation(value) + if err != nil { + return nil, err + } + if !sameLifecycleOwner(op.lifecycleIdentity, backupLifecycleIdentity(&gs, b, false)) { + return nil, nil + } + } return &gs, nil } @@ -321,6 +344,9 @@ func (r *BackupReconciler) releaseBackupTarget(ctx context.Context, b *gameplane if err != nil || gs == nil { return err } + if err := clearLifecycleOperation(ctx, r.Client, r.backupAPIReader(), gs, backupLifecycleIdentity(gs, b, false)); err != nil { + return err + } delete(gs.Annotations, backupGuardAnnotation) delete(gs.Annotations, backupRecoveryLineageAnnotation) return r.Update(ctx, gs) diff --git a/operator/internal/controller/backup_guard_test.go b/operator/internal/controller/backup_guard_test.go index ac0e2be97..52efc597e 100644 --- a/operator/internal/controller/backup_guard_test.go +++ b/operator/internal/controller/backup_guard_test.go @@ -8,6 +8,7 @@ import ( "time" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -27,13 +28,30 @@ import ( type guardQuiescer struct { quiesced, unquiesced int quiesceErr error + unquiesceErr error + checkErr error + afterQuiesce func() } func (q *guardQuiescer) Quiesce(context.Context, string, string) error { q.quiesced++ return q.quiesceErr } -func (q *guardQuiescer) Unquiesce(context.Context, string, string) error { q.unquiesced++; return nil } +func (q *guardQuiescer) Unquiesce(context.Context, string, string) error { + q.unquiesced++ + return q.unquiesceErr +} +func (q *guardQuiescer) CheckPod(context.Context, string, string, string) error { return q.checkErr } +func (q *guardQuiescer) QuiescePod(ctx context.Context, namespace, server, _ string) error { + err := q.Quiesce(ctx, namespace, server) + if q.afterQuiesce != nil { + q.afterQuiesce() + } + return err +} +func (q *guardQuiescer) UnquiescePod(ctx context.Context, namespace, server, _ string) error { + return q.Unquiesce(ctx, namespace, server) +} func guardBackup(name string) *gameplanev1alpha1.Backup { return &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "ns", UID: types.UID(name + "-uid"), Finalizers: []string{gameplanev1alpha1.BackupFinalizer, gameplanev1alpha1.BackupSnapshotFinalizer}}, Spec: gameplanev1alpha1.BackupSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: "server"}, RepoRef: &gameplanev1alpha1.SecretKeySelector{Name: "repo"}, Quiesce: true}} @@ -45,7 +63,13 @@ func guardFixture(t *testing.T, backups ...*gameplanev1alpha1.Backup) (*BackupRe if err := snapshotv1.AddToScheme(s); err != nil { t.Fatal(err) } + if err := appsv1.AddToScheme(s); err != nil { + t.Fatal(err) + } objects := []client.Object{&gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "repo", Namespace: "ns"}, Data: map[string][]byte{"repo": []byte("repo"), "password": []byte("password")}}} + one := int32(1) + objects = append(objects, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "workload-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "GameServer", Name: "server", UID: "server-uid", Controller: ownerBoolPtr(true)}}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}}) + objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "game-pod-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: "server", UID: "workload-uid", Controller: ownerBoolPtr(true)}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}) for _, b := range backups { objects = append(objects, b) } @@ -85,12 +109,23 @@ func TestBackupGuardSerializesAcrossBackupsAndRestart(t *testing.T) { if len(jobs.Items) != 1 || q.quiesced != 1 || q.unquiesced != 0 { t.Fatalf("overlapping protected copies: jobs=%d, agent=%+v", len(jobs.Items), q) } + first := guardGetBackup(t, r, "first") + if first.Status.CaptureManifest == nil || first.Status.CaptureManifest.QuiesceOutcome != "confirmed" { + t.Fatalf("worker started without confirmed capture-time metadata: %+v", first.Status) + } + if second := guardGetBackup(t, r, "second"); second.Status.CaptureManifest != nil { + t.Fatalf("queued copy recorded metadata before admission: %+v", second.Status.CaptureManifest) + } job := &jobs.Items[0] job.Status.Succeeded = 1 if err := r.Status().Update(context.Background(), job); err != nil { t.Fatal(err) } guardPass(t, r, "first") + first = guardGetBackup(t, r, "first") + if condition := meta.FindStatusCondition(first.Status.Conditions, "Consistency"); condition == nil || condition.Status != metav1.ConditionTrue || condition.Reason != "Quiesced" { + t.Fatalf("completed copy did not expose achieved quiesce: %+v", first.Status) + } guardPass(t, r, "second") if q.quiesced != 2 || q.unquiesced != 1 { t.Fatalf("second did not acquire after first cleanup: %+v", q) diff --git a/operator/internal/controller/backup_manifest.go b/operator/internal/controller/backup_manifest.go new file mode 100644 index 000000000..682a2214c --- /dev/null +++ b/operator/internal/controller/backup_manifest.go @@ -0,0 +1,180 @@ +package controller + +import ( + "context" + "sort" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// Called after lifecycle ownership and the workload fence, before creating the +// copy worker. Requeues and later template edits never rewrite this manifest. +func (r *BackupReconciler) recordBackupManifest(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer) error { + if b.Status.CaptureManifest != nil { + return nil + } + reader := r.backupAPIReader() + var worker client.Object = &batchv1.Job{} + if b.Spec.Strategy == "volume-snapshot" { + worker = &snapshotv1.VolumeSnapshot{} + } + if err := reader.Get(ctx, client.ObjectKeyFromObject(b), worker); err == nil { + // Existing workers predate this metadata. Current layout cannot prove + // what they captured, even if their requested quiesce was successful. + b.Status.CaptureManifest = &gameplanev1alpha1.BackupCaptureManifest{State: "LegacyUnknown"} + return r.Status().Update(ctx, b) + } else if !apierrors.IsNotFound(err) { + return err + } + var ss appsv1.StatefulSet + err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + var layout corev1.PodSpec + source := "Unavailable" + if err == nil && metav1.IsControlledBy(&ss, gs) { + layout, source = ss.Spec.Template.Spec, "StatefulSet" + var pod corev1.Pod + err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil && metav1.IsControlledBy(&pod, &ss) { + layout, source = pod.Spec, "Pod" + } + } + var tmpl gameplanev1alpha1.GameTemplate + if err := reader.Get(ctx, client.ObjectKey{Name: gs.Spec.TemplateRef.Name}, &tmpl); err != nil && !apierrors.IsNotFound(err) { + return err + } + primary := gs.Name + "-data" + volumes := map[string]*gameplanev1alpha1.BackupVolumeCoverage{ + primary: {ClaimName: primary, Included: true, Classification: "data", CoverageReason: "PrimaryDataClaim"}, + } + podNames := map[string]string{} + for _, volume := range layout.Volumes { + if volume.PersistentVolumeClaim == nil { + continue + } + name := volume.PersistentVolumeClaim.ClaimName + podNames[volume.Name] = name + if volumes[name] == nil { + volumes[name] = &gameplanev1alpha1.BackupVolumeCoverage{ClaimName: name, Classification: "unclassified", CoverageReason: "NotCovered"} + } + } + for _, container := range layout.Containers { + if container.Name != "game" { + continue + } + for _, mount := range container.VolumeMounts { + if volume := volumes[podNames[mount.Name]]; volume != nil { + volume.MountPaths = append(volume.MountPaths, mount.MountPath) + } + } + } + var claims corev1.PersistentVolumeClaimList + if err := reader.List(ctx, &claims, client.InNamespace(gs.Namespace)); err != nil { + return err + } + for _, claim := range claims.Items { + volume := volumes[claim.Name] + if volume == nil && metav1.IsControlledBy(&claim, gs) { + volume = &gameplanev1alpha1.BackupVolumeCoverage{ClaimName: claim.Name, Classification: "unclassified", CoverageReason: "RetainedUnmountedClaim"} + volumes[claim.Name] = volume + } + if volume != nil { + volume.ClaimUID = string(claim.UID) + } + } + for _, extra := range effectiveExtraVolumes(gs, &tmpl) { + volume := volumes[extraPVCName(gs, extra.Name)] + if volume == nil { + continue + } + // A newer declaration for a different mount must not classify the + // frozen workload's volume as safely rebuildable. + for _, mount := range volume.MountPaths { + if mount == extra.MountPath && extra.BackupRole != "" { + volume.Classification = extra.BackupRole + } + } + } + now := metav1.Now() + manifest := &gameplanev1alpha1.BackupCaptureManifest{State: "Recorded", RecordedAt: &now, + ServerUID: string(gs.UID), Strategy: b.Spec.Strategy, LayoutSource: source, QuiesceOutcome: backupQuiesceOutcome(b)} + if manifest.Strategy == "" { + manifest.Strategy = "restic-snapshot" + } + for _, volume := range volumes { + sort.Strings(volume.MountPaths) + manifest.Volumes = append(manifest.Volumes, *volume) + } + sort.Slice(manifest.Volumes, func(i, j int) bool { return manifest.Volumes[i].ClaimName < manifest.Volumes[j].ClaimName }) + b.Status.CaptureManifest = manifest + return r.Status().Update(ctx, b) +} + +func backupQuiesceOutcome(b *gameplanev1alpha1.Backup) string { + if preparation, err := parseBackupPreparation(b); err == nil && preparation != nil && preparation.Outcome == "already-stopped" { + return "already-stopped" + } + switch b.Annotations[annoQuiesceAttempted] { + case "true": + return "confirmed" + case "unsupported": + return "unsupported" + case "pending": + return "unknown" + default: + if !b.Spec.Quiesce { + return "not-requested" + } + return "agent-unavailable" + } +} + +// Consistency=True means the copy completed after confirmed quiesce or from an +// already-stopped source whose writers had drained before capture admission. +// False is an explicit weaker guarantee, not a synonym for copy failure. +func backupConsistencyCondition(b *gameplanev1alpha1.Backup) metav1.Condition { + condition := metav1.Condition{Type: "Consistency", ObservedGeneration: b.Generation, Status: metav1.ConditionUnknown} + switch { + case b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed: + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "CaptureFailed", "The capture failed; no usable consistency guarantee was achieved." + case b.Status.CaptureManifest == nil || b.Status.CaptureManifest.State != "Recorded": + condition.Reason, condition.Message = "LegacyMetadataUnknown", "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template." + case b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || b.Status.SnapshotID == "": + condition.Reason, condition.Message = "CapturePending", "The capture has not completed; its achieved consistency is not yet known." + case b.Status.CaptureManifest.QuiesceOutcome == "confirmed": + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "Quiesced", "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup." + case b.Status.CaptureManifest.QuiesceOutcome == "already-stopped": + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "StoppedSource", "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup." + case b.Status.CaptureManifest.Strategy == "volume-snapshot": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "CrashConsistentSnapshot", "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed ("+b.Status.CaptureManifest.QuiesceOutcome+")." + case b.Status.CaptureManifest.QuiesceOutcome == "unsupported": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "UnsupportedRawCopy", "The game does not support quiesce. The primary volume was copied while it could still change." + case b.Status.CaptureManifest.QuiesceOutcome == "agent-unavailable": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "AgentUnavailableRawCopy", "No agent quiesce capability was configured. The primary volume was copied without pausing game writes." + default: + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "RawCopy", "The primary volume was copied without confirmed application quiesce." + } + return condition +} + +func (r *BackupReconciler) syncBackupConsistency(ctx context.Context, b *gameplanev1alpha1.Backup) error { + conditions := upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) + if sameConditions(b.Status.Conditions, conditions) { + return nil + } + b.Status.Conditions = conditions + return r.Status().Update(ctx, b) +} diff --git a/operator/internal/controller/backup_manifest_test.go b/operator/internal/controller/backup_manifest_test.go new file mode 100644 index 000000000..5ccb1f0c7 --- /dev/null +++ b/operator/internal/controller/backup_manifest_test.go @@ -0,0 +1,144 @@ +package controller + +import ( + "reflect" + "testing" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestBackupManifestPinsCoverageAndClassification(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + _ = batchv1.AddToScheme(scheme) + _ = gameplanev1alpha1.AddToScheme(scheme) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "game-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}} + tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template"}, Spec: gameplanev1alpha1.GameTemplateSpec{Storage: gameplanev1alpha1.GameStorageSpec{ + Extra: []gameplanev1alpha1.ExtraVolumeSpec{{Name: "install", MountPath: "/serverfiles", BackupRole: "rebuildable"}}, + }}} + b := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "copy", Namespace: "games", UID: "backup-uid", Annotations: map[string]string{annoQuiesceAttempted: "unsupported"}}, Spec: gameplanev1alpha1.BackupSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: gs.Name}, Quiesce: true}} + owner := *metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer")) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "game", VolumeMounts: []corev1.VolumeMount{{Name: "data", MountPath: "/world"}, {Name: "extra-install", MountPath: "/serverfiles"}, {Name: "mods", MountPath: "/world/mods"}}}}, + Volumes: []corev1.Volume{ + {Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-data"}}}, + {Name: "extra-install", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-extra-install"}}}, + {Name: "mods", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-mods-active"}}}, + }, + }}}, + } + objects := []client.Object{gs, tmpl, b, ss} + for _, name := range []string{"game-data", "game-extra-install", "game-mods-active", "game-mods-old"} { + objects = append(objects, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: gs.Namespace, UID: types.UID("uid-" + name), OwnerReferences: []metav1.OwnerReference{owner}}}) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(b).WithObjects(objects...).Build() + r := &BackupReconciler{Client: c, APIReader: c, Scheme: scheme} + if err := r.recordBackupManifest(t.Context(), b, gs); err != nil { + t.Fatal(err) + } + manifest := b.Status.CaptureManifest + if manifest == nil || manifest.State != "Recorded" || manifest.QuiesceOutcome != "unsupported" || len(manifest.Volumes) != 4 { + t.Fatalf("incomplete manifest: %+v", manifest) + } + byName := map[string]gameplanev1alpha1.BackupVolumeCoverage{} + for _, volume := range manifest.Volumes { + byName[volume.ClaimName] = volume + } + if !byName["game-data"].Included || byName["game-data"].ClaimUID != "uid-game-data" || !reflect.DeepEqual(byName["game-data"].MountPaths, []string{"/world"}) { + t.Fatalf("primary coverage: %+v", byName["game-data"]) + } + if byName["game-extra-install"].Included || byName["game-extra-install"].Classification != "rebuildable" || byName["game-mods-active"].Included || byName["game-mods-old"].Included { + t.Fatalf("extra or nested mod volume incorrectly covered: %+v", manifest.Volumes) + } + before := manifest.DeepCopy() + tmpl.Spec.Storage.Extra[0].BackupRole = "data" + if err := c.Update(t.Context(), tmpl); err != nil { + t.Fatal(err) + } + if err := r.recordBackupManifest(t.Context(), b, gs); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, b.Status.CaptureManifest) { + t.Fatal("template change rewrote capture manifest") + } +} + +func TestBackupManifestDoesNotInventHistoryForExistingWorker(t *testing.T) { + for _, strategy := range []string{"restic-snapshot", "volume-snapshot"} { + t.Run(strategy, func(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = batchv1.AddToScheme(scheme) + _ = snapshotv1.AddToScheme(scheme) + _ = gameplanev1alpha1.AddToScheme(scheme) + b := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "legacy", Namespace: "games"}, Spec: gameplanev1alpha1.BackupSpec{Strategy: strategy}} + var worker client.Object = &batchv1.Job{ObjectMeta: b.ObjectMeta} + if strategy == "volume-snapshot" { + worker = &snapshotv1.VolumeSnapshot{ObjectMeta: b.ObjectMeta} + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(b).WithObjects(b, worker).Build() + r := &BackupReconciler{Client: c, APIReader: c, Scheme: scheme} + if err := r.recordBackupManifest(t.Context(), b, &gameplanev1alpha1.GameServer{}); err != nil { + t.Fatal(err) + } + if b.Status.CaptureManifest.State != "LegacyUnknown" || len(b.Status.CaptureManifest.Volumes) != 0 { + t.Fatalf("invented history: %+v", b.Status.CaptureManifest) + } + }) + } +} + +func TestBackupConsistencyDescribesAchievedResult(t *testing.T) { + for _, tc := range []struct { + strategy, outcome, reason string + status metav1.ConditionStatus + }{ + {"restic-snapshot", "confirmed", "Quiesced", metav1.ConditionTrue}, + {"restic-snapshot", "already-stopped", "StoppedSource", metav1.ConditionTrue}, + {"restic-snapshot", "unsupported", "UnsupportedRawCopy", metav1.ConditionFalse}, + {"restic-snapshot", "not-requested", "RawCopy", metav1.ConditionFalse}, + {"restic-snapshot", "agent-unavailable", "AgentUnavailableRawCopy", metav1.ConditionFalse}, + {"volume-snapshot", "unsupported", "CrashConsistentSnapshot", metav1.ConditionFalse}, + {"volume-snapshot", "confirmed", "Quiesced", metav1.ConditionTrue}, + {"volume-snapshot", "already-stopped", "StoppedSource", metav1.ConditionTrue}, + } { + t.Run(tc.strategy+"/"+tc.outcome, func(t *testing.T) { + b := &gameplanev1alpha1.Backup{Status: gameplanev1alpha1.BackupStatus{Phase: gameplanev1alpha1.BackupPhaseSucceeded, SnapshotID: "snapshot-id", + CaptureManifest: &gameplanev1alpha1.BackupCaptureManifest{State: "Recorded", Strategy: tc.strategy, QuiesceOutcome: tc.outcome}}} + condition := backupConsistencyCondition(b) + if condition.Reason != tc.reason || condition.Status != tc.status { + t.Fatalf("got %+v", condition) + } + b.Status.SnapshotID = "" + if condition := backupConsistencyCondition(b); condition.Reason != "CapturePending" || condition.Status != metav1.ConditionUnknown { + t.Fatalf("copy without restorable identity claims achieved consistency: %+v", condition) + } + b.Status.SnapshotID = "snapshot-id" + b.Status.Phase = gameplanev1alpha1.BackupPhaseRunning + if condition := backupConsistencyCondition(b); condition.Status != metav1.ConditionUnknown { + t.Fatalf("unfinished copy claims achieved consistency: %+v", condition) + } + b.Status.Phase = gameplanev1alpha1.BackupPhaseFailed + if condition := backupConsistencyCondition(b); condition.Reason != "CaptureFailed" || condition.Status != metav1.ConditionFalse { + t.Fatalf("failed copy claims usable result: %+v", condition) + } + }) + } + b := &gameplanev1alpha1.Backup{Status: gameplanev1alpha1.BackupStatus{Phase: gameplanev1alpha1.BackupPhaseSucceeded}} + if condition := backupConsistencyCondition(b); condition.Status != metav1.ConditionUnknown || condition.Reason != "LegacyMetadataUnknown" { + t.Fatalf("legacy backup guarantee invented: %+v", condition) + } +} diff --git a/operator/internal/controller/backup_preparation.go b/operator/internal/controller/backup_preparation.go new file mode 100644 index 000000000..4f682ab53 --- /dev/null +++ b/operator/internal/controller/backup_preparation.go @@ -0,0 +1,234 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + backupPreparationAnnotation = "backup.gameplane.local/preparation" + backupCleanupOutcomeAnnotation = "backup.gameplane.local/cleanup-outcome" +) + +var errBackupSourceChanged = errors.New("backup source changed after preparation") + +type podAgentQuiescer interface { + CheckPod(context.Context, string, string, string) error + QuiescePod(context.Context, string, string, string) error + UnquiescePod(context.Context, string, string, string) error +} + +func agentQuiesceAvailable(q AgentQuiescer) bool { + if q == nil { + return false + } + if configured, ok := q.(interface{ QuiesceEnabled() bool }); ok { + return configured.QuiesceEnabled() + } + return true +} + +func callBackupQuiesce(ctx context.Context, q AgentQuiescer, namespace, server string, podUID types.UID, undo bool) error { + if !agentQuiesceAvailable(q) { + return fmt.Errorf("agent quiesce client is not configured") + } + if podUID != "" { + bound, ok := q.(podAgentQuiescer) + if !ok { + return fmt.Errorf("agent client does not support pod-bound quiesce") + } + if undo { + return bound.UnquiescePod(ctx, namespace, server, string(podUID)) + } + return bound.QuiescePod(ctx, namespace, server, string(podUID)) + } + if undo { + return q.Unquiesce(ctx, namespace, server) + } + return q.Quiesce(ctx, namespace, server) +} + +// Source identity is immutable for one copy. RPC outcome remains in the +// quiesce annotations, so a stopped source is never described as quiesced. +type backupPreparation struct { + Outcome string `json:"outcome"` + TargetUID types.UID `json:"targetUID"` + WorkloadUID types.UID `json:"workloadUID"` + WorkloadGeneration int64 `json:"workloadGeneration"` + PodUID types.UID `json:"podUID,omitempty"` +} + +func parseBackupPreparation(b *gameplanev1alpha1.Backup) (*backupPreparation, error) { + value := b.Annotations[backupPreparationAnnotation] + if value == "" { + return nil, nil + } + var p backupPreparation + if err := json.Unmarshal([]byte(value), &p); err != nil { + return nil, fmt.Errorf("decode backup preparation: %w", err) + } + if p.TargetUID == "" || p.WorkloadUID == "" || (p.Outcome != "live" && p.Outcome != "already-stopped") || (p.Outcome == "live" && p.PodUID == "") || (p.Outcome == "already-stopped" && p.PodUID != "") { + return nil, fmt.Errorf("invalid backup preparation identity") + } + return &p, nil +} + +// Call only after the shared owner and StatefulSet fence are held. Readiness +// and desired suspend/idle state are not evidence that writers have exited. +func (r *BackupReconciler) prepareBackupSource(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer) (bool, error) { + if !b.Spec.Quiesce { + return true, nil + } + p, err := parseBackupPreparation(b) + if err != nil { + return false, err + } + if p == nil && !agentQuiesceAvailable(r.AgentClient) { + return true, nil + } + // Pre-upgrade attempts have no source identity. Do not invent a new one + // after their RPC; preserve their existing cleanup contract. + if p == nil && b.Annotations[annoQuiesceAttempted] != "" { + return true, nil + } + var ss appsv1.StatefulSet + if err := r.backupAPIReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + if apierrors.IsNotFound(err) && p != nil { + return false, fmt.Errorf("%w: workload absent", errBackupSourceChanged) + } + return false, client.IgnoreNotFound(err) + } + if p != nil && (p.TargetUID != gs.UID || p.WorkloadUID != ss.UID || p.WorkloadGeneration != ss.Generation) { + return false, fmt.Errorf("%w: workload identity", errBackupSourceChanged) + } + if !metav1.IsControlledBy(&ss, gs) { + return false, fmt.Errorf("backup source workload is not owned by the bound GameServer") + } + if ss.Spec.Replicas == nil || ss.Status.ObservedGeneration < ss.Generation { + return false, nil + } + if p != nil && ((p.Outcome == "already-stopped") != (*ss.Spec.Replicas == 0)) { + return false, fmt.Errorf("%w: desired replicas", errBackupSourceChanged) + } + var candidate backupPreparation + candidate.TargetUID, candidate.WorkloadUID, candidate.WorkloadGeneration = gs.UID, ss.UID, ss.Generation + if *ss.Spec.Replicas == 0 { + if ss.Status.Replicas != 0 { + if p != nil { + return false, fmt.Errorf("%w: stopped workload has replicas", errBackupSourceChanged) + } + return false, nil + } + drained, err := r.backupSourceDrained(ctx, b, gs, &ss) + if err != nil || !drained { + if err == nil && p != nil { + return false, fmt.Errorf("%w: stopped source acquired a writer", errBackupSourceChanged) + } + return false, err + } + candidate.Outcome = "already-stopped" + } else { + var pod corev1.Pod + err := r.backupAPIReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if apierrors.IsNotFound(err) { + if p != nil { + return false, fmt.Errorf("%w: original pod absent", errBackupSourceChanged) + } + return false, nil + } + if err != nil { + return false, err + } + if p != nil && (p.PodUID != pod.UID || !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning) { + return false, fmt.Errorf("%w: original process no longer running", errBackupSourceChanged) + } + if !metav1.IsControlledBy(&pod, &ss) || pod.UID == "" { + return false, fmt.Errorf("backup source pod is not owned by the bound workload") + } + if !pod.DeletionTimestamp.IsZero() || pod.Status.Phase != corev1.PodRunning { + return false, nil + } + candidate.Outcome, candidate.PodUID = "live", pod.UID + } + if p != nil { + if p.Outcome != candidate.Outcome || p.PodUID != candidate.PodUID { + return false, fmt.Errorf("%w: process identity", errBackupSourceChanged) + } + return true, nil + } + encoded, err := json.Marshal(candidate) + if err != nil { + return false, err + } + patchBackupAnnotations(b, map[string]string{backupPreparationAnnotation: string(encoded)}) + return true, r.Update(ctx, b) +} + +// Current-copy workers are readers and are already protected by the owner; +// ignore only those, so requeues can observe their completion. Any other PVC +// user or unscheduled writer Job prevents stopped-source admission. +func (r *BackupReconciler) backupSourceDrained(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer, ss *appsv1.StatefulSet) (bool, error) { + var jobs batchv1.JobList + if err := r.backupAPIReader().List(ctx, &jobs, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + copyJobs := map[types.UID]bool{} + for i := range jobs.Items { + job := &jobs.Items[i] + if job.Name == b.Name && metav1.IsControlledBy(job, b) { + copyJobs[job.UID] = true + continue + } + if ((job.Status.Succeeded == 0 && !jobPermanentlyFailed(job)) || job.Status.Active > 0 || !job.DeletionTimestamp.IsZero()) && podMountsRestoreTarget(&corev1.Pod{Spec: job.Spec.Template.Spec}, gs.Name) { + return false, nil + } + } + var pods corev1.PodList + if err := r.backupAPIReader().List(ctx, &pods, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + for i := range pods.Items { + pod := &pods.Items[i] + if !liveRestorePod(pod) { + continue + } + owner := metav1.GetControllerOf(pod) + if owner != nil && owner.Kind == "Job" && copyJobs[owner.UID] { + continue + } + if metav1.IsControlledBy(pod, ss) || metav1.IsControlledBy(pod, gs) || pod.Name == gs.Name+"-0" || podMountsRestoreTarget(pod, gs.Name) { + return false, nil + } + } + return true, nil +} + +// An absent/replaced original pod cannot retain save-off state. A timeout, +// unready pod, or terminating pod with the same UID is never proof of exit. +// Empty UID is legacy state: only absence or an observed terminal pod suffices. +func backupQuiesceProcessGone(ctx context.Context, reader client.Reader, namespace, server string, uid types.UID) (bool, error) { + var pod corev1.Pod + err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: server + "-0"}, &pod) + if apierrors.IsNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + if uid != "" && pod.UID != uid { + return true, nil + } + return pod.DeletionTimestamp.IsZero() && (pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed), nil +} diff --git a/operator/internal/controller/backup_stopped_test.go b/operator/internal/controller/backup_stopped_test.go new file mode 100644 index 000000000..eb07407b8 --- /dev/null +++ b/operator/internal/controller/backup_stopped_test.go @@ -0,0 +1,389 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + "github.com/GameplanePanel/gameplane/operator/internal/agent" +) + +func stoppedBackupFixture(t *testing.T, suspended bool) (*BackupReconciler, *guardQuiescer) { + t.Helper() + r, q := guardFixture(t, guardBackup("first"), guardBackup("second")) + ctx := t.Context() + var gs gameplanev1alpha1.GameServer + key := client.ObjectKey{Namespace: "ns", Name: "server"} + if err := r.Get(ctx, key, &gs); err != nil { + t.Fatal(err) + } + gs.Spec.Suspend = suspended + if !suspended { + gs.Annotations = map[string]string{IdleAsleepSinceAnnotation: "2026-10-21T00:00:00Z"} + } + if err := r.Update(ctx, &gs); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(ctx, key, &ss); err != nil { + t.Fatal(err) + } + zero := int32(0) + ss.Spec.Replicas = &zero + if err := r.Update(ctx, &ss); err != nil { + t.Fatal(err) + } + if err := r.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + return r, q +} + +// A regression to unconditional agent quiesce would fail the first admission; +// releasing before worker drain would let the second Job appear prematurely. +func TestBackupStoppedCopiesWithoutWakingAndReleasesForNextCopy(t *testing.T) { + for _, suspended := range []bool{false, true} { + t.Run(map[bool]string{false: "idle", true: "suspended"}[suspended], func(t *testing.T) { + r, q := stoppedBackupFixture(t, suspended) + q.quiesceErr = errors.New("absent agent") + guardPass(t, r, "first") + first := guardGetBackup(t, r, "first") + var prep backupPreparation + if err := json.Unmarshal([]byte(first.Annotations[backupPreparationAnnotation]), &prep); err != nil { + t.Fatal(err) + } + if prep.Outcome != "already-stopped" || prep.TargetUID != "server-uid" || prep.WorkloadUID != "workload-uid" || prep.PodUID != "" { + t.Fatalf("wrong source proof: %+v", prep) + } + if first.Status.CaptureManifest == nil || first.Status.CaptureManifest.QuiesceOutcome != "already-stopped" { + t.Fatalf("stopped source was not recorded at admission: %+v", first.Status.CaptureManifest) + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatal(err) + } + // A restart and concurrent wake request must preserve the zero-replica fence. + r = &BackupReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme, AgentClient: q, LogReader: r.LogReader} + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[IdleWakeRequestedAnnotation] = "wake" + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if err := gr.reconcileStatefulSet(t.Context(), &gs, singlePortTemplate(true, "minecraft"), nil, nil, 1); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 { + t.Fatal("wake crossed the backup fence") + } + guardPass(t, r, "second") + var second batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &second); !apierrors.IsNotFound(err) { + t.Fatalf("second copy admitted before drain: %v", err) + } + job.Status.Succeeded = 1 + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + guardPass(t, r, "first") + if got := guardGetBackup(t, r, "first"); got.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || got.Status.SnapshotID == "" { + t.Fatalf("not restorable: %+v", got.Status) + } + first = guardGetBackup(t, r, "first") + if condition := backupConsistencyCondition(first); condition.Status != metav1.ConditionTrue || condition.Reason != "StoppedSource" { + t.Fatalf("stopped copy claims wrong consistency: %+v", condition) + } + guardPass(t, r, "second") + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &second); err != nil { + t.Fatal(err) + } + if q.quiesced != 0 || q.unquiesced != 0 { + t.Fatalf("stopped copy contacted absent agent: %+v", q) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + if gs.Spec.Suspend != suspended || gs.Annotations[IdleWakeRequestedAnnotation] != "wake" || (!suspended && gs.Annotations[IdleAsleepSinceAnnotation] != "2026-10-21T00:00:00Z") { + t.Fatal("copy changed power intent") + } + }) + } +} + +func TestBackupStoppedRequiresLiveDrainProof(t *testing.T) { + for _, tc := range []string{"desired-one", "observed-replicas", "stale-generation", "terminating-game", "foreign-writer", "pending-writer-job", "read-error"} { + t.Run(tc, func(t *testing.T) { + r, q := stoppedBackupFixture(t, true) + base := r.Client + r.APIReader = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + if ss, ok := obj.(*appsv1.StatefulSet); ok { + switch tc { + case "desired-one": + one := int32(1) + ss.Spec.Replicas = &one + case "observed-replicas": + ss.Status.Replicas = 1 + case "stale-generation": + ss.Generation = 2 + ss.Status.ObservedGeneration = 1 + } + } + return nil + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := c.List(ctx, list, opts...); err != nil { + return err + } + if pods, ok := list.(*corev1.PodList); ok { + if tc == "read-error" { + return errors.New("live list failed") + } + if tc == "terminating-game" { + now := metav1.Now() + pods.Items = append(pods.Items, corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", DeletionTimestamp: &now}, Status: corev1.PodStatus{Phase: corev1.PodSucceeded}}) + } + if tc == "foreign-writer" { + pods.Items = append(pods.Items, corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "writer", Namespace: "ns"}, Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "server-data"}}}}}}) + } + } + if jobs, ok := list.(*batchv1.JobList); ok && tc == "pending-writer-job" { + jobs.Items = append(jobs.Items, batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "writer", Namespace: "ns"}, Spec: batchv1.JobSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "server-data"}}}}}}}}) + } + return nil + }, + }) + _, _ = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "first"}}) + var job batchv1.Job + if err := base.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatalf("copy admitted without drain proof: %v", err) + } + if q.quiesced != 0 { + t.Fatal("no proven live process to quiesce") + } + }) + } +} + +func TestBackupLiveTimeoutRetainsOwnerUntilOriginalPodGone(t *testing.T) { + r, q := guardFixture(t, guardBackup("first"), guardBackup("second")) + q.quiesceErr = errors.New("live agent unreachable") + q.unquiesceErr = errors.New("live agent still unreachable") + _, _ = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "first"}}) + first := guardGetBackup(t, r, "first") + if first.Status.Phase != gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("timeout did not fail closed: %+v", first.Status) + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatal("live timeout created copy") + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + op, _, err := lifecycleOwner(&gs) + if err != nil || op.QuiescePodUID != "game-pod-uid" || !op.NeedsUnquiesce { + t.Fatalf("cleanup identity not durable: %+v %v", op, err) + } + guardPass(t, r, "first") + guardPass(t, r, "second") + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &job); !apierrors.IsNotFound(err) { + t.Fatal("timeout released the live process owner") + } + cleanupAttempts := q.unquiesced + // The replacement must not receive its predecessor's delayed save-on. + if err := r.Delete(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns"}}); err != nil { + t.Fatal(err) + } + if err := r.Create(t.Context(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "replacement"}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}); err != nil { + t.Fatal(err) + } + guardPass(t, r, "first") + if q.unquiesced != cleanupAttempts { + t.Fatal("replacement inherited original process cleanup") + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" { + t.Fatal("absent original process kept owner forever") + } +} + +func TestBackupOldAgentFailsBeforeCleanupIntent(t *testing.T) { + r, q := guardFixture(t, guardBackup("first")) + q.checkErr = errors.New("identity endpoint: status 404") + _, _ = r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "first"}}) + b := guardGetBackup(t, r, "first") + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Annotations[annoQuiesceAttempted] != "" || q.quiesced != 0 { + t.Fatalf("unsupported identity created cleanup debt: backup=%+v agent=%+v", b, q) + } + guardPass(t, r, "first") + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" { + t.Fatal("old agent failure blocked the workload upgrade") + } +} + +func TestBackupDisabledAgentPreservesRawCopyWithoutClaimingQuiesce(t *testing.T) { + r, _ := guardFixture(t, guardBackup("first")) + r.AgentClient = &agent.Client{Disabled: true} + guardPass(t, r, "first") + b := guardGetBackup(t, r, "first") + if b.Annotations[backupPreparationAnnotation] != "" || b.Annotations[annoQuiesceAttempted] != "" { + t.Fatal("disabled agent claimed preparation or quiesce") + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatalf("disabled client prevented raw copy: %v", err) + } + if err := callBackupQuiesce(t.Context(), r.AgentClient, "ns", "server", "", true); err == nil { + t.Fatal("disabled client cleared an existing cleanup obligation") + } +} + +func TestBackupDisabledAgentCannotBypassPendingLiveQuiesce(t *testing.T) { + r, _ := guardFixture(t, guardBackup("first")) + b := guardGetBackup(t, r, "first") + if ok, err := r.claimBackupTarget(t.Context(), b); err != nil || !ok { + t.Fatalf("claim: %t %v", ok, err) + } + gs, err := r.ownedBackupTarget(t.Context(), b) + if err != nil { + t.Fatal(err) + } + if ready, err := r.prepareBackupSource(t.Context(), b, gs); err != nil || !ready { + t.Fatalf("prepare: %t %v", ready, err) + } + patchBackupAnnotations(b, map[string]string{annoQuiesceAttempted: "pending"}) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + r.AgentClient = &agent.Client{Disabled: true} + if err := r.maybeQuiesce(t.Context(), b); err == nil || b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("disabled client bypassed pending quiesce: %v %+v", err, b.Status) + } +} + +func TestBackupSourceReplacementFailsBeforeCopyOrSuccess(t *testing.T) { + for _, duringRPC := range []bool{true, false} { + t.Run(map[bool]string{true: "during-quiesce", false: "during-copy"}[duringRPC], func(t *testing.T) { + r, q := guardFixture(t, guardBackup("first")) + replace := func() { + var old corev1.Pod + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server-0"}, &old); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), &old); err != nil { + t.Fatal(err) + } + old.UID, old.ResourceVersion = "replacement", "" + if err := r.Create(t.Context(), &old); err != nil { + t.Fatal(err) + } + } + if duringRPC { + q.afterQuiesce = replace + } else { + guardPass(t, r, "first") + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { + t.Fatal(err) + } + job.Status.Succeeded = 1 + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + replace() + } + guardPass(t, r, "first") + if got := guardGetBackup(t, r, "first"); got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || got.Status.SnapshotID != "" { + t.Fatalf("replacement allowed a restorable copy: %+v", got.Status) + } + if duringRPC { + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { + t.Fatal("replacement during RPC created a copy") + } + } + if q.unquiesced != 0 { + t.Fatal("replacement received the old process cleanup") + } + }) + } +} + +func TestBackupCleanupRequiresOriginalProcessExit(t *testing.T) { + for _, state := range []string{"absent", "replacement", "terminal", "terminating", "unready", "legacy-running", "read-error"} { + t.Run(state, func(t *testing.T) { + r, _ := guardFixture(t) + uid := types.UID("game-pod-uid") + var pod corev1.Pod + key := client.ObjectKey{Namespace: "ns", Name: "server-0"} + if err := r.Get(t.Context(), key, &pod); err != nil { + t.Fatal(err) + } + reader := interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if state == "read-error" { + return errors.New("apiserver unavailable") + } + if state == "absent" { + return apierrors.NewNotFound(corev1.Resource("pods"), key.Name) + } + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + p := obj.(*corev1.Pod) + switch state { + case "replacement": + p.UID = "successor" + case "terminal": + p.Status.Phase = corev1.PodFailed + case "terminating": + now := metav1.Now() + p.DeletionTimestamp = &now + case "unready": + p.Status.Phase = corev1.PodPending + } + return nil + }}) + if state == "legacy-running" { + uid = "" + } + gone, err := backupQuiesceProcessGone(t.Context(), reader, "ns", "server", uid) + wantGone := state == "absent" || state == "replacement" || state == "terminal" + if gone != wantGone || (err != nil) != (state == "read-error") { + t.Fatalf("exit proof = %t, %v; want gone %t", gone, err, wantGone) + } + }) + } +} diff --git a/operator/internal/controller/backup_volumesnapshot.go b/operator/internal/controller/backup_volumesnapshot.go index b454f6fad..e90a0b6c9 100644 --- a/operator/internal/controller/backup_volumesnapshot.go +++ b/operator/internal/controller/backup_volumesnapshot.go @@ -38,6 +38,9 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( if err := r.backupAPIReader().Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: b.Name}, &existing); err != nil && !apierrors.IsNotFound(err) { return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot API unavailable: %v", err)) } + if legacyBackupOwesUnquiesce(b) && metav1.IsControlledBy(&existing, b) { + return r.observeVolumeSnapshot(ctx, b, &existing) + } // Quiesce first (idempotent across requeues). A CSI snapshot is only // crash-consistent; flushing the game to disk first (RCON save-all, // etc.) makes it application-consistent. No-op when the backup opts out @@ -69,6 +72,10 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot creation failed: %v", err)) } + return r.observeVolumeSnapshot(ctx, b, vs) +} + +func (r *BackupReconciler) observeVolumeSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup, vs *snapshotv1.VolumeSnapshot) (ctrl.Result, error) { switch { case vs.Status != nil && vs.Status.Error != nil && vs.Status.Error.Message != nil: // The CSI driver rejected the snapshot. Surface it and unquiesce so @@ -112,6 +119,7 @@ func (r *BackupReconciler) markVolumeSnapshotRunning( Reason: string(gameplanev1alpha1.BackupPhaseRunning), ObservedGeneration: b.Generation, }) + b.Status.Conditions = upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) if err := r.Status().Update(ctx, b); err != nil { return ctrl.Result{}, err } @@ -152,6 +160,7 @@ func (r *BackupReconciler) completeVolumeSnapshot( Reason: "Succeeded", ObservedGeneration: b.Generation, }) + b.Status.Conditions = upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) if err := r.Status().Update(ctx, b); err != nil { return ctrl.Result{}, err } diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 67e8be4e3..4c76c16d5 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -100,6 +100,8 @@ type GameServerReconciler struct { // during a soft stop. May be nil (or a disabled client) in dev clusters, // in which case the operator falls back to a timed scale-to-zero. AgentClient AgentStopper + // OperationQuiescer recovers orphaned backups before admitting a wipe. + OperationQuiescer AgentQuiescer // PodAttacher runs the module-declared stop sequence over a pod attach // to the game container's stdin, for consoleMode: pty games that @@ -1194,6 +1196,45 @@ func (r *GameServerReconciler) desiredReplicas( idle idleState, inventory ...*wipeWorkerState, ) (int32, time.Duration, error) { + // Read ownership live before restart acknowledgment or any stop RPC. The + // stop timestamp uses a checked Update, so a claim racing that write wins + // either before the command or after the stop has become visible. + var live gameplanev1alpha1.GameServer + if err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &live); err != nil { + return 0, 0, err + } + if live.UID != gs.UID { + return 0, 0, fmt.Errorf("GameServer changed identity before lifecycle decision") + } + if op, held, err := lifecycleOwner(&live); err != nil { + return 0, 0, err + } else if held && op.InitialStartup && len(op.Recovery) == 0 { + if live.Spec.Suspend { + return 0, 5 * time.Second, nil + } + return 1, 5 * time.Second, nil + } else if held && (op.Kind == "Backup" || len(op.Recovery) != 0) { + var ss appsv1.StatefulSet + err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if apierrors.IsNotFound(err) { + return 0, 5 * time.Second, nil + } + if err != nil { + return 0, 0, err + } + if !metav1.IsControlledBy(&ss, gs) { + return 0, 0, fmt.Errorf("protected StatefulSet belongs to another GameServer") + } + if ss.Spec.Replicas == nil { + return 1, 5 * time.Second, nil + } + return *ss.Spec.Replicas, 5 * time.Second, nil + } else if held && (live.Annotations[restoreGuardAnnotation] != gs.Annotations[restoreGuardAnnotation] || live.Annotations[wipeGuardAnnotation] != gs.Annotations[wipeGuardAnnotation]) { + // This reconcile rendered config from an older GameServer. Use the + // live owner only for stopping; do not replace the caller's snapshot + // and mix a newly-read spec with already-materialized configuration. + return r.softStop(ctx, &live, tmpl) + } if gs.Annotations[wipeGuardAnnotation] != "" || pendingWipe(gs) { replicas, poll, err := r.softStop(ctx, gs, tmpl) if err == nil && r.settledWipeFailure(ctx, gs, wipeInventory(inventory)) { @@ -1456,6 +1497,33 @@ func (r *GameServerReconciler) reconcileStatefulSet( if owner := metav1.GetControllerOf(ss); owner != nil && owner.UID != live.UID { return fmt.Errorf("StatefulSet belongs to a different GameServer identity") } + op, held, err := lifecycleOwner(&live) + if err != nil { + return err + } + if held { + if ss.Annotations == nil { + ss.Annotations = map[string]string{} + } + ss.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + if op.InitialStartup { + actualReplicas = replicas + delete(ss.Annotations, restoreGuardAnnotation) + } + if op.Kind == "Backup" || len(op.Recovery) != 0 { + if !ss.CreationTimestamp.IsZero() || ss.ResourceVersion != "" { + // Preserve replicas AND the complete pod template, including + // config hashes, images, init containers and mount topology. + return nil + } + // No workload exists yet. Create it stopped; this gives backup + // admission a real object to fence against stale creates. + actualReplicas = 0 + replicas = 0 + } + } else { + delete(ss.Annotations, lifecycleOperationAnnotation) + } if live.Annotations[restoreGuardAnnotation] == "" { // Restore cleanup can race a callback that restamps the workload // before the checked GameServer release. That release guarantees @@ -1464,6 +1532,9 @@ func (r *GameServerReconciler) reconcileStatefulSet( delete(ss.Annotations, restoreGuardAnnotation) } for _, annotation := range []string{restoreGuardAnnotation, wipeGuardAnnotation} { + if annotation == restoreGuardAnnotation && held && op.InitialStartup { + continue + } if guard := live.Annotations[annotation]; guard != "" { actualReplicas = 0 if ss.Annotations == nil { @@ -1492,6 +1563,11 @@ func (r *GameServerReconciler) reconcileStatefulSet( } } ss.Spec.Replicas = &actualReplicas + if held && (!ss.CreationTimestamp.IsZero() || ss.ResourceVersion != "") { + // Restore and wipe may stop the existing workload, but rollout + // waits until the operation and its worker cleanup have finished. + return nil + } ss.Spec.ServiceName = gs.Name ss.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} ss.Spec.Template.Labels = labels @@ -2282,6 +2358,7 @@ func buildAgentContainer( env := []corev1.EnvVar{ {Name: "GAMEPLANE_SERVER_NAME", Value: gs.Name}, {Name: "GAMEPLANE_SERVER_UID", Value: string(gs.UID)}, + {Name: "GAMEPLANE_POD_UID", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.uid"}}}, {Name: "GAMEPLANE_TEMPLATE", Value: tmpl.Name}, {Name: "GAMEPLANE_GAME", Value: tmpl.Spec.Game}, // Games without RCON (consoleMode pty/none) must not have the diff --git a/operator/internal/controller/gameserver_idle.go b/operator/internal/controller/gameserver_idle.go index 9869ece72..055071975 100644 --- a/operator/internal/controller/gameserver_idle.go +++ b/operator/internal/controller/gameserver_idle.go @@ -292,6 +292,17 @@ func untilNext(next, now time.Time) time.Duration { func (r *GameServerReconciler) reconcileIdle( ctx context.Context, gs *gameplanev1alpha1.GameServer, ) (idleState, *gameplanev1alpha1.IdleStatus, time.Duration, error, error) { + // Keep both sleep and wake intent unchanged while an operation owns the + // workload. In particular, waking a stopped backup source is deferred. + if _, held, err := lifecycleOwner(gs); err != nil { + return idleAwake, nil, 0, nil, err + } else if held { + state := idleAwake + if parseAnnotationTime(gs, IdleAsleepSinceAnnotation) != nil { + state = idleAsleep + } + return state, gs.Status.Idle.DeepCopy(), idleReevaluate, nil, nil + } in := idleInputs{ spec: gs.Spec.Idle, suspended: gs.Spec.Suspend, @@ -349,7 +360,7 @@ func (r *GameServerReconciler) applyIdleTransition( gs.Annotations[IdleWakeCompletedAnnotation] = tok } } - if err := r.Patch(ctx, gs, client.MergeFrom(base)); err != nil { + if err := r.Patch(ctx, gs, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})); err != nil { return fmt.Errorf("patch idle annotations on %s/%s: %w", gs.Namespace, gs.Name, err) } return nil diff --git a/operator/internal/controller/gameserver_restart.go b/operator/internal/controller/gameserver_restart.go index 33159927d..af1ddf48d 100644 --- a/operator/internal/controller/gameserver_restart.go +++ b/operator/internal/controller/gameserver_restart.go @@ -79,5 +79,5 @@ func (r *GameServerReconciler) ackRestart(ctx context.Context, gs *gameplanev1al } gs.Annotations[RestartCompletedAnnotation] = gs.Annotations[RestartRequestedAnnotation] delete(gs.Annotations, stopRequestedAtAnnotation) - return r.Patch(ctx, gs, client.MergeFrom(base)) + return r.Patch(ctx, gs, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) } diff --git a/operator/internal/controller/gameserver_wipe.go b/operator/internal/controller/gameserver_wipe.go index ca96d7d63..c59ecf998 100644 --- a/operator/internal/controller/gameserver_wipe.go +++ b/operator/internal/controller/gameserver_wipe.go @@ -51,7 +51,7 @@ func (r *GameServerReconciler) reconcileWipe( if err != nil { return err } - if gs.Annotations[restoreGuardAnnotation] != "" { + if gs.Annotations[restoreGuardAnnotation] != "" || gs.Annotations[backupGuardAnnotation] != "" { return nil } req := gs.Annotations[WipeRequestedAnnotation] diff --git a/operator/internal/controller/gameserver_wipe_guard.go b/operator/internal/controller/gameserver_wipe_guard.go index f86a1969d..bdf00bfc8 100644 --- a/operator/internal/controller/gameserver_wipe_guard.go +++ b/operator/internal/controller/gameserver_wipe_guard.go @@ -165,8 +165,11 @@ func (r *GameServerReconciler) ensureWipeGuard(ctx context.Context, gs *gameplan return false, fmt.Errorf("GameServer changed identity before wipe acquisition") } *gs = live - if gs.Annotations[wipeGuardAnnotation] != "" || gs.Annotations[restoreGuardAnnotation] != "" { - return false, nil + if token := gs.Annotations[wipeGuardAnnotation]; token != "" { + ready, err := claimLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, wipeLifecycleIdentity(gs, token), r.OperationQuiescer) + // Upgrading an existing guard does not require another pass. Only + // unfinished predecessor cleanup must prevent this wipe proceeding. + return err == nil && !ready, err } state := wipeInventory(inventory) if err := r.loadWipeWorkers(ctx, gs, state); err != nil { @@ -189,20 +192,8 @@ func (r *GameServerReconciler) ensureWipeGuard(ctx context.Context, gs *gameplan if token == "" { token = wipeOrphanGuard } - if gs.Annotations == nil { - gs.Annotations = make(map[string]string) - } - gs.Annotations[wipeGuardAnnotation] = token - // Recovery without a pending request is cleanup, including legacy orphan - // pods with no token. The guard fences live workers without changing the - // current resume/stop intent. Pending and failed matching wipes suspend. - if pendingWipe(gs) { - gs.Spec.Suspend = true - } - if err := r.Update(ctx, gs); err != nil { - return false, err - } - return true, nil + _, err = claimLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, wipeLifecycleIdentity(gs, token), r.OperationQuiescer) + return err == nil && gs.Annotations[wipeGuardAnnotation] == token, err } // Stamp the owned workload with a checked write before workers launch. A stale @@ -299,6 +290,9 @@ func (r *GameServerReconciler) releaseWipeGuard(ctx context.Context, gs *gamepla if err := r.clearWipeWorkloadGuard(ctx, &live, token); err != nil { return err } + if err := clearLifecycleOperation(ctx, r.Client, r.wipeReader(), &live, wipeLifecycleIdentity(&live, token)); err != nil { + return err + } if success { live.Annotations[WipeCompletedAnnotation] = token live.Spec.Suspend = false @@ -307,6 +301,7 @@ func (r *GameServerReconciler) releaseWipeGuard(ctx context.Context, gs *gamepla // Transfer only after old Jobs and live pods have drained; next pass // fences the workload for this new token before launching a worker. live.Annotations[wipeGuardAnnotation] = live.Annotations[WipeRequestedAnnotation] + live.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: wipeLifecycleIdentity(&live, live.Annotations[WipeRequestedAnnotation])}) live.Spec.Suspend = true } else { delete(live.Annotations, wipeGuardAnnotation) diff --git a/operator/internal/controller/helpers.go b/operator/internal/controller/helpers.go index be11001c0..8bd3890d6 100644 --- a/operator/internal/controller/helpers.go +++ b/operator/internal/controller/helpers.go @@ -13,8 +13,10 @@ import ( ) // upsertCondition replaces a condition by Type if present, otherwise -// appends. Sets LastTransitionTime to now when the status flips. +// appends. Sets LastTransitionTime to now when the status flips. Preserve the +// input slice so callers can compare old and new conditions before persisting. func upsertCondition(conds []metav1.Condition, c metav1.Condition) []metav1.Condition { + conds = append([]metav1.Condition(nil), conds...) now := metav1.Now() for i := range conds { if conds[i].Type != c.Type { diff --git a/operator/internal/controller/helpers_envtest_test.go b/operator/internal/controller/helpers_envtest_test.go index 7db1ea7c1..fbb6dccb0 100644 --- a/operator/internal/controller/helpers_envtest_test.go +++ b/operator/internal/controller/helpers_envtest_test.go @@ -774,6 +774,14 @@ func (f *fakeAgent) Unquiesce(_ context.Context, _, _ string) error { return f.unquiesceErr } +func (f *fakeAgent) CheckPod(context.Context, string, string, string) error { return nil } +func (f *fakeAgent) QuiescePod(ctx context.Context, namespace, server, _ string) error { + return f.Quiesce(ctx, namespace, server) +} +func (f *fakeAgent) UnquiescePod(ctx context.Context, namespace, server, _ string) error { + return f.Unquiesce(ctx, namespace, server) +} + func (f *fakeAgent) quiesceCount() int { f.mu.Lock() defer f.mu.Unlock() @@ -797,13 +805,32 @@ func (f *fakeLogReader) BackupLogs(_ context.Context, _, _ string) (io.ReadClose return io.NopCloser(strings.NewReader(f.body)), nil } -// seedGameServer creates a bare GameServer so reconcilers that resolve -// Backup/Restore serverRefs find their target. No GameServer reconciler -// runs in these suites, so the object stays inert (no children). +// seedGameServer creates a GameServer and observed workload for backup tests. +// No StatefulSet controller runs in envtest; report its generation explicitly +// so backup admission can exercise the real workload fence. func seedGameServer(t *testing.T, ns, name string) { t.Helper() gs := buildGameServer(ns, name, "seed-template") if err := k8sClient.Create(context.Background(), gs); err != nil { t.Fatalf("seed gameserver %s/%s: %v", ns, name, err) } + seedBackupWorkload(t, gs) +} + +func seedBackupWorkload(t *testing.T, gs *gameplanev1alpha1.GameServer) { + t.Helper() + name, ns := gs.Name, gs.Namespace + one := int32(1) + labels := map[string]string{"app": name} + ss := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "GameServer", Name: name, UID: gs.UID, Controller: ownerBoolPtr(true)}}}, + Spec: appsv1.StatefulSetSpec{Replicas: &one, ServiceName: name, Selector: &metav1.LabelSelector{MatchLabels: labels}, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: labels}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "game", Image: "test/game:fixture"}}}}}, + } + if err := k8sClient.Create(context.Background(), ss); err != nil { + t.Fatalf("seed workload: %v", err) + } + ss.Status.ObservedGeneration = ss.Generation + if err := k8sClient.Status().Update(context.Background(), ss); err != nil { + t.Fatalf("observe workload: %v", err) + } } diff --git a/operator/internal/controller/helpers_extra_test.go b/operator/internal/controller/helpers_extra_test.go index 85a8d2ccf..ea6ab4ce0 100644 --- a/operator/internal/controller/helpers_extra_test.go +++ b/operator/internal/controller/helpers_extra_test.go @@ -53,6 +53,19 @@ func TestUpsertCondition_StatusFlip_BumpsTimestamp(t *testing.T) { } } +func TestUpsertConditionPreservesComparisonBase(t *testing.T) { + for _, status := range []metav1.ConditionStatus{metav1.ConditionFalse, metav1.ConditionTrue} { + t.Run(string(status), func(t *testing.T) { + original := metav1.Condition{Type: "SnapshotCleaned", Status: metav1.ConditionFalse, Reason: "Pending", LastTransitionTime: metav1.Now()} + conditions := []metav1.Condition{original} + updated := upsertCondition(conditions, metav1.Condition{Type: original.Type, Status: status, Reason: "Updated"}) + if conditions[0] != original || sameConditions(conditions, updated) { + t.Fatalf("upsert hid the change from status persistence: before=%+v after=%+v", conditions, updated) + } + }) + } +} + func TestSameConditions(t *testing.T) { a := []metav1.Condition{ {Type: "Ready", Status: metav1.ConditionTrue, Reason: "Up"}, diff --git a/operator/internal/controller/lifecycle_operation.go b/operator/internal/controller/lifecycle_operation.go new file mode 100644 index 000000000..555a85054 --- /dev/null +++ b/operator/internal/controller/lifecycle_operation.go @@ -0,0 +1,432 @@ +package controller + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const lifecycleOperationAnnotation = "operations.gameplane.local/owner" + +var errLifecycleGenerationChanged = errors.New("lifecycle operation spec changed after admission") + +// The GameServer is the arbitration object. A wipe has no separate CR, so its +// request token and GameServer UID identify the operation. Generation is pinned +// at admission; subsequent power requests must not change that identity. +type lifecycleIdentity struct { + Kind string `json:"kind"` + Name string `json:"name"` + UID types.UID `json:"uid"` + TargetUID types.UID `json:"targetUID"` + Generation int64 `json:"generation"` + Token string `json:"token,omitempty"` + NeedsUnquiesce bool `json:"needsUnquiesce,omitempty"` + QuiescePodUID types.UID `json:"quiescePodUID,omitempty"` + // InitialStartup permits only the first workload of a CSI restore's new + // target (including WaitForFirstConsumer PVCs), while other operations wait. + InitialStartup bool `json:"initialStartup,omitempty"` +} + +type lifecycleOperation struct { + lifecycleIdentity + // Recovery survives deletion of a successor while it is draining an older + // owner's workers. No new work is authorized until every entry is cleaned. + Recovery []lifecycleIdentity `json:"recovery,omitempty"` +} + +func lifecycleOperationJSON(op lifecycleOperation) string { + value, _ := json.Marshal(op) + return string(value) +} + +func parseLifecycleOperation(value string) (lifecycleOperation, error) { + var op lifecycleOperation + if err := json.Unmarshal([]byte(value), &op); err != nil { + return op, fmt.Errorf("decode lifecycle owner: %w", err) + } + for _, id := range append([]lifecycleIdentity{op.lifecycleIdentity}, op.Recovery...) { + if id.Name == "" || id.UID == "" || id.TargetUID == "" || (id.Kind != "Backup" && id.Kind != "Restore" && id.Kind != "Wipe") || (id.Kind == "Wipe" && id.Token == "") { + return op, fmt.Errorf("invalid lifecycle owner identity") + } + if id.TargetUID != op.TargetUID { + return op, fmt.Errorf("lifecycle recovery target changed identity") + } + } + return op, nil +} + +func backupLifecycleIdentity(gs *gameplanev1alpha1.GameServer, b *gameplanev1alpha1.Backup, quiesce bool) lifecycleIdentity { + return lifecycleIdentity{Kind: "Backup", Name: b.Name, UID: b.UID, TargetUID: gs.UID, Generation: b.Generation, NeedsUnquiesce: quiesce} +} + +func restoreLifecycleIdentity(gs *gameplanev1alpha1.GameServer, rs *gameplanev1alpha1.Restore) lifecycleIdentity { + return lifecycleIdentity{Kind: "Restore", Name: rs.Name, UID: rs.UID, TargetUID: gs.UID, Generation: rs.Generation} +} + +func wipeLifecycleIdentity(gs *gameplanev1alpha1.GameServer, token string) lifecycleIdentity { + return lifecycleIdentity{Kind: "Wipe", Name: gs.Name, UID: gs.UID, TargetUID: gs.UID, Generation: gs.Generation, Token: token} +} + +func sameLifecycleOwner(a, b lifecycleIdentity) bool { + return a.Kind == b.Kind && a.Name == b.Name && a.UID == b.UID && a.TargetUID == b.TargetUID && a.Token == b.Token +} + +// Existing guards remain readable during upgrades and provide the compatibility +// markers used by stop/drain cleanup. They are never independent permission to +// start new work. Contradictory legacy guards fail closed until their owners +// complete cleanup; migration must not select one concurrent writer as safe. +func lifecycleOwner(gs *gameplanev1alpha1.GameServer) (lifecycleOperation, bool, error) { + if value := gs.Annotations[lifecycleOperationAnnotation]; value != "" { + op, err := parseLifecycleOperation(value) + if err == nil && op.TargetUID != gs.UID { + err = fmt.Errorf("lifecycle owner belongs to a replaced GameServer") + } + return op, true, err + } + op := lifecycleOperation{} + count := 0 + if value := gs.Annotations[backupGuardAnnotation]; value != "" { + id, err := parseBackupIdentity(value) + if err != nil { + return op, true, err + } + op.lifecycleIdentity = lifecycleIdentity{Kind: "Backup", Name: id.Name, UID: id.UID, TargetUID: gs.UID, NeedsUnquiesce: true} + lineage, err := parseBackupRecoveryLineage(gs.Annotations[backupRecoveryLineageAnnotation]) + if err != nil { + return op, true, err + } + for _, predecessor := range lineage { + op.Recovery = append(op.Recovery, lifecycleIdentity{Kind: "Backup", Name: predecessor.Name, UID: predecessor.UID, TargetUID: gs.UID, NeedsUnquiesce: true}) + } + count++ + } + if value := gs.Annotations[restoreGuardAnnotation]; value != "" { + id, err := parseRestoreIdentity(value) + if err != nil { + return op, true, err + } + op.lifecycleIdentity = lifecycleIdentity{Kind: "Restore", Name: id.Name, UID: id.UID, TargetUID: gs.UID} + count++ + } + if token := gs.Annotations[wipeGuardAnnotation]; token != "" { + op.lifecycleIdentity = wipeLifecycleIdentity(gs, token) + count++ + } + if count > 1 { + return op, true, fmt.Errorf("overlapping legacy lifecycle owners require cleanup before new work") + } + return op, count != 0, nil +} + +func lifecycleOwnerExists(ctx context.Context, reader client.Reader, namespace string, id lifecycleIdentity) (bool, error) { + var obj client.Object + switch id.Kind { + case "Backup": + obj = &gameplanev1alpha1.Backup{} + case "Restore": + obj = &gameplanev1alpha1.Restore{} + case "Wipe": + // Only the GameServer reconciler may cancel/replace a wipe token after + // its workers drain. A removed request is not proof of worker absence. + return true, nil + default: + return false, fmt.Errorf("unknown lifecycle operation %q", id.Kind) + } + err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: id.Name}, obj) + if apierrors.IsNotFound(err) { + return false, nil + } + return err == nil && obj.GetUID() == id.UID, err +} + +// claimLifecycleOperation refreshes gs with the uncached reader and writes the +// shared owner plus compatibility guard in ONE resourceVersion-checked Update. +// Ownership is retained during recovery. A losing claimant performs no cleanup +// or worker creation. No timeout ever permits stealing from an existing owner. +func claimLifecycleOperation(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer, want lifecycleIdentity, quiescer AgentQuiescer) (bool, error) { + var live gameplanev1alpha1.GameServer + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &live); err != nil { + return false, err + } + if live.UID != want.TargetUID || !live.DeletionTimestamp.IsZero() { + return false, fmt.Errorf("lifecycle target changed identity or is being deleted") + } + *gs = live + op, occupied, err := lifecycleOwner(gs) + if err != nil { + return false, err + } + if occupied && !sameLifecycleOwner(op.lifecycleIdentity, want) { + exists, err := lifecycleOwnerExists(ctx, reader, gs.Namespace, op.lifecycleIdentity) + if err != nil || exists { + return false, err + } + op.Recovery = append(op.Recovery, op.lifecycleIdentity) + // A force-deleted restore may have left partial data. Preserve its + // suspension even when the next operation is a backup. + if op.Kind == "Restore" { + gs.Spec.Suspend = true + } + op.lifecycleIdentity = want + } else if occupied { + if gs.Annotations[lifecycleOperationAnnotation] != "" && op.Kind != "Wipe" && op.Generation != want.Generation { + return false, errLifecycleGenerationChanged + } + if gs.Annotations[lifecycleOperationAnnotation] == "" { + op.lifecycleIdentity = want + } + } else { + if want.Kind != "Wipe" { + workers, err := wipeWorkers(ctx, reader, gs) + if err != nil || pendingWipe(gs) || workers.busy { + return false, err + } + } + // A stop command already sent may still save data and exit. Waiting + // here also lets pending restarts finish instead of freezing mid-stop. + if want.Kind == "Backup" { + pending, err := backupLifecycleTransitionPending(ctx, reader, gs) + if err != nil || pending { + return false, err + } + } + busy, err := legacyLifecycleCopiesLive(ctx, c, reader, gs, want) + if err != nil || busy { + return false, err + } + op = lifecycleOperation{lifecycleIdentity: want} + } + before := gs.DeepCopy() + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + delete(gs.Annotations, backupGuardAnnotation) + delete(gs.Annotations, restoreGuardAnnotation) + delete(gs.Annotations, wipeGuardAnnotation) + switch want.Kind { + case "Backup": + gs.Annotations[backupGuardAnnotation] = backupIdentityJSON(want.Name, want.UID) + case "Restore": + gs.Annotations[restoreGuardAnnotation] = restoreIdentityJSON(want.Name, want.UID) + if !op.InitialStartup { + gs.Spec.Suspend = true + } + case "Wipe": + gs.Annotations[wipeGuardAnnotation] = want.Token + if pendingWipe(gs) { + gs.Spec.Suspend = true + } + } + // An acquired owner can be re-read many times while its worker runs. + // Avoid needless resourceVersion churn, but always CAS new admission. + if before.Annotations[lifecycleOperationAnnotation] != gs.Annotations[lifecycleOperationAnnotation] || before.Spec.Suspend != gs.Spec.Suspend { + if err := c.Update(ctx, gs); err != nil { + return false, err + } + } + return recoverLifecycleOperation(ctx, c, reader, gs, op, quiescer) +} + +// New code must also respect workers started before shared ownership existed, +// including quiesce=false copies which had no old backup guard at all. +func legacyLifecycleCopiesLive(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer, want lifecycleIdentity) (bool, error) { + var backups gameplanev1alpha1.BackupList + if err := reader.List(ctx, &backups, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + r := &BackupReconciler{Client: c, APIReader: reader, Scheme: c.Scheme()} + for i := range backups.Items { + b := &backups.Items[i] + if b.Spec.ServerRef.Name != gs.Name || (want.Kind == "Backup" && b.UID == want.UID) { + continue + } + if legacyBackupOwesUnquiesce(b) { + return true, nil + } + busy, err := r.backupWorkersLive(ctx, gs.Namespace, backupIdentity{Name: b.Name, UID: b.UID}, false) + if err != nil || busy { + return busy, err + } + } + return false, nil +} + +func recoverLifecycleOperation(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer, op lifecycleOperation, quiescer AgentQuiescer) (bool, error) { + for _, predecessor := range op.Recovery { + var busy bool + var err error + switch predecessor.Kind { + case "Backup": + r := &BackupReconciler{Client: c, APIReader: reader, Scheme: c.Scheme()} + busy, err = r.backupWorkersLive(ctx, gs.Namespace, backupIdentity{Name: predecessor.Name, UID: predecessor.UID}, true) + case "Restore": + r := &RestoreReconciler{Client: c, APIReader: reader, Scheme: c.Scheme()} + busy, err = r.restoreWorkersLive(ctx, gs.Namespace, restoreIdentity{Name: predecessor.Name, UID: predecessor.UID}, true) + default: + return false, fmt.Errorf("unsupported orphan lifecycle owner %q", predecessor.Kind) + } + if err != nil || busy { + return false, err + } + } + for _, predecessor := range op.Recovery { + if !predecessor.NeedsUnquiesce { + continue + } + gone, err := backupQuiesceProcessGone(ctx, reader, gs.Namespace, gs.Name, predecessor.QuiescePodUID) + if err != nil { + return false, err + } + if gone { + continue + } + if !agentQuiesceAvailable(quiescer) { + return false, fmt.Errorf("cannot recover orphaned quiesce without an agent client") + } + if err := callBackupQuiesce(ctx, quiescer, gs.Namespace, gs.Name, predecessor.QuiescePodUID, true); err != nil { + return false, fmt.Errorf("recover orphaned lifecycle quiesce: %w", err) + } + } + if len(op.Recovery) > 0 { + op.Recovery = nil + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + delete(gs.Annotations, backupRecoveryLineageAnnotation) + if err := c.Update(ctx, gs); err != nil { + return false, err + } + } + return true, nil +} + +func backupLifecycleTransitionPending(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer) (bool, error) { + if tokenPending(gs, RestartRequestedAnnotation, RestartCompletedAnnotation) { + return true, nil + } + if gs.Annotations[stopRequestedAtAnnotation] == "" { + return false, nil + } + var ss appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return true, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&ss, gs) || ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation { + return true, nil + } + var pods corev1.PodList + if err := reader.List(ctx, &pods, client.InNamespace(gs.Namespace)); err != nil { + return true, err + } + for i := range pods.Items { + if liveRestorePod(&pods.Items[i]) && (pods.Items[i].Name == gs.Name+"-0" || podMountsRestoreTarget(&pods.Items[i], gs.Name)) { + return true, nil + } + } + return false, nil +} + +// Carry cleanup intent on the surviving arbitration object as well as Backup. +// A force-deleted Backup cannot hide a partial/ambiguous save-off RPC. +func (r *BackupReconciler) setLifecycleUnquiesceIntent(ctx context.Context, b *gameplanev1alpha1.Backup, needed bool) error { + gs, err := r.ownedBackupTarget(ctx, b) + if err != nil || gs == nil || gs.Annotations[lifecycleOperationAnnotation] == "" { + return err + } + op, err := parseLifecycleOperation(gs.Annotations[lifecycleOperationAnnotation]) + if err != nil { + return err + } + var podUID types.UID + if needed { + preparation, err := parseBackupPreparation(b) + if err != nil { + return err + } + if preparation != nil { + podUID = preparation.PodUID + } + } + if op.NeedsUnquiesce == needed && (!needed || op.QuiescePodUID == podUID) { + return nil + } + op.NeedsUnquiesce = needed + if needed { + op.QuiescePodUID = podUID + } + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(op) + return r.Update(ctx, gs) +} + +// fenceBackupWorkload performs a checked write to the workload before a copy +// can start. A stale workload Update either precedes this fence (and is fully +// observed before admission) or conflicts with it. StatefulSet mutation also +// reads the live owner inside its callback and preserves the whole pod template. +func fenceBackupWorkload(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer) (bool, error) { + var ss appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + if apierrors.IsNotFound(err) { + return false, nil // wait for the workload reconciler to create a fenced set + } + return false, err + } + if !metav1.IsControlledBy(&ss, gs) { + return false, fmt.Errorf("backup StatefulSet is not controlled by the bound GameServer") + } + value := gs.Annotations[lifecycleOperationAnnotation] + if value == "" { + return false, fmt.Errorf("backup has no shared lifecycle owner") + } + if ss.Annotations[lifecycleOperationAnnotation] != value { + if ss.Annotations == nil { + ss.Annotations = map[string]string{} + } + ss.Annotations[lifecycleOperationAnnotation] = value + if err := c.Update(ctx, &ss); err != nil { + return false, err + } + } + if ss.Status.ObservedGeneration < ss.Generation || (ss.Status.CurrentRevision != "" && ss.Status.CurrentRevision != ss.Status.UpdateRevision) { + return false, nil + } + return true, nil +} + +// clearLifecycleOperation prepares the release; the caller removes its legacy +// guard and commits the GameServer Update only AFTER worker/agent cleanup and a +// durable completion decision. Release never changes the caller's power intent. +func clearLifecycleOperation(ctx context.Context, c client.Client, reader client.Reader, gs *gameplanev1alpha1.GameServer, want lifecycleIdentity) error { + value := gs.Annotations[lifecycleOperationAnnotation] + if value == "" { + return nil + } + op, err := parseLifecycleOperation(value) + if err != nil { + return err + } + if !sameLifecycleOwner(op.lifecycleIdentity, want) || len(op.Recovery) != 0 { + return fmt.Errorf("cannot release lifecycle ownership before recovery completes") + } + var ss appsv1.StatefulSet + err = reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil && metav1.IsControlledBy(&ss, gs) && ss.Annotations[lifecycleOperationAnnotation] == value { + delete(ss.Annotations, lifecycleOperationAnnotation) + if err := c.Update(ctx, &ss); err != nil { + return err + } + } + delete(gs.Annotations, lifecycleOperationAnnotation) + return nil +} diff --git a/operator/internal/controller/lifecycle_operation_envtest_test.go b/operator/internal/controller/lifecycle_operation_envtest_test.go new file mode 100644 index 000000000..7dc623a1a --- /dev/null +++ b/operator/internal/controller/lifecycle_operation_envtest_test.go @@ -0,0 +1,73 @@ +//go:build envtest + +package controller + +import ( + "sync" + "testing" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// A real apiserver arbitrates across controller types; no process-local mutex +// or informer timing can allow both claimants to create workers. +func TestLifecycleCASAcrossControllers(t *testing.T) { + ns := newNamespace(t) + seedGameServer(t, ns, "target") + b := buildBackup(ns, "backup", "target", "repo") + b.Spec.Quiesce = false + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: ns}, Spec: gameplanev1alpha1.RestoreSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: "target"}, BackupRef: gameplanev1alpha1.LocalObjectRef{Name: "source"}}} + if err := k8sClient.Create(t.Context(), b); err != nil { + t.Fatal(err) + } + if err := k8sClient.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + var gs gameplanev1alpha1.GameServer + if err := k8sClient.Get(t.Context(), client.ObjectKey{Namespace: ns, Name: "target"}, &gs); err != nil { + t.Fatal(err) + } + type outcome struct { + claimed bool + err error + } + results := make(chan outcome, 2) + start := make(chan struct{}) + var ready sync.WaitGroup + ready.Add(2) + for _, want := range []lifecycleIdentity{backupLifecycleIdentity(&gs, b, false), restoreLifecycleIdentity(&gs, rs)} { + go func() { + target := gs.DeepCopy() + ready.Done() + <-start + claimed, err := claimLifecycleOperation(t.Context(), k8sClient, k8sClient, target, want, nil) + results <- outcome{claimed: claimed, err: err} + }() + } + ready.Wait() + close(start) + winners := 0 + for range 2 { + result := <-results + if result.err != nil && !apierrors.IsConflict(result.err) { + t.Fatalf("claim: %v", result.err) + } + if result.claimed { + winners++ + } + } + if winners != 1 { + t.Fatalf("exclusive admission had %d winners", winners) + } + if err := k8sClient.Get(t.Context(), client.ObjectKeyFromObject(&gs), &gs); err != nil { + t.Fatal(err) + } + op, held, err := lifecycleOwner(&gs) + if err != nil || !held || op.TargetUID != gs.UID { + t.Fatalf("owner was not durable: held=%t owner=%+v err=%v", held, op, err) + } +} diff --git a/operator/internal/controller/lifecycle_operation_test.go b/operator/internal/controller/lifecycle_operation_test.go new file mode 100644 index 000000000..b41e61cc7 --- /dev/null +++ b/operator/internal/controller/lifecycle_operation_test.go @@ -0,0 +1,387 @@ +package controller + +import ( + "context" + "fmt" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestLifecycleAllBackupVariantsExcludeRestoreAndWipe(t *testing.T) { + for _, quiesce := range []bool{false, true} { + for _, strategy := range []string{"restic-snapshot", "volume-snapshot"} { + t.Run(fmt.Sprintf("%s/quiesce=%t", strategy, quiesce), func(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce, b.Spec.Strategy = quiesce, strategy + r, _ := guardFixture(t, b) + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim backup: claimed=%t err=%v", claimed, err) + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns", UID: "restore-uid"}} + if err := r.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + rr := &RestoreReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if claimed, err := rr.claimRestoreTarget(t.Context(), rs, &gs); err != nil || claimed { + t.Fatalf("restore overlapped backup: claimed=%t err=%v", claimed, err) + } + gs.Annotations[WipeRequestedAnnotation] = "wipe" + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if changed, err := gr.ensureWipeGuard(t.Context(), &gs); err != nil || changed || gs.Annotations[wipeGuardAnnotation] != "" { + t.Fatalf("wipe overlapped backup: changed=%t err=%v", changed, err) + } + }) + } + } +} + +func TestLifecycleRestoreExcludesBackupCreatedAfterDrainCheck(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = false + r, _ := guardFixture(t, b) + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns", UID: "restore-uid"}} + if err := r.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + rr := &RestoreReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if claimed, err := rr.claimRestoreTarget(t.Context(), rs, &gs); err != nil || !claimed { + t.Fatalf("claim restore: claimed=%t err=%v", claimed, err) + } + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || claimed { + t.Fatalf("unquiesced backup entered restore: claimed=%t err=%v", claimed, err) + } +} + +func TestLifecycleUnquiescedCleanupRetainsOwnerUntilWorkerDrains(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = false + r, q := guardFixture(t, b) + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orphan", Namespace: "ns", Labels: map[string]string{backupUIDLabel: string(b.UID)}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + b.Status.Phase = gameplanev1alpha1.BackupPhaseFailed + if err := r.Status().Update(t.Context(), b); err != nil { + t.Fatal(err) + } + if res, err := r.runUnquiesce(t.Context(), b); err != nil || res.RequeueAfter == 0 { + t.Fatalf("cleanup released live worker: result=%+v err=%v", res, err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if _, err := r.runUnquiesce(t.Context(), b); err != nil { + t.Fatal(err) + } + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + if gs.Annotations[lifecycleOperationAnnotation] != "" || q.unquiesced != 0 { + t.Fatalf("unquiesced cleanup: owner=%q agent calls=%d", gs.Annotations[lifecycleOperationAnnotation], q.unquiesced) + } +} + +func TestLifecycleBackupFencePreservesReplicasAndTemplate(t *testing.T) { + s := wipeScheme(t) + gs := wipeGameServer(false, "") + ss := wipeStoppedSet(gs) + one := int32(1) + ss.Spec.Replicas = &one + ss.Spec.Template.Spec.Containers = []corev1.Container{{Name: "game", Image: "before"}} + b := guardBackup("copy") + b.Spec.ServerRef.Name = gs.Name + gs.Annotations[backupGuardAnnotation] = backupIdentityJSON(b.Name, b.UID) + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: backupLifecycleIdentity(gs, b, false)}) + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, ss, b).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: s} + stale := gs.DeepCopy() + stale.Annotations = nil + stale.Spec.Suspend = true + if err := r.reconcileStatefulSet(t.Context(), stale, &gameplanev1alpha1.GameTemplate{}, nil, &materializedConfig{}, 0); err != nil { + t.Fatal(err) + } + var current appsv1.StatefulSet + if err := c.Get(t.Context(), client.ObjectKeyFromObject(ss), ¤t); err != nil { + t.Fatal(err) + } + if *current.Spec.Replicas != 1 || current.Spec.Template.Spec.Containers[0].Image != "before" { + t.Fatal("stale reconcile stopped or rolled protected source") + } + if replicas, delay, err := r.desiredReplicas(t.Context(), stale, &gameplanev1alpha1.GameTemplate{}, idleAsleep); err != nil || replicas != 1 || delay == 0 { + t.Fatalf("stop/idle bypassed live owner: replicas=%d delay=%s err=%v", replicas, delay, err) + } +} + +func TestLifecycleConcurrentClaimsUseResourceVersion(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = false + r, _ := guardFixture(t, b) + intercepted := false + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if gs, ok := obj.(*gameplanev1alpha1.GameServer); ok && !intercepted && gs.Annotations[lifecycleOperationAnnotation] != "" { + intercepted = true + var current gameplanev1alpha1.GameServer + if err := c.Get(ctx, client.ObjectKeyFromObject(gs), ¤t); err != nil { + return err + } + if current.Annotations == nil { + current.Annotations = map[string]string{} + } + current.Annotations[WipeRequestedAnnotation] = "winner" + current.Annotations[wipeGuardAnnotation] = "winner" + returnAfter := c.Update(ctx, ¤t) + if returnAfter != nil { + return returnAfter + } + } + return c.Update(ctx, obj, opts...) + }, + }) + if claimed, err := r.claimBackupTarget(t.Context(), b); claimed || !apierrors.IsConflict(err) { + t.Fatalf("lost CAS started backup: claimed=%t err=%v", claimed, err) + } + var jobs batchv1.JobList + if err := r.List(t.Context(), &jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("losing claimant started workers: jobs=%d err=%v", len(jobs.Items), err) + } +} + +func TestLifecycleIdentityRejectsReplacementAndMalformedOwner(t *testing.T) { + for _, owner := range []string{"broken", lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: lifecycleIdentity{Kind: "Backup", Name: "other", UID: types.UID("other"), TargetUID: "replaced"}})} { + t.Run(owner, func(t *testing.T) { + b := guardBackup("copy") + r, _ := guardFixture(t, b) + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{lifecycleOperationAnnotation: owner} + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + if claimed, err := r.claimBackupTarget(t.Context(), b); claimed || err == nil { + t.Fatalf("invalid owner admitted backup: claimed=%t err=%v", claimed, err) + } + }) + } +} + +func TestLifecycleOrphanBackupRecoveryFencesRestoreUntilDrain(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = false + r, _ := guardFixture(t, b) + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orphan", Namespace: "ns", Labels: map[string]string{backupUIDLabel: string(b.UID)}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + forceDeleteGuardBackup(t, r, b.Name) + rs := lifecycleRestore("restore", "restore-uid") + if err := r.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "server-uid"}} + rr := &RestoreReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if claimed, err := rr.claimRestoreTarget(t.Context(), rs, gs); err != nil || claimed { + t.Fatalf("restore admitted before orphan drained: %t %v", claimed, err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if replicas, _, err := gr.desiredReplicas(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, idleAwake); err != nil || replicas != 1 { + t.Fatalf("recovery stopped source while copy worker alive: replicas=%d err=%v", replicas, err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if claimed, err := rr.claimRestoreTarget(t.Context(), rs, gs); err != nil || !claimed { + t.Fatalf("restore did not recover after drain: %t %v", claimed, err) + } +} + +func TestLifecycleIdleTransitionConflictsWithNewBackupOwner(t *testing.T) { + b := guardBackup("copy") + r, _ := guardFixture(t, b) + var stale gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &stale); err != nil { + t.Fatal(err) + } + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if err := gr.applyIdleTransition(t.Context(), &stale, idleOutcome{sleep: true}, time.Now()); !apierrors.IsConflict(err) { + t.Fatalf("stale idle transition bypassed ownership CAS: %v", err) + } + var current gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&stale), ¤t); err != nil { + t.Fatal(err) + } + if current.Annotations[IdleAsleepSinceAnnotation] != "" { + t.Fatal("backup acquisition allowed stale sleep") + } +} + +func TestLifecycleBackupFenceConflictsWithStaleWorkloadWrite(t *testing.T) { + b := guardBackup("copy") + r, _ := guardFixture(t, b) + var stale appsv1.StatefulSet + key := client.ObjectKey{Namespace: "ns", Name: "server"} + if err := r.Get(t.Context(), key, &stale); err != nil { + t.Fatal(err) + } + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + gs, err := r.ownedBackupTarget(t.Context(), b) + if err != nil || gs == nil { + t.Fatalf("owned target: %v", err) + } + if ready, err := fenceBackupWorkload(t.Context(), r.Client, r.Client, gs); err != nil || !ready { + t.Fatalf("fence: %t %v", ready, err) + } + zero := int32(0) + stale.Spec.Replicas = &zero + stale.Spec.Template.Spec.Containers = []corev1.Container{{Name: "game", Image: "new-revision"}} + if err := r.Update(t.Context(), &stale); !apierrors.IsConflict(err) { + t.Fatalf("stale workload write defeated fence: %v", err) + } +} + +func TestLifecycleCSIInitialStartupExcludesBackupAndRollout(t *testing.T) { + r, gs, rs := restoreLifecycleFixture(t) + gs.Spec.Suspend = false + gs.Annotations = map[string]string{restoreGuardAnnotation: restoreIdentityJSON(rs.Name, rs.UID)} + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + want := restoreLifecycleIdentity(gs, rs) + want.InitialStartup = true + if ready, err := claimLifecycleOperation(t.Context(), r.Client, r.apiReader(), gs, want, nil); err != nil || !ready { + t.Fatalf("claim initial startup: %t %v", ready, err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + ss.Spec.Template.Spec.Containers = []corev1.Container{{Name: "game", Image: "snapshot-version"}} + if err := r.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + gr := &GameServerReconciler{Client: r.Client, APIReader: r.apiReader(), Scheme: r.Scheme} + replicas, _, err := gr.desiredReplicas(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, idleAwake) + if err != nil || replicas != 1 { + t.Fatalf("CSI initial startup blocked first consumer: replicas=%d err=%v", replicas, err) + } + if err := gr.reconcileStatefulSet(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}, nil, &materializedConfig{}, replicas); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + if *ss.Spec.Replicas != 1 || ss.Spec.Template.Spec.Containers[0].Image != "snapshot-version" { + t.Fatal("CSI restore either blocked startup or adopted a different template") + } + b := guardBackup("new-copy") + if err := r.Create(t.Context(), b); err != nil { + t.Fatal(err) + } + br := &BackupReconciler{Client: r.Client, APIReader: r.apiReader(), Scheme: r.Scheme} + if claimed, err := br.claimBackupTarget(t.Context(), b); err != nil || claimed { + t.Fatalf("backup entered CSI restore startup: %t %v", claimed, err) + } +} + +func TestLifecycleIndependentServersRemainConcurrent(t *testing.T) { + first, second := guardBackup("first"), guardBackup("second") + second.Spec.ServerRef.Name = "other" + r, _ := guardFixture(t, first, second) + other := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: "ns", UID: "other-uid"}} + if err := r.Create(t.Context(), other); err != nil { + t.Fatal(err) + } + for _, b := range []*gameplanev1alpha1.Backup{first, second} { + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("independent target %s blocked: %t %v", b.Spec.ServerRef.Name, claimed, err) + } + } +} + +type lifecycleStopper struct{ calls int } + +func (s *lifecycleStopper) Enabled() bool { return true } +func (s *lifecycleStopper) Stop(context.Context, string, string) error { + s.calls++ + return nil +} + +func TestLifecycleBackupDefersRestartAndStopRPC(t *testing.T) { + for _, quiesce := range []bool{false, true} { + t.Run(fmt.Sprintf("quiesce=%t", quiesce), func(t *testing.T) { + b := guardBackup("copy") + b.Spec.Quiesce = quiesce + r, _ := guardFixture(t, b) + if claimed, err := r.claimBackupTarget(t.Context(), b); err != nil || !claimed { + t.Fatalf("claim: %t %v", claimed, err) + } + var gs gameplanev1alpha1.GameServer + key := client.ObjectKey{Namespace: "ns", Name: "server"} + if err := r.Get(t.Context(), key, &gs); err != nil { + t.Fatal(err) + } + gs.Annotations[RestartRequestedAnnotation] = "restart-after-copy" + gs.Spec.Suspend = true + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), key, &ss); err != nil { + t.Fatal(err) + } + ss.Status.Replicas, ss.Status.ReadyReplicas = 1, 1 + if err := r.Update(t.Context(), &ss); err != nil { + t.Fatal(err) + } + stopper := &lifecycleStopper{} + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme, AgentClient: stopper} + tmpl := &gameplanev1alpha1.GameTemplate{Spec: gameplanev1alpha1.GameTemplateSpec{RCON: &gameplanev1alpha1.RCONSpec{Protocol: "source"}, Capabilities: &gameplanev1alpha1.CapabilitiesSpec{Lifecycle: &gameplanev1alpha1.LifecycleSpec{Stop: []string{"stop"}}}}} + if replicas, poll, err := gr.desiredReplicas(t.Context(), &gs, tmpl, idleAsleep); err != nil || replicas != 1 || poll == 0 { + t.Fatalf("restart changed protected source: replicas=%d poll=%s err=%v", replicas, poll, err) + } + if err := r.Get(t.Context(), key, &gs); err != nil { + t.Fatal(err) + } + if stopper.calls != 0 || gs.Annotations[RestartCompletedAnnotation] != "" || gs.Annotations[stopRequestedAtAnnotation] != "" { + t.Fatal("restart issued or acknowledged a stop during the copy") + } + }) + } +} diff --git a/operator/internal/controller/restore_controller.go b/operator/internal/controller/restore_controller.go index 4cbf94e10..42cc2a7b2 100644 --- a/operator/internal/controller/restore_controller.go +++ b/operator/internal/controller/restore_controller.go @@ -2,6 +2,7 @@ package controller import ( "context" + "errors" "fmt" "time" @@ -41,6 +42,9 @@ type RestoreReconciler struct { // DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds. JobBackoffLimit *int32 JobActiveDeadlineSeconds int64 + // OperationQuiescer completes an orphaned backup's cleanup before this + // restore can acquire the same target. It never quiesces a restore. + OperationQuiescer AgentQuiescer } // apiReader returns the uncached reader for live-consistency checks, @@ -125,7 +129,10 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } return ctrl.Result{}, err } - if src.Spec.Strategy == "volume-snapshot" && rs.Annotations[restoreTargetAnnotation] == "" { + // Binding the target does not change the restore strategy. CSI's own + // preparation path checks that binding before awaiting startup, repairing + // references and enforcing the deadline on every subsequent reconcile. + if src.Spec.Strategy == "volume-snapshot" { return r.reconcileVolumeSnapshotRestore(ctx, &rs, &src) } @@ -156,6 +163,9 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } claimed, err := r.claimRestoreTarget(ctx, &rs, &gs) if err != nil { + if errors.Is(err, errLifecycleGenerationChanged) { + return r.fail(ctx, &rs, err.Error()) + } return ctrl.Result{}, err } if !claimed { diff --git a/operator/internal/controller/restore_lifecycle.go b/operator/internal/controller/restore_lifecycle.go index a405b967a..f54d09756 100644 --- a/operator/internal/controller/restore_lifecycle.go +++ b/operator/internal/controller/restore_lifecycle.go @@ -70,7 +70,15 @@ func (r *RestoreReconciler) bindRestoreTarget(ctx context.Context, rs *gameplane // cannot both claim the same GameServer, even across operator processes. Suspend // is written in that same update; a crash cannot leave a lock without suspension. func (r *RestoreReconciler) claimRestoreTarget(ctx context.Context, rs *gameplanev1alpha1.Restore, gs *gameplanev1alpha1.GameServer) (bool, error) { - owner := restoreIdentityJSON(rs.Name, rs.UID) + // Refresh before both compatibility checks and shared-owner admission. + var live gameplanev1alpha1.GameServer + if err := r.apiReader().Get(ctx, client.ObjectKeyFromObject(gs), &live); err != nil { + return false, err + } + if live.UID != gs.UID { + return false, fmt.Errorf("restore target changed identity") + } + *gs = live workers, err := wipeWorkers(ctx, r.apiReader(), gs) if err != nil { return false, err @@ -95,38 +103,7 @@ func (r *RestoreReconciler) claimRestoreTarget(ctx context.Context, rs *gameplan } return false, nil } - if guard := gs.Annotations[restoreGuardAnnotation]; guard != "" && guard != owner { - identity, err := parseRestoreIdentity(guard) - if err != nil { - return false, err - } // Malformed guards fail closed. - var holder gameplanev1alpha1.Restore - err = r.apiReader().Get(ctx, types.NamespacedName{Namespace: gs.Namespace, Name: identity.Name}, &holder) - if err == nil && holder.UID == identity.UID { - return false, nil - } - if err != nil && !apierrors.IsNotFound(err) { - return false, err - } - // Force-deleted owners can leave their Job and pods behind. Cancel the - // old UID's Job and wait for its workers before reclaiming the guard. - busy, err := r.restoreWorkersLive(ctx, gs.Namespace, identity, true) - if err != nil || busy { - return false, err - } - } - if gs.Annotations[restoreGuardAnnotation] == owner && gs.Spec.Suspend { - return true, nil - } - if gs.Annotations == nil { - gs.Annotations = make(map[string]string) - } - gs.Annotations[restoreGuardAnnotation] = owner - gs.Spec.Suspend = true - if err := r.Update(ctx, gs); err != nil { - return false, err - } - return true, nil + return claimLifecycleOperation(ctx, r.Client, r.apiReader(), gs, restoreLifecycleIdentity(gs, rs), r.OperationQuiescer) } func liveRestorePod(pod *corev1.Pod) bool { @@ -280,6 +257,19 @@ func (r *RestoreReconciler) cleanupRestore(ctx context.Context, rs *gameplanev1a return ctrl.Result{}, err } if err == nil && gs.UID == target.UID && gs.Annotations[restoreGuardAnnotation] == restoreIdentityJSON(rs.Name, rs.UID) { + if value := gs.Annotations[lifecycleOperationAnnotation]; value != "" { + op, err := parseLifecycleOperation(value) + if err != nil { + return ctrl.Result{}, err + } + ready, err := recoverLifecycleOperation(ctx, r.Client, r.apiReader(), &gs, op, r.OperationQuiescer) + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } + } + if err := clearLifecycleOperation(ctx, r.Client, r.apiReader(), &gs, restoreLifecycleIdentity(&gs, rs)); err != nil { + return ctrl.Result{}, err + } var ss appsv1.StatefulSet ssErr := r.apiReader().Get(ctx, client.ObjectKeyFromObject(&gs), &ss) if ssErr != nil && !apierrors.IsNotFound(ssErr) { diff --git a/operator/internal/controller/restore_volumesnapshot.go b/operator/internal/controller/restore_volumesnapshot.go index c2d1819f5..00705401c 100644 --- a/operator/internal/controller/restore_volumesnapshot.go +++ b/operator/internal/controller/restore_volumesnapshot.go @@ -76,6 +76,13 @@ func (r *RestoreReconciler) reconcileVolumeSnapshotRestore( "target server %q already exists; volume-snapshot restores create a new server", rs.Spec.ServerRef.Name)) } + ready, err := r.prepareSnapshotRestoreTarget(ctx, rs, &newGS) + if errors.Is(err, errLifecycleGenerationChanged) { + return r.fail(ctx, rs, err.Error()) + } + if err != nil || !ready { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } return r.awaitRestoredServer(ctx, rs, src, &newGS) case !apierrors.IsNotFound(err): return ctrl.Result{}, err @@ -94,9 +101,14 @@ func (r *RestoreReconciler) reconcileVolumeSnapshotRestore( created := &gameplanev1alpha1.GameServer{ ObjectMeta: metav1.ObjectMeta{ - Name: rs.Spec.ServerRef.Name, - Namespace: rs.Namespace, - Annotations: map[string]string{annoRestoreCreatedBy: rs.Name}, + Name: rs.Spec.ServerRef.Name, + Namespace: rs.Namespace, + Annotations: map[string]string{ + annoRestoreCreatedBy: rs.Name, + // Reserve the target atomically with creation. Once its UID is + // assigned, bind the full shared owner before permitting startup. + restoreGuardAnnotation: restoreIdentityJSON(rs.Name, rs.UID), + }, }, } orig.Spec.DeepCopyInto(&created.Spec) @@ -211,6 +223,26 @@ func (r *RestoreReconciler) ensureRestoredRefsFromSpec( return nil } +func (r *RestoreReconciler) prepareSnapshotRestoreTarget(ctx context.Context, rs *gameplanev1alpha1.Restore, gs *gameplanev1alpha1.GameServer) (bool, error) { + changed, err := r.bindRestoreTarget(ctx, rs, gs) + if err != nil { + return false, err + } + if changed { + return false, nil + } + target, err := parseRestoreIdentity(rs.Annotations[restoreTargetAnnotation]) + if err != nil { + return false, err + } + if target.UID != gs.UID || target.Name != gs.Name { + return false, fmt.Errorf("CSI restore target changed identity") + } + want := restoreLifecycleIdentity(gs, rs) + want.InitialStartup = true + return claimLifecycleOperation(ctx, r.Client, r.apiReader(), gs, want, r.OperationQuiescer) +} + // awaitRestoredServer drives the Restore to a terminal phase based on the // newly-provisioned server: Running (with its references copied) → // Succeeded, Failed → Failed, otherwise keep polling while it starts up diff --git a/operator/internal/controller/restore_volumesnapshot_dispatch_test.go b/operator/internal/controller/restore_volumesnapshot_dispatch_test.go new file mode 100644 index 000000000..5ca38869b --- /dev/null +++ b/operator/internal/controller/restore_volumesnapshot_dispatch_test.go @@ -0,0 +1,81 @@ +package controller + +import ( + "encoding/json" + "testing" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// Exercise the public dispatch after target binding. Calling the CSI helper +// directly misses a regression that diverted later passes into Restic restore. +func TestVolumeSnapshotRestoreReconcileKeepsBoundStrategy(t *testing.T) { + for _, state := range []string{"running", "repair-reference", "deadline"} { + t.Run(state, func(t *testing.T) { + scheme := scrapeScheme(t) + if err := appsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + orig := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "source", Namespace: "ns", UID: "source-uid"}} + target := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "target", Namespace: "ns", UID: "target-uid", Annotations: map[string]string{annoRestoreCreatedBy: "restore", restoreGuardAnnotation: restoreIdentityJSON("restore", "restore-uid")}}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}} + backup := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "backup", Namespace: "ns", UID: "backup-uid"}, Spec: gameplanev1alpha1.BackupSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: orig.Name}, Strategy: "volume-snapshot"}, Status: gameplanev1alpha1.BackupStatus{Phase: gameplanev1alpha1.BackupPhaseSucceeded, SnapshotID: "snapshot", VolumeSnapshotContentName: "content"}} + started := metav1.Now() + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: "ns", UID: "restore-uid", Finalizers: []string{restoreCleanupFinalizer}, Annotations: map[string]string{restoreTargetAnnotation: restoreIdentityJSON(target.Name, target.UID)}}, Spec: gameplanev1alpha1.RestoreSpec{BackupRef: gameplanev1alpha1.LocalObjectRef{Name: backup.Name}, ServerRef: gameplanev1alpha1.LocalObjectRef{Name: target.Name}}, Status: gameplanev1alpha1.RestoreStatus{Phase: gameplanev1alpha1.RestorePhaseRunning, SnapshotID: "snapshot", StartTime: &started}} + objects := []client.Object{orig, target, backup, rs} + want := gameplanev1alpha1.RestorePhaseRunning + switch state { + case "running": + target.Status.Phase = gameplanev1alpha1.GameServerPhaseRunning + want = gameplanev1alpha1.RestorePhaseSucceeded + case "repair-reference": + plan, err := json.Marshal([]refCopy{{Kind: secretRefKind, OrigName: "source-secret", CopyName: "target-secret"}}) + if err != nil { + t.Fatal(err) + } + rs.Annotations["restore.gameplane.local/copy-plan"] = string(plan) + target.Spec.Env = []corev1.EnvVar{{Name: "SECRET", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "target-secret"}, Key: "key"}}}} + objects = append(objects, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "source-secret", Namespace: "ns", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(orig, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Data: map[string][]byte{"key": []byte("preserved-value")}}) + case "deadline": + started = metav1.NewTime(time.Now().Add(-VolumeSnapshotRestoreDeadline - time.Minute)) + want = gameplanev1alpha1.RestorePhaseFailed + } + cl := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...).WithStatusSubresource(rs, target, backup).Build() + r := &RestoreReconciler{Client: cl, APIReader: cl, Scheme: scheme} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(rs)}); err != nil { + t.Fatal(err) + } + var got gameplanev1alpha1.Restore + if err := cl.Get(t.Context(), client.ObjectKeyFromObject(rs), &got); err != nil { + t.Fatal(err) + } + if got.Status.Phase != want { + t.Fatalf("bound CSI restore phase = %s, want %s: %s", got.Status.Phase, want, got.Status.Message) + } + if state == "repair-reference" { + var copied corev1.Secret + if err := cl.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "target-secret"}, &copied); err != nil { + t.Fatalf("bound CSI restore did not repair reference: %v", err) + } + if !metav1.IsControlledBy(&copied, target) || string(copied.Data["key"]) != "preserved-value" { + t.Fatal("repaired reference lost ownership or data") + } + } + var jobs batchv1.JobList + if err := cl.List(t.Context(), &jobs); err != nil { + t.Fatal(err) + } + if len(jobs.Items) != 0 { + t.Fatal("CSI restore dispatched into Restic Job creation") + } + }) + } +} diff --git a/operator/internal/controller/restore_volumesnapshot_requeue_test.go b/operator/internal/controller/restore_volumesnapshot_requeue_test.go index 78b963678..64ebfb51b 100644 --- a/operator/internal/controller/restore_volumesnapshot_requeue_test.go +++ b/operator/internal/controller/restore_volumesnapshot_requeue_test.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -75,7 +76,7 @@ func TestVolumeSnapshotRestore_TransientErrorsRequeueNotFail(t *testing.T) { }, } restore := &gameplanev1alpha1.Restore{ - ObjectMeta: metav1.ObjectMeta{Name: "rs", Namespace: ns}, + ObjectMeta: metav1.ObjectMeta{Name: "rs", Namespace: ns, UID: "uid-restore"}, Spec: gameplanev1alpha1.RestoreSpec{ BackupRef: gameplanev1alpha1.LocalObjectRef{Name: "smp-vs"}, ServerRef: gameplanev1alpha1.LocalObjectRef{Name: restoredName}, @@ -105,6 +106,8 @@ func TestVolumeSnapshotRestore_TransientErrorsRequeueNotFail(t *testing.T) { switch o := obj.(type) { case *gameplanev1alpha1.GameServer: if o.Name == restoredName { + // The fake client does not assign API-server identities. + o.UID = "uid-restored" restoredCreated = true } case *corev1.Secret: @@ -149,13 +152,32 @@ func TestVolumeSnapshotRestore_TransientErrorsRequeueNotFail(t *testing.T) { if err := pass("stale-cache pass"); err != nil { t.Fatalf("stale-cache pass: want requeue without error, got %v", err) } - // Pass 3: cache caught up; the await path re-ensures the copy but the - // write still fails transiently. + // Pass 3: cache caught up; persist the new target's UID and cleanup + // finalizer before claiming its durable lifecycle owner on the next pass. staleRestored = false + if err := pass("bind target pass"); err != nil { + t.Fatalf("bind target pass: unexpected error: %v", err) + } + var bound gameplanev1alpha1.Restore + if err := cl.Get(ctx, client.ObjectKeyFromObject(restore), &bound); err != nil { + t.Fatalf("get bound restore: %v", err) + } + target, err := parseRestoreIdentity(bound.Annotations[restoreTargetAnnotation]) + if err != nil || target.Name != restoredName || target.UID != "uid-restored" { + t.Fatalf("bound target = %+v, error = %v; want restored server identity", target, err) + } + if !controllerutil.ContainsFinalizer(&bound, restoreCleanupFinalizer) { + t.Fatal("bound restore is missing its cleanup finalizer") + } + if failSecretCreates != 1 { + t.Fatalf("binding attempted a copy before admission: remaining failures = %d, want 1", failSecretCreates) + } + // Pass 4: the shared owner is acquired; the await path re-ensures the + // copy but the write still fails transiently. if err := pass("await pass with write error"); err == nil { t.Fatal("await pass: want the transient copy error returned for requeue, got nil") } - // Pass 4: errors cleared; the copy is made. + // Pass 5: errors cleared; the copy is made. if err := pass("await pass"); err != nil { t.Fatalf("await pass: unexpected error: %v", err) } @@ -165,6 +187,10 @@ func TestVolumeSnapshotRestore_TransientErrorsRequeueNotFail(t *testing.T) { if err := cl.Get(ctx, types.NamespacedName{Namespace: ns, Name: restoredName}, &restored); err != nil { t.Fatalf("get restored server: %v", err) } + owner, occupied, err := lifecycleOwner(&restored) + if err != nil || !occupied || !sameLifecycleOwner(owner.lifecycleIdentity, restoreLifecycleIdentity(&restored, restore)) || !owner.InitialStartup { + t.Fatalf("restored lifecycle owner = %+v, occupied = %v, error = %v; want initial-startup restore owner", owner, occupied, err) + } if got := restored.Spec.Env[0].ValueFrom.SecretKeyRef.Name; got != copyName { t.Errorf("restored env secret ref = %q, want copy %q", got, copyName) } diff --git a/operator/specs.md b/operator/specs.md index c7aea8225..15ef3ef73 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -218,11 +218,17 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - **Responsibility:** Drive Backup to completion: coordinate quiesce (if agent available), spawn restic Job, track snapshot ID. - **Key functions:** - Quiesce orchestration: call agent Quiesce before Job, Unquiesce after completion. - - Quiesced copies are serialized per target GameServer UID across schedules and on-demand requests. The Backup pins its target in `backup.gameplane.local/target`; the GameServer's `backup.gameplane.local/owner` stores the owning Backup name/UID. Live API reads and resourceVersion updates protect acquisition/release across reconciler restarts. Quiesce cleanup intent is recorded before calling the agent, including partial failures. Cleanup waits for live copy workers; deletion cancels the copy Job with foreground propagation first. Force-deleted owners are recovered under exclusive ownership, draining their workers and restoring saving before the next copy. Pre-upgrade quiesced Backups without a target annotation finish and release all legacy workers before new protected copies may start. - - Orphan recovery preserves all predecessor names/UIDs in the GameServer's `backup.gameplane.local/recovering-owners` annotation, atomically with ownership transfers, so deleting a recovering Backup cannot hide an earlier worker. Normal deletion, orphan recovery, and legacy cleanup also wait for pending owned CSI captures to report ready or an explicit error; VolumeSnapshots are reclaimed by owner-reference GC after cleanup. + - With quiesce and an agent client enabled, preparation is recorded once in `backup.gameplane.local/preparation`, bound to the target UID and StatefulSet UID/generation. `already-stopped` requires uncached proof of desired and observed zero replicas, caught-up observed generation, and no live game/PVC user pods or pending writer Jobs. Current copy workers are excluded by Job ownership. This path sends no agent calls and preserves suspend/idle/wake intent under the shared workload fence. It applies equally to restic and CSI copies. + - Live preparation also pins the game Pod UID. The operator checks the agent's read-only `/v1/pods/{uid}/identity` endpoint before recording the first cleanup intent, then uses pod-bound quiesce/unquiesce routes; the agent rejects a different UID before RCON. A missing endpoint or identity mismatch fails closed without a legacy-route fallback. Ambiguous quiesce errors retain cleanup intent and the original Pod UID on both the Backup and shared owner. Only confirmed save-on or uncached evidence that the original pod is absent/replaced/terminal permits cleanup; an HTTP timeout, unready pod, or terminating pod is insufficient. Legacy attempts without a UID retire only on absence/terminal proof and never adopt a new UID. `Unquiesced=True, Reason=SourceProcessGone` distinguishes process exit from an actual save-on RPC. + - Source proof is checked before and after quiesce and on each copy reconcile. Observed workload/process replacement fails the Backup before it can report a restorable success. Owner recovery retains the pod UID even if the Backup is force-deleted, drains predecessor workers, and never sends its cleanup to a successor pod. + - Upgrade the operator and agent image together and allow existing GameServer pods to roll out the new agent and `GAMEPLANE_POD_UID` DownwardAPI env. Older agents fail the identity preflight before cleanup debt, so the workload can continue its upgrade after terminal cleanup releases the owner. Retry failed copies after rollout; preflight never bypasses protection. This does not automatically stop a running game. + - Every copy, including `quiesce=false` and CSI snapshots, acquires the shared lifecycle owner before starting. The Backup pins its target in `backup.gameplane.local/target`; `backup.gameplane.local/owner` remains a compatibility marker. The shared owner is described below. Quiesce cleanup intent is recorded on both the Backup and the surviving GameServer before calling the agent, including partial failures. Cleanup waits for live copy workers; deletion cancels the copy Job with foreground propagation first. Every Backup carries the cleanup finalizer. Pre-upgrade quiesced workers are observed through completion even when overlapping legacy copies cannot acquire the new exclusive owner. + - Orphan recovery preserves predecessor operation identities in the shared owner's `recovery` array, atomically with ownership transfer. Existing `backup.gameplane.local/recovering-owners` lineage is imported during migration. Deleting a recovering owner cannot hide an earlier worker. Normal deletion, orphan recovery, and legacy cleanup also wait for pending owned CSI captures to report ready or an explicit error; VolumeSnapshots are reclaimed by owner-reference GC after cleanup. - VolumeSnapshot API availability is checked before quiescing. Snapshot creation errors fail the Backup and run unquiesce cleanup immediately instead of retrying creation indefinitely with saving disabled. API/creation failures persist `backup.gameplane.local/snapshot-aborted` to allow cleanup without waiting for a failed/nonrestorable capture. Failed unquiesce remains visible and retried. A transport failure can leave a CSI snapshot object despite a Failed Backup; this does not make that Backup restorable. - Restic Job: mount data PVC, run restic backup to configured destination (S3, B2, rest-server, local fs). Job carries BackoffLimit and ActiveDeadlineSeconds from the reconciler's JobBackoffLimit / JobActiveDeadlineSeconds fields (operator flags --backup-job-backoff-limit / --backup-job-active-deadline-seconds, Helm operator.backupJobBackoffLimit / operator.backupJobActiveDeadlineSeconds; defaults 2 and 86400, applied when unset; spec 018 OD-027); only a Failed=True Job condition (set by jobPermanentlyFailed check) marks the Backup Failed. Pod SecurityContext takes FSGroup from the target GameTemplate's spec.security.fsGroup (fallback 65532 if template missing). - Snapshot tracking: parse restic JSON output from container logs to extract snapshot ID. + - Capture metadata: after ownership, workload fencing and quiesce admission, record `status.captureManifest` once before creating the first copy worker. It pins the GameServer UID, strategy, observed Pod/StatefulSet mount layout, PVC names/UIDs, included primary claim and excluded extra/mod/retained claims. `storage.extra[].backupRole` optionally declares `data` or `rebuildable`; absent declarations remain unclassified and excluded. Later template edits never rewrite the manifest. Existing workers and old terminal Backups have unknown historical coverage, never reconstructed from the current template. + - Achieved consistency: the `Consistency` condition is separate from copy phase and volume coverage. It stays Unknown until a usable snapshot identity is recorded; failed copies report `CaptureFailed`. Agent-confirmed quiesce, already-stopped/drained sources, unsupported/no-agent raw copy and unquiesced CSI snapshots have distinct reasons. Quiesce depends on the module commands and does not imply that excluded volumes were captured. The API exposes the stored status and the backup drawer shows consistency and included/excluded claims. `StoppedSource` records an observation, without adding a stop/copy/restart policy or multi-volume backup. - Snapshot cleanup on delete (`backup_forget.go`): restic-strategy Backups carry `BackupSnapshotFinalizer` (`gameplane.local/backup-snapshot-finalizer`, added on first reconcile, also for pre-existing non-Failed Backups; not added for volume-snapshot or Failed Backups). `finalizeDelete` runs the unquiesce stage (`BackupFinalizer`) first, then `finalizeSnapshot`: it deletes the Backup's own still-running Job (foreground, awaited), waits while a non-terminal Restore pins the Backup or its snapshot id, then runs Job `-forget` (`restic unlock || true; restic forget --prune --retry-lock 10m`, backoff 3, deadline 1800s, 50m/128Mi requests and 512Mi memory limit, no PVC; the snapshot id is passed through env `FORGET_SNAPSHOT_ID` and must match `^[0-9a-f]{8,64}$`). Success sets `SnapshotForgotten=True` and a Normal `SnapshotForgotten` event; a Failed Job, a missing/incomplete repo Secret, a terminating namespace, or `maxSnapshotForgetWait` (45m, from annotation `backup.gameplane.local/forget-started-at`) sets `SnapshotForgotten=False` with a Warning `SnapshotForgetAbandoned` event and releases the Backup, leaving the snapshot in the repository. A Backup with no recorded snapshot id is released with `SnapshotForgetSkipped` when it may have written one. Backup Jobs run `restic backup /data --json --retry-lock 10m ...`. RBAC markers on `backup_controller.go` grant `backups` update/patch, `backups/status`, `backups/finalizers` and `jobs` create/update/patch/delete. Tests: `backup_forget_test.go`, `backup_forget_envtest_test.go`, `retention_terminating_test.go`; e2e `TestBackup_DeleteForgetsSnapshot` and the extended `TestBackupSchedule_RetentionTrimsPast`. Spec: `specs/done_020-backup-snapshot-forget/`. - Status: report phase (Pending/Running/Succeeded/Failed), error details. - **Integrations:** Agent client (quiesce), Kubernetes Pod logs (restic output), VolumeSnapshot API (CSI backups). @@ -238,6 +244,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li ### RestoreReconciler - **Responsibility:** Drive Restore to completion: suspend GameServer, run restic restore Job, resume. - **Key functions:** + - Both restore strategies participate in shared lifecycle exclusion. A CSI restore reserves its new target with a Restore guard in the GameServer Create, then pins the target UID and adds the full owner. Its `initialStartup` mode permits the first workload to run, including PVCs using WaitForFirstConsumer; backup, wipe, restart, idle transitions and template rollout still wait for restore completion. The original server is unaffected. - Pin snapshot ID at observation (immutable during restore). - Restic restores pin the target GameServer name/UID on the Restore and add a cleanup finalizer before acquiring an annotation guard on that GameServer. The guard records the owning Restore name/UID and is acquired with an optimistic resourceVersion update that also sets spec.suspend=true. Other restores targeting that server wait; restores of independent targets may run concurrently. The game workload remains suspended while the guard exists, even if a power request clears spec.suspend; data-wipe reconciliation also waits for the guard to clear. - A nonempty pending data-wipe request blocks guard acquisition, including after wipe failure. Restore status explains this wait. An uncached read of the fixed-name wipe Job identifies a failed current request only when the current GameServer controls it, its token matches, and it reports Failed=True; DataWipe=False alone is insufficient. Confirmed failure reports that restore is blocked and asks the operator to inspect wipe Job logs and resolve the wipe failure. The restore does not cancel or acknowledge the wipe, since an in-flight wipe reconciliation could still create a writer. The wait message clears when restore acquires the target normally. @@ -248,6 +255,37 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Status: report phase (Pending/Suspending/Running/Succeeded/Failed). `RestorePhaseResuming` is declared on the type but never assigned by this reconciler (F-260). - Volume-snapshot restore reference handling: When a volume-snapshot restore stands up a new GameServer from the original server's spec, any references in spec.env (SecretKeyRef, ConfigMapKeyRef) and spec.networking.tunnel.credentialsSecretRef that are owned by the original server must be copied to new objects owned by the restored server, with rewritten names, because the restored server does not own the originals. Ownership of a source object is an OwnerReference to the original server (Kind, Name, UID) and is re-verified against the source object immediately before each copy, not just once during planning. An object that already exists under a copy name is accepted only if the restored server controls it (a controller OwnerReference, `controller: true`); otherwise the restore fails. The restore fails before the new GameServer is created if the original server does not own every referenced object. The references are rewritten in the new GameServer's spec when it is created, and the copies are made right after creation and re-ensured on every reconcile pass. Only an ownership refusal fails the restore; transient errors (stale cache reads, write conflicts, a referenced object not yet visible in the cache) requeue it. A restore's 10-minute deadline starts when the restore first plans the new server (before it is created) and covers both creating the server with its reference copies and waiting for it to reach Running; it is enforced even while a transient error (such as a referenced object that stays missing) keeps recurring, so the restore cannot run past the limit, and once it expires the restore fails with a deadline message. +### Shared lifecycle exclusion + +`lifecycle_operation.go` is the admission boundary for Backup, Restore and data +wipe. `operations.gameplane.local/owner` on each GameServer stores kind, name, +operation UID, target UID and admitted generation. Wipes use the GameServer UID +plus their existing request token. Acquisition refreshes the target with the +uncached reader and commits the owner and compatibility guard in one checked +resourceVersion Update. Existing owners never expire on time alone. Independent +servers remain concurrent; compatible reader modes are not supported. + +After acquisition, backups fence the owned StatefulSet with a checked metadata +write and wait for its current generation and rollout revision to be observed. +Workload mutation reads ownership inside its callback: backup ownership freezes +replicas and the complete PodTemplate, while restore/wipe may stop the existing +workload but cannot roll it. A missing workload is created stopped so it can be +fenced. Idle sleep/wake and restart acknowledgment use checked writes; a stop RPC +is preceded by a checked durable stop intent which prevents backup admission +until that process drains. Ordinary Services, credentials and status continue +reconciling while operations hold the workload. + +A missing owner is recovered by a successor under exclusive ownership. Recovery +lineage is retained on the GameServer across repeated force deletion. Owned Jobs +are canceled with UID preconditions and foreground propagation; independently +inventoried live or terminating worker pods and pending CSI captures retain the +guard. Saving is restored only after recovery workers drain. The workload remains +frozen during recovery even when the successor is a Restore or wipe. Terminal +state and existing cleanup intent are preserved before releasing the owner; +failed/canceled restores remain suspended. Generation changes to active Backup +or Restore specs fail the operation and enter cleanup rather than authorizing a +different copy under the old owner. See [operator operation recovery](../docs/operator-operations.md). + ### ModuleSourceReconciler - **Responsibility:** Index module sources and surface available modules into status. - **Key functions:** @@ -446,7 +484,7 @@ Forgetting codegen leaves the YAML out of sync with types — CI's `make manifes 6. **Module sources are immutable once pulled.** Digest pinning (spec.digest) defeats tag moves; version pinning (spec.version) tracks a specific semver. Floating (unset version/digest) tracks latest and re-pulls on ModuleSource refresh. -7. **Backup quiesce is best-effort.** If agent unavailable, backup proceeds raw (no pause). If quiesce unsupported (agent returns ErrUnsupported), backup continues degraded (success-with-note). +7. **Backup quiesce fails closed on live transport errors.** With no configured agent client or `quiesce=false`, a copy proceeds raw. With a client enabled, a drained stopped source skips RPC, but an unreachable live agent prevents the copy. A game that explicitly reports quiesce unsupported can continue without claiming it was quiesced. 8. **Remote cluster health checks are non-blocking.** A Cluster with health Unhealthy does not prevent GameServer creation on the local cluster; it surfaces the issue so operators can intervene. diff --git a/test/e2e/backup_stopped_e2e_test.go b/test/e2e/backup_stopped_e2e_test.go new file mode 100644 index 000000000..d469a635d --- /dev/null +++ b/test/e2e/backup_stopped_e2e_test.go @@ -0,0 +1,112 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// Exercise real PVC data and restic for two consecutive protected copies while +// the agent is absent. This test needs no API login and runs in the operator bucket. +func TestBackup_StoppedCopiesRemainRestorable(t *testing.T) { + t.Parallel() + ctx := context.Background() + ns, gs, tmpl := "gameplane-games", "e2e-stopped-backup", "e2e-stopped-backup-template" + ensureResticRepo(t) + applyBusyboxTemplate(t, tmpl) + applyBusyboxGameServer(t, ns, gs, tmpl) + waitPVCBound(t, ns, gs+"-data", 90*time.Second) + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "game" && status.Ready { + return true, "" + } + } + return false, "game container not ready" + }) + if out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "sh", "-c", "echo stopped-backup-marker > /data/marker.txt && sync"); err != nil { + t.Fatalf("write marker: %v\n%s", err, out) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, gs, types.MergePatchType, []byte(`{"spec":{"suspend":true}}`), metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + ss, err := envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, gs, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + if ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation { + return false, "workload not drained" + } + _, err = envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + return apierrors.IsNotFound(err), fmt.Sprintf("pod absence: %v", err) + }) + for i := 1; i <= 2; i++ { + name := fmt.Sprintf("e2e-stopped-copy-%d", i) + backup := &unstructured.Unstructured{Object: map[string]any{ + "apiVersion": "gameplane.local/v1alpha1", "kind": "Backup", + "metadata": map[string]any{"name": name, "namespace": ns}, + "spec": map[string]any{"serverRef": map[string]any{"name": gs}, "repoRef": map[string]any{"name": "e2e-restic-creds", "key": "repo"}, "strategy": "restic-snapshot", "quiesce": true}, + }} + if _, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Create(ctx, backup, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { deleteBackupCR(t, ns, name) }) + var snapshotID string + envInstance.Eventually(t, 5*time.Minute, func() (bool, string) { + // Check throughout the copy, not only after its terminal result. + _, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + t.Fatalf("backup woke game or absence could not be confirmed: %v", err) + } + got, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + phase, _, _ := unstructured.NestedString(got.Object, "status", "phase") + if phase == "Failed" { + t.Fatalf("stopped backup failed: %+v", got.Object["status"]) + } + if phase != "Succeeded" { + return false, "phase=" + phase + } + var preparation struct { + Outcome string `json:"outcome"` + } + if err := json.Unmarshal([]byte(got.GetAnnotations()["backup.gameplane.local/preparation"]), &preparation); err != nil || preparation.Outcome != "already-stopped" { + t.Fatalf("missing stopped-source proof: %+v, %v", preparation, err) + } + if got.GetAnnotations()["backup.gameplane.local/quiesce-attempted"] != "" { + t.Fatal("stopped source contacted absent agent") + } + snapshotID, _, _ = unstructured.NestedString(got.Object, "status", "snapshotID") + return snapshotID != "", "snapshot ID pending" + }) + if !regexp.MustCompile(`^[0-9a-f]{8,64}$`).MatchString(snapshotID) { + t.Fatalf("unexpected snapshot ID %q", snapshotID) + } + runResticJob(t, "restic restore "+snapshotID+" --target /tmp/stopped-restore --retry-lock 5m && test \"$(cat /tmp/stopped-restore/data/marker.txt)\" = stopped-backup-marker") + } + got, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, gs, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + suspended, _, _ := unstructured.NestedBool(got.Object, "spec", "suspend") + if !suspended { + t.Fatal("backup changed manual suspend intent") + } +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d60a26f14..b743bd484 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -83,6 +83,7 @@ TestModuleSourceUpload TestModule_ScaffoldAndPackage TestModule_SteamcmdScaffoldKeepsNonRootSecurity TestBackup_OperatorMaterializesJob +TestBackup_StoppedCopiesRemainRestorable TestBackup_FailsOnMissingPVC TestBackup_FailsOnBadCredentials TestBackup_DeleteForgetsSnapshot diff --git a/web/src/components/backups/BackupDetailDrawer.test.tsx b/web/src/components/backups/BackupDetailDrawer.test.tsx index 8e5056db3..835470dd7 100644 --- a/web/src/components/backups/BackupDetailDrawer.test.tsx +++ b/web/src/components/backups/BackupDetailDrawer.test.tsx @@ -8,6 +8,34 @@ import { makeBackup } from "@/test/factories"; import { BackupDetailDrawer } from "./BackupDetailDrawer"; describe("BackupDetailDrawer", () => { + it("shows captured and excluded claims independently of copy success", async () => { + server.use(http.get("/backups/coverage", () => HttpResponse.json(makeBackup({ + metadata: { name: "coverage" }, + status: { + phase: "Succeeded", snapshotID: "copy", + conditions: [{ type: "Consistency", status: "False", reason: "UnsupportedRawCopy", message: "The game does not support quiesce." }], + captureManifest: { state: "Recorded", recordedAt: "2026-10-21T12:00:00Z", volumes: [ + { claimName: "game-data", claimUID: "world-uid", mountPaths: ["/world"], included: true, classification: "data", reason: "PrimaryDataClaim" }, + { claimName: "game-extra-install", mountPaths: ["/serverfiles"], included: false, classification: "rebuildable", reason: "NotCovered" }, + { claimName: "game-mods", mountPaths: ["/world/mods"], included: false, classification: "unclassified", reason: "NotCovered" }, + ] }, + }, + })))); + renderWithQuery( {}} onRestore={() => {}} />); + expect(await screen.findByText("The game does not support quiesce.")).toBeInTheDocument(); + expect(screen.getByText("game-data")).toBeInTheDocument(); + expect(screen.getByText("/world/mods")).toBeInTheDocument(); + expect(screen.getAllByText("Not covered")).toHaveLength(2); + expect(screen.getByText("Rebuildable")).toBeInTheDocument(); + }); + + it("shows legacy coverage as unknown without inventing a current layout", async () => { + server.use(http.get("/backups/legacy", () => HttpResponse.json(makeBackup({ metadata: { name: "legacy" }, status: { phase: "Succeeded" } })))); + renderWithQuery( {}} onRestore={() => {}} />); + expect(await screen.findByText(/Capture-time volume metadata is unavailable/)).toBeInTheDocument(); + expect(screen.queryByText("Included")).not.toBeInTheDocument(); + }); + it("does not render content when name is null", () => { renderWithQuery( {}} onRestore={() => {}} />); expect(screen.queryByText(/Backup details/i)).not.toBeInTheDocument(); diff --git a/web/src/components/backups/BackupDetailDrawer.tsx b/web/src/components/backups/BackupDetailDrawer.tsx index 7e9422231..eff9875e1 100644 --- a/web/src/components/backups/BackupDetailDrawer.tsx +++ b/web/src/components/backups/BackupDetailDrawer.tsx @@ -9,6 +9,7 @@ import type { Backup } from "@/types"; import { PhaseChip } from "@/components/ui/PhaseChip"; import { ErrorBanner } from "./ErrorBanner"; import { useOptionalIntl } from "@/lib/i18n/format"; +import { useConditionMessage } from "@/lib/i18n/conditionMessage"; interface Props { target?: ResourceTarget; @@ -44,6 +45,8 @@ export function BackupDetailDrawer({ name, ns, onClose, onRestore, target: expli const restorable = backup?.status?.phase === "Succeeded" && Boolean(backup.status.snapshotID); + const consistency = useConditionMessage(backup?.status?.conditions?.find((c) => c.type === "Consistency")); + const manifest = backup?.status?.captureManifest; return ( {backup.spec.serverRef.name} + + {consistency ?? intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.consistency_unknown", defaultMessage: "Consistency has not been reported.", description: "Backup details: no achieved-consistency condition is available" })} + + + {manifest?.state === "Recorded" ? ( +
+ {manifest.recordedAt &&
{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.coverage_recorded", defaultMessage: "Recorded: {time}", description: "Backup details: timestamp of the immutable capture manifest" }, { time: manifest.recordedAt })}
} +

{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.coverage_scope", defaultMessage: "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", description: "Backup details: explains the current single-volume backup scope and rebuildable declarations" })}

+
    + {manifest.volumes?.map((volume) => ( +
  • +
    {volume.claimName}
    +
    {volume.included + ? intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.included", defaultMessage: "Included", description: "Backup volume: this claim is selected for copying; success is reported separately" }) + : intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.not_covered", defaultMessage: "Not covered", description: "Backup volume: contents of this claim are not included" })}
    +
    {intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.classification", defaultMessage: "{role, select, data {Data} rebuildable {Rebuildable} other {Unclassified}}", description: "Backup volume: optional declaration from the module or server at capture time" }, { role: volume.classification })}
    + {volume.mountPaths?.map((path) =>
    {path}
    )} + {volume.reason === "RetainedUnmountedClaim" &&
    {intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.retained_unmounted", defaultMessage: "Retained claim; not mounted in the captured workload.", description: "Backup volume: an older server-owned claim was retained but was not part of the observed pod layout" })}
    } +
    • + ))} +
+
+ ) : intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.legacy_coverage", defaultMessage: "Capture-time volume metadata is unavailable. Inspect the original backup and module configuration before recovery.", description: "Backup details: older backups have no reliable volume manifest, so the current template is not used to invent coverage" })} + {backup.status?.snapshotID ?? intl.formatMessage({ id: "web.common.empty_value", defaultMessage: "—", description: "Shown in place of a value that is not set" })} diff --git a/web/src/routes/CreateServer.tsx b/web/src/routes/CreateServer.tsx index 25c52bfbf..074dea08e 100644 --- a/web/src/routes/CreateServer.tsx +++ b/web/src/routes/CreateServer.tsx @@ -598,6 +598,7 @@ function PickTemplate({ templates: GameTemplate[]; templateText: (moduleName: string) => ModuleText; }) { + const intl = useOptionalIntl(); const [q, setQ] = useState(""); const [cat, setCat] = useState("all"); diff --git a/web/src/types.ts b/web/src/types.ts index 880c2b72c..fa5aa686a 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -644,6 +644,23 @@ export interface ServerEvent { count: number; } +export interface BackupCaptureManifest { + state: "Recorded" | "LegacyUnknown"; + recordedAt?: string; + serverUID?: string; + strategy?: string; + layoutSource?: string; + quiesceOutcome?: string; + volumes?: Array<{ + claimName: string; + claimUID?: string; + mountPaths?: string[]; + included: boolean; + classification: "data" | "rebuildable" | "unclassified"; + reason: string; + }>; +} + export interface Backup { metadata: ObjectMeta; // strategy: restic-snapshot writes to a restic repo; volume-snapshot takes a @@ -655,6 +672,8 @@ export interface Backup { completionTime?: string; snapshotID?: string; size?: string; + captureManifest?: BackupCaptureManifest; + conditions?: Array<{ type: string; status: string; reason?: string; message?: string }>; }; }