fix: back up stopped servers and bind quiesce to pod identity

.github/workflows/ci.yaml

@@ -372,7 +372,7 @@ jobs:
     needs: [changes]
     if: needs.changes.outputs.images == 'true'
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
     permissions:
       contents: read
     steps:

agent/cmd/main.go

@@ -209,6 +209,13 @@ func main() {
 		protected.Use(authCheck.Middleware, target.Guard(serverUID))
 		mountProtected(protected)
 	})
+	// Backups bind save-off/save-on to the process they prepared. The service
+	// may already route to a replacement pod, which must reject the old UID.
+	r.Route("/v1/pods/{uid}", func(protected chi.Router) {
+		protected.Use(authCheck.Middleware, target.Guard(envOr("GAMEPLANE_POD_UID", "")), middleware.Timeout(30*time.Second))
+		protected.Get("/identity", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })
+		quiesce.Mount(protected, rconClient, gameName, quiesceSpec)
+	})
 
 	srv := &http.Server{
 		Addr:              addr,

agent/internal/quiesce/quiesce_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/go-chi/chi/v5"
 
 	"github.com/GameplanePanel/gameplane/agent/internal/caps"
+	"github.com/GameplanePanel/gameplane/agent/internal/target"
 )
 
 type fakeRcon struct {
@@ -86,6 +87,33 @@ func newTestRouter(rc Rcon, spec *caps.Quiesce) *chi.Mux {
 	return r
 }
 
+func TestPodBoundRoutesRejectReplacementBeforeRCON(t *testing.T) {
+	for _, actual := range []string{"original", "replacement", ""} {
+		t.Run("pod="+actual, func(t *testing.T) {
+			rc := &fakeRcon{}
+			r := chi.NewRouter()
+			r.Route("/v1/pods/{uid}", func(bound chi.Router) {
+				bound.Use(target.Guard(actual))
+				Mount(bound, rc, "test", minecraftQuiesceSpec())
+			})
+			for _, path := range []string{"quiesce", "unquiesce"} {
+				response := httptest.NewRecorder()
+				r.ServeHTTP(response, httptest.NewRequestWithContext(t.Context(), http.MethodPost, "/v1/pods/original/"+path, nil))
+				if actual == "original" {
+					if response.Code != http.StatusOK {
+						t.Fatalf("matching process rejected: %d", response.Code)
+					}
+				} else if response.Code != http.StatusNotFound {
+					t.Fatalf("wrong process admitted: %d", response.Code)
+				}
+			}
+			if (len(rc.calls()) > 0) != (actual == "original") {
+				t.Fatalf("commands reached wrong process: %v", rc.calls())
+			}
+		})
+	}
+}
+
 func decodeResponse(t *testing.T, body []byte) response {
 	t.Helper()
 	var got response

agent/internal/rcon/nuclearoption_test.go

@@ -1275,18 +1275,47 @@ type pipeFrameCase struct {
 	forbidErr string // substring the error must NOT contain (e.g. a detail suffix)
 }
 
+type responseDeadlineConn struct {
+	net.Conn
+	deadlineSet chan<- struct{}
+}
+
+func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error {
+	if err := c.Conn.SetDeadline(deadline); err != nil {
+		return err
+	}
+	select {
+	case c.deadlineSet <- struct{}{}:
+	default:
+	}
+	return nil
+}
+
 // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe
 // and checks the body or the error.
 func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) {
 	t.Helper()
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			deadlineSet := make(chan struct{}, 1)
 			client := newPipeClient(t, func(remote net.Conn) {
 				if _, _, err := readNuclearOptionRequest(remote); err != nil {
 					return
 				}
+				if len(tt.frame) == 0 {
+					// Exec sets its response deadline after sending the request.
+					// Closing first can fail SetDeadline instead of the status read.
+					select {
+					case <-deadlineSet:
+					case <-t.Context().Done():
+						return
+					}
+				}
 				writeFrameSegments(remote, tt.frame, tt.oneByte)
 			})
+			if len(tt.frame) == 0 {
+				client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet}
+			}
 
 			body, err := client.Exec("test-command")
 			if tt.wantErr != "" {

agent/internal/target/target.go

@@ -1,4 +1,4 @@
-// Package target binds versioned agent routes to an immutable GameServer UID.
+// Package target binds versioned agent routes to an immutable resource UID.
 package target
 
 import (
@@ -7,12 +7,13 @@ import (
 	"github.com/go-chi/chi/v5"
 )
 
-// Guard rejects a route for any server other than the one that started this
-// agent. Legacy routes stay available for local callers during rolling upgrades.
-func Guard(serverUID string) func(http.Handler) http.Handler {
+// Guard rejects a route for any resource other than the one that started this
+// agent (GameServer or Pod, according to the route). Legacy routes stay available
+// for local callers during rolling upgrades.
+func Guard(resourceUID string) func(http.Handler) http.Handler {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-			if serverUID == "" || chi.URLParam(req, "uid") != serverUID {
+			if resourceUID == "" || chi.URLParam(req, "uid") != resourceUID {
 				http.NotFound(w, req)
 				return
 			}

agent/specs.md

@@ -177,6 +177,8 @@ Resource usage env vars (set by the operator):
 | `/players/banned` | GET | Currently banned players; response: `[ { name, reason?, source? } ]`; 502 if RCON unavailable |
 | `/quiesce` | POST | Pause auto-saves before snapshot; response: `{ quiesced, reason? }` (boolean, reason optional); unsupported games return `quiesced: false` with a reason |
 | `/unquiesce` | POST | Resume auto-saves after snapshot; response: `{ quiesced, reason? }` |
+| `/v1/pods/{uid}/identity` | GET | Operator preflight: 204 only when `{uid}` matches `GAMEPLANE_POD_UID`; never mutates game state |
+| `/v1/pods/{uid}/quiesce`, `/v1/pods/{uid}/unquiesce` | POST | Pod-bound backup RPCs with the same response contract as the legacy routes; reject missing/mismatched Pod UID with 404 before RCON |
 | `/lifecycle/stop` | POST | Run stop sequence before scale-to-zero; RCON connection drop is the expected outcome |
 | `/actions/run` | POST | Execute module-declared operator action; request: `{ id, params }`; response: `{ ok, raw? }` |
 | `/status` | GET | Live game metrics (module-declared); response: bare JSON array `[ { id, displayName?, value, unit? } ]` |
@@ -209,6 +211,7 @@ All endpoints on the `--addr` control mux, except `/healthz`, return `401 Unauth
 - **Extracted mod files stay world-readable**: `mods.moduleFileMode` is `0o644`, not a tighter `0o600`, because the mods volume is shared with the game container, which runs as whatever uid its image needs — a different uid than the agent's own. At `0o600` the game process couldn't read its own mods. This is the rationale behind the `.golangci.yml` gosec G302 exclusion scoped to `agent/internal/mods/mods.go`.
 - **Resource usage is in-pod**: The `usage` package reads from `/proc` or cgroups; no external metrics pipeline required. Cgroup mode is a fallback for older clusters; proc mode (default in production) requires the operator to set `ShareProcessNamespace: true`.
 - **Module capabilities drive behavior**: Every game-specific handler (players, quiesce, lifecycle, status, actions) reads its config from `--capabilities` (JSON unmarshaled into `caps.Spec`). New games require no agent code change.
+- **Backup process identity**: The operator injects `GAMEPLANE_POD_UID` from `metadata.uid` through the DownwardAPI. Pod-bound backup routes require the existing mTLS authentication and check this UID at the final network hop, including when the Service resolves to a replacement pod. Missing UID fails closed. Deploy the updated agent image and allow pod rollout when upgrading the operator; older agents return 404 and the operator does not fall back to unbound routes. Legacy endpoints remain available to existing callers during rollout.
 - **RCON connection errors are graceful**: A lost RCON connection does not crash the agent. `console`, `players`, `quiesce`, and `lifecycle` handlers catch connection errors and return appropriate HTTP status (e.g., `502 Bad Gateway`).
 - **Source RCON sends complete frames**: Each AUTH or command frame is submitted in one socket write, with its existing byte-size limit, request ID, type, body, and two NUL terminators preserved. This avoids separate header-only writes rejected by vanilla Minecraft's RCON reader. A short write is an error and drops the connection without replaying the command; response grace and healthy connection reuse remain unchanged.
 - **Log streams are tail-only**: The `logs` package does not support random-access reads. It streams from the current end (live mode) or from file start (backlog mode); clients must handle partial output and reconnection.

api/internal/handlers/resources_annotations_test.go

@@ -39,6 +39,7 @@ func TestGenericServerWritesCannotInjectLifecycleAnnotations(t *testing.T) {
 					wipeRequestedAnnotation:                        "injected",
 					"gameplane.local/wipe-data-completed":          "forged",
 					"gameplane.local/restore-guard":                "forged",
+					"operations.gameplane.local/owner":             "forged",
 					"gameserver.gameplane.local/stop-requested-at": "forged",
 					"restore.gameplane.local/created-by":           "forged",
 					ownerIDAnnotation:                              "999",
@@ -51,7 +52,7 @@ func TestGenericServerWritesCannotInjectLifecycleAnnotations(t *testing.T) {
 					t.Fatalf("write failed: %d %s", response.Code, response.Body)
 				}
 				got := serverAnnotations(t, k, "alpha")
-				for _, key := range []string{wipeRequestedAnnotation, "gameplane.local/wipe-data-completed", "gameplane.local/restore-guard", "gameserver.gameplane.local/stop-requested-at", "restore.gameplane.local/created-by"} {
+				for _, key := range []string{wipeRequestedAnnotation, "gameplane.local/wipe-data-completed", "gameplane.local/restore-guard", "operations.gameplane.local/owner", "gameserver.gameplane.local/stop-requested-at", "restore.gameplane.local/created-by"} {
 					if _, found := got[key]; found {
 						t.Errorf("generic write injected %s", key)
 					}
@@ -98,6 +99,7 @@ func TestGenericServerUpdatePreservesControllerState(t *testing.T) {
 				wipeRequestedAnnotation:                        "pending",
 				"gameplane.local/wipe-data-completed":          "previous",
 				"gameplane.local/backup-guard":                 "active-backup",
+				"operations.gameplane.local/owner":             `{"kind":"Backup","name":"copy","uid":"copy-uid","targetUID":"server-uid","generation":1}`,
 				"gameserver.gameplane.local/stop-requested-at": "original-time",
 				"restore.gameplane.local/created-by":           "original-restore",
 			}

docs/operator-operations.md

@@ -0,0 +1,49 @@
+# Backup, restore and wipe ownership
+
+Gameplane serializes these operations per GameServer. All backups participate,
+including backups with `quiesce: false` and CSI VolumeSnapshots. A held operation
+also defers idle sleep/wake, restart, and pod-template rollout. Requests remain
+pending and reconciliation resumes after the operation's cleanup completes.
+
+The GameServer annotation `operations.gameplane.local/owner` identifies the
+operation kind, name, UID, admitted generation and target UID. A wipe is identified
+by the GameServer UID and its request token. The existing backup, restore and wipe
+owner annotations remain compatibility markers. A `recovery` array identifies
+older owners whose workers or quiesce obligations must be cleaned up first.
+
+Deleting an operation requests cleanup. Restic Jobs are canceled with foreground
+deletion; live and terminating worker pods must drain before the owner is released.
+Quiesced backups also owe a successful save-on call. The existing bounded Backup
+deletion retry may release its finalizer, but an unconfirmed save-on leaves the
+durable owner behind for recovery by the next operation. Time alone never releases
+ownership. A pending CSI capture retains the current capture-drain behavior; a
+separate CSI timeout change is tracked in issue #719.
+
+If an operation waits unexpectedly, inspect the referenced Backup/Restore, its
+conditions, Jobs and worker pods. Resolve terminating workers, repository failures
+or agent connectivity before retrying. Removing owner annotations or force-removing
+finalizers cannot establish that a worker has stopped. A failed or canceled restore
+leaves its target suspended for inspection.
+
+Upgrades recognize existing owner annotations and import backup recovery lineage.
+Already-running legacy quiesced copies continue being observed until their workers
+finish and cleanup succeeds. Legacy raw workers also block new admission. Conflicting
+legacy guards fail closed until their existing operations finish cleanup. Keep a
+single active operator version during upgrade; older binaries do not honor the
+shared owner. No new CRD fields or Kubernetes permissions are required.
+
+CSI restores create a new target and reserve it for the Restore. Initial startup
+is allowed under that reservation so storage using WaitForFirstConsumer can bind.
+Other operations wait until the restore completes. Existing data-PVC restores
+continue to require an observed stopped workload and drained writers.
+
+If initial startup fails or the Restore is deleted, cleanup first retires its
+startup permission and suspends the target. The operation remains held until the
+StatefulSet has observed zero replicas and every live or terminating game/PVC
+writer has drained. Recovery of a force-deleted Restore applies the same check
+before a successor can proceed, preserving the existing pod template while
+requesting zero replicas. A successful Restore leaves its initial workload running.
+
+Stopped-source preparation (#716), Restic preparation and quiesce deadlines (#717),
+CSI capture deadlines (#719), and staged template adoption (#731) are separate
+changes building on this ownership boundary.

i18n/locales/en/app.json

@@ -599,6 +599,10 @@
     "defaultMessage": "The server could not be resumed after the backup. Its world may still have auto-save turned off.",
     "description": "Unquiesced condition when the unfreeze failed."
   },
+  "condition.unquiesced.sourceprocessgone": {
+    "defaultMessage": "The original game process has exited. No resume command was needed.",
+    "description": "Unquiesced condition when the original source pod no longer runs."
+  },
   "condition.unquiesced.unquiesced": {
     "defaultMessage": "The server was resumed after the backup.",
     "description": "Unquiesced condition when the world was unfrozen."

operator/api/v1alpha1/backup_types.go

@@ -17,11 +17,10 @@ const (
 	BackupPhaseFailed    BackupPhase = "Failed"
 )
 
-// BackupFinalizer blocks a quiesced Backup's deletion until the operator has
-// sent the matching unquiesce. Without it, deleting a Backup while
-// quiesce-attempted=true drops the unquiesce entirely and the game is left
-// with auto-save off indefinitely (F-048). Added only for Backups with
-// spec.quiesce=true; see backup_forget.go's finalizeDelete.
+// BackupFinalizer blocks deletion until copy workers have drained and any
+// matching unquiesce has completed. Every Backup carries it, including raw
+// copies and CSI snapshots, so deletion cannot release shared lifecycle
+// ownership while a copy is still using the target. See finalizeDelete.
 const BackupFinalizer = "gameplane.local/backup-finalizer"
 
 // BackupSnapshotFinalizer blocks the deletion of a restic-strategy Backup until

operator/cmd/main.go

@@ -390,6 +390,7 @@ func main() {
 		AgentCASecretName:      agentCASecretName,
 		AgentCASecretNamespace: agentCASecretNamespace,
 		AgentClient:            agentClient,
+		OperationQuiescer:      agentClient,
 		PodAttacher: &controller.StopAttachClient{
 			Config:    mgr.GetConfig(),
 			Clientset: kubernetes.NewForConfigOrDie(mgr.GetConfig()),
@@ -437,6 +438,7 @@ func main() {
 		Client:                   mgr.GetClient(),
 		APIReader:                mgr.GetAPIReader(),
 		Scheme:                   mgr.GetScheme(),
+		OperationQuiescer:        agentClient,
 		ResticImage:              resticImage,
 		JobBackoffLimit:          &backupJobBackoff32,
 		JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds,

operator/hack/condition-ids/main.go

@@ -140,6 +140,7 @@ var rows = []row{
 	{types: []string{"Completed"}, reason: "Pending", english: "The backup is waiting to start.", description: "Completed condition on a Backup that has not started (dynamic: reason is the phase).", dynamic: true},
 	{types: []string{"Completed"}, reason: "Running", english: "The backup is running.", description: "Completed condition on a Backup in progress (dynamic: reason is the phase).", dynamic: true},
 	{types: []string{"Unquiesced"}, reason: "Unquiesced", english: "The server was resumed after the backup.", description: "Unquiesced condition when the world was unfrozen."},
+	{types: []string{"Unquiesced"}, reason: "SourceProcessGone", english: "The original game process has exited. No resume command was needed.", description: "Unquiesced condition when the original source pod no longer runs."},
 	{types: []string{"Unquiesced"}, reason: "Failed", english: "The server could not be resumed after the backup. Its world may still have auto-save turned off.", description: "Unquiesced condition when the unfreeze failed."},
 	{types: []string{"SnapshotForgotten"}, reason: "Forgotten", english: "The snapshot was removed from the backup repository.", description: "SnapshotForgotten condition after a deleted Backup's snapshot was forgotten."},
 	{types: []string{"SnapshotForgotten"}, reason: "ForgetAbandoned", english: "The snapshot was not removed from the backup repository when the backup was deleted.", description: "SnapshotForgotten condition when the forget step was abandoned."},

operator/internal/agent/client.go

@@ -14,6 +14,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"time"
 )
@@ -119,6 +120,47 @@ func (c *Client) Unquiesce(ctx context.Context, namespace, server string) error
 	return c.callQuiesce(ctx, namespace, server, "/unquiesce", false)
 }
 
+// QuiesceEnabled reports whether this client can perform quiesce operations.
+func (c *Client) QuiesceEnabled() bool { return c != nil && !c.Disabled }
+
+// CheckPod verifies that the responding agent belongs to the selected Pod.
+// Older agents return 404; callers must not fall back to unbound mutations.
+func (c *Client) CheckPod(ctx context.Context, namespace, server, podUID string) error {
+	if c.Disabled || podUID == "" {
+		return fmt.Errorf("agent: pod identity check requires an enabled client and pod UID")
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, agentURL(namespace, server, "/v1/pods/"+url.PathEscape(podUID)+"/identity"), nil)
+	if err != nil {
+		return err
+	}
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return fmt.Errorf("agent: pod identity check: %w", err)
+	}
+	defer func() { _ = resp.Body.Close() }()
+	if resp.StatusCode != http.StatusNoContent {
+		return fmt.Errorf("agent: pod identity check: status %d", resp.StatusCode)
+	}
+	return nil
+}
+
+// QuiescePod disables saving only on the selected Pod's agent.
+func (c *Client) QuiescePod(ctx context.Context, namespace, server, podUID string) error {
+	return c.callPodQuiesce(ctx, namespace, server, podUID, "/quiesce", true)
+}
+
+// UnquiescePod restores saving only on the selected Pod's agent.
+func (c *Client) UnquiescePod(ctx context.Context, namespace, server, podUID string) error {
+	return c.callPodQuiesce(ctx, namespace, server, podUID, "/unquiesce", false)
+}
+
+func (c *Client) callPodQuiesce(ctx context.Context, namespace, server, podUID, path string, expectQuiesced bool) error {
+	if c.Disabled || podUID == "" {
+		return fmt.Errorf("agent: pod-bound quiesce requires an enabled client and pod UID")
+	}
+	return c.callQuiesce(ctx, namespace, server, "/v1/pods/"+url.PathEscape(podUID)+path, expectQuiesced)
+}
+
 // expectQuiesced is what the response should report on a successful
 // completion: true after /quiesce, false after /unquiesce. A mismatch
 // is a protocol bug worth surfacing.