test: exercise panel database and encryption-key recovery

.github/workflows/ci.yaml

@@ -439,7 +439,7 @@ jobs:
     needs: [changes]
     if: (needs.changes.outputs.images == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'build-images')
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    timeout-minutes: 30
     permissions:
       contents: read
     steps:
@@ -1792,8 +1792,9 @@ jobs:
         # t.Parallel(); `parallel` bounds in-flight tests per bucket
         # (operator does zero logins and is wait-dominated, so it runs
         # wide; the api buckets stay within the login limiter's budget).
-        # The arm64 job below uses the same steps and per-bucket settings.
-        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components]
+        # The arm64 job below uses the same steps/settings except for the
+        # SQLite recovery drill, which runs once on amd64.
+        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, recovery]
         arch: [amd64]
         include:
           - bucket: operator
@@ -1829,6 +1830,10 @@ jobs:
             parallel: 1
             test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order
             job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite.
+          - bucket: recovery
+            parallel: 1
+            test_timeout: 20m # isolated panel backup/loss/restore; game workload stays running
+            job_timeout: 60
     env:
       CLUSTER: gameplane-e2e
       CTX: kind-gameplane-e2e
@@ -1859,12 +1864,22 @@ jobs:
         run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e
       - name: run go e2e (${{ matrix.bucket }})
         if: env.RUN_REGEX != ''
+        env:
+          GAMEPLANE_RECOVERY_REPORT_DIR: ${{ github.workspace }}/recovery-reports
         run: |
           cd test/e2e && \
             GAMEPLANE_E2E_REUSE_CLUSTER=1 \
             GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
             go test -tags=e2e -timeout ${{ matrix.test_timeout }} -v \
               -parallel ${{ matrix.parallel }} -run "$RUN_REGEX" ./...
+      - name: upload measured panel recovery report
+        if: always() && matrix.bucket == 'recovery'
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: panel-recovery-${{ matrix.arch }}-${{ github.run_attempt }}
+          path: recovery-reports/*.json
+          if-no-files-found: warn
+          retention-days: 7
       - name: run go e2e (ratelimit tail)
         # LAST on purpose: this test deliberately drains the shared per-IP
         # and per-username login buckets; anything running after it queues
@@ -2660,7 +2675,7 @@ jobs:
               'go-e2e-unit': (n) => n === 'go (test/e2e)',
               'api-postgres': (n) => n === 'api (postgres)',
               'e2e-buckets': (n) => n === 'e2e bucket coverage',
-              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '),
+              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|recovery) \//.test(n) && n.includes('/ amd64 '),
               'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '),
               'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '),
               'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '),
@@ -2813,7 +2828,7 @@ jobs:
             const bucketSet = new Set();
             for (const j of apiJobs) {
               const n = j.name || '';
-              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//);
+              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|recovery) \//);
               if (m) bucketSet.add(m[1]);
               if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster');
               if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade');

agent/internal/rcon/nuclearoption.go

@@ -112,14 +112,21 @@ func (c *NuclearOption) Close() error {
 // If the response has no body (body length = 0), an empty string is returned.
 // Non-2000 status codes produce an error that includes the status code and,
 // if present, the error detail from the response body.
+// One deadline bounds the complete exchange, including dialing and request writes.
 func (c *NuclearOption) Exec(cmd string) (string, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	d := c.execDeadline
+	if d <= 0 {
+		d = defaultNuclearOptionExecDeadline
+	}
+	exchangeDeadline := time.Now().Add(d)
+
 	// Ensure connection is open.
 	if c.conn == nil {
 		d := net.Dialer{Timeout: c.dialTimeout}
-		ctx, cancel := context.WithTimeout(context.Background(), c.dialTimeout)
+		ctx, cancel := context.WithDeadline(context.Background(), exchangeDeadline)
 		defer cancel()
 		conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(c.host, fmt.Sprint(c.port)))
 		if err != nil {
@@ -144,6 +151,14 @@ func (c *NuclearOption) Exec(cmd string) (string, error) {
 		return "", fmt.Errorf("nuclearoption rcon: marshal request: %w", err)
 	}
 
+	// Bound writes as well as reads. Arm this before sending anything: a
+	// server that stops reading must not retain the shared client forever.
+	if err := c.conn.SetDeadline(exchangeDeadline); err != nil {
+		_ = c.conn.Close()
+		c.conn = nil
+		return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err)
+	}
+
 	// Write request: 4-byte LE length + body.
 	reqLen := len(reqBody)
 	if reqLen < 0 || reqLen > math.MaxUint32 {
@@ -162,17 +177,6 @@ func (c *NuclearOption) Exec(cmd string) (string, error) {
 		return "", fmt.Errorf("nuclearoption rcon: write body: %w", err)
 	}
 
-	// Set deadline for the entire response read.
-	d := c.execDeadline
-	if d <= 0 {
-		d = defaultNuclearOptionExecDeadline
-	}
-	if err := c.conn.SetDeadline(time.Now().Add(d)); err != nil {
-		_ = c.conn.Close()
-		c.conn = nil
-		return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err)
-	}
-
 	// Read response: 4-byte status + 4-byte body length + body.
 	statusBuf := make([]byte, 4)
 	if _, err := io.ReadFull(c.conn, statusBuf); err != nil {

agent/internal/rcon/nuclearoption_deadline_test.go

@@ -0,0 +1,46 @@
+package rcon
+
+import (
+	"io"
+	"net"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestNuclearOptionDeadlineIncludesRequestWrites(t *testing.T) {
+	t.Parallel()
+	for _, readHeader := range []bool{false, true} {
+		stage := "write length"
+		if readHeader {
+			stage = "write body"
+		}
+		t.Run(stage, func(t *testing.T) {
+			t.Parallel()
+			local, remote := net.Pipe()
+			defer local.Close()
+			defer remote.Close()
+			client := NewNuclearOption("ignored", 1, nil)
+			client.conn = local
+			client.execDeadline = 100 * time.Millisecond
+			if readHeader {
+				go func() { _, _ = io.CopyN(io.Discard, remote, 4) }()
+			}
+			done := make(chan error, 1)
+			go func() { _, err := client.Exec("get-server-id"); done <- err }()
+			select {
+			case err := <-done:
+				if err == nil || !isTimeout(err) || !strings.Contains(err.Error(), stage) {
+					t.Fatalf("stalled %s must time out: %v", stage, err)
+				}
+			case <-time.After(2 * time.Second):
+				_ = remote.Close()
+				<-done
+				t.Fatalf("stalled %s exceeded the command budget", stage)
+			}
+			if client.conn != nil {
+				t.Fatal("timed-out partial request must discard the connection")
+			}
+		})
+	}
+}

agent/internal/rcon/nuclearoption_test.go

@@ -740,7 +740,7 @@ func TestNuclearOptionWriteBodyError(t *testing.T) {
 }
 
 // TestNuclearOptionSetDeadlineError verifies that a failure setting the
-// response read deadline (after the request was fully written) is reported
+// exchange deadline (before sending the request) is reported
 // and the connection is reset.
 func TestNuclearOptionSetDeadlineError(t *testing.T) {
 	local, remote := net.Pipe()
@@ -1275,18 +1275,47 @@ type pipeFrameCase struct {
 	forbidErr string // substring the error must NOT contain (e.g. a detail suffix)
 }
 
+type responseDeadlineConn struct {
+	net.Conn
+	deadlineSet chan<- struct{}
+}
+
+func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error {
+	if err := c.Conn.SetDeadline(deadline); err != nil {
+		return err
+	}
+	select {
+	case c.deadlineSet <- struct{}{}:
+	default:
+	}
+	return nil
+}
+
 // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe
 // and checks the body or the error.
 func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) {
 	t.Helper()
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			deadlineSet := make(chan struct{}, 1)
 			client := newPipeClient(t, func(remote net.Conn) {
 				if _, _, err := readNuclearOptionRequest(remote); err != nil {
 					return
 				}
+				if len(tt.frame) == 0 {
+					// Wait until the deadline is installed before closing so the
+					// intended error comes from the status read, not SetDeadline.
+					select {
+					case <-deadlineSet:
+					case <-t.Context().Done():
+						return
+					}
+				}
 				writeFrameSegments(remote, tt.frame, tt.oneByte)
 			})
+			if len(tt.frame) == 0 {
+				client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet}
+			}
 
 			body, err := client.Exec("test-command")
 			if tt.wantErr != "" {

agent/specs.md

@@ -244,7 +244,7 @@ The agent and operator modules use `k8s.io/apimachinery`/`k8s.io/client-go` v0.3
   - `battleye`: UDP-based CRC32 packet framing.
   - `satisfactory`: HTTPS JSON function-call API with session token.
   - `palworld`: HTTP Basic REST admin API.
-  - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol.
+  - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. One 30-second deadline bounds dialing, request writes, and response reads; responses are capped at 10 MiB. A timed-out partial exchange closes the connection before the next command.
   - `rest`: Generic HTTP/JSON REST console client with lazy auth (`PassFn`), 1 MiB bounded body read, configurable timeouts, auth failure cooldown, and loopback TLS protection. Provides adapters for FiveM (txAdmin: `POST /fxserver/commands`, bearer/X-TxAdmin-Token, `{"action":"console","parameter":"<cmd>"}`), Farming Simulator 25 (`POST /api/console`, Basic auth, `{"command":"<cmd>"}`), and generic REST (`POST /api/command`).
   - `cli`: Container stdin/PTY / local execution client (Option A in OPEN-DECISIONS.md). Drives commands via named FIFO pipe (`GAMEPLANE_CLI_PIPE`) or local process shell execution without requiring remote TCP networking or password secrets, while web console continues to attach via pod-attach PTY.