From 007733bce9076cb7097acf9c5cbf4924e352c90c Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:31:31 -0400 Subject: [PATCH 1/8] test: exercise isolated panel database and key recovery Signed-off-by: Baylem --- .github/workflows/ci.yaml | 23 +- docs/panel-recovery.md | 167 +++++++ docs/standalone-panel.md | 5 + test/e2e/buckets.sh | 10 +- test/e2e/panel_recovery_e2e_test.go | 526 ++++++++++++++++++++++ test/e2e/specs.md | 14 + test/e2e/testdata/recovery/maintenance.py | 116 +++++ 7 files changed, 856 insertions(+), 5 deletions(-) create mode 100644 docs/panel-recovery.md create mode 100644 test/e2e/panel_recovery_e2e_test.go create mode 100644 test/e2e/testdata/recovery/maintenance.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a1c9aecdd..e84f04cc1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1663,8 +1663,9 @@ jobs: # t.Parallel(); `parallel` bounds in-flight tests per bucket # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). - # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + # The arm64 job below uses the same steps/settings except for the + # SQLite recovery drill, which runs once on amd64. + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, recovery] arch: [amd64] include: - bucket: operator @@ -1700,6 +1701,10 @@ jobs: parallel: 1 test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: recovery + parallel: 1 + test_timeout: 20m # isolated panel backup/loss/restore; game workload stays running + job_timeout: 60 env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e @@ -1720,6 +1725,8 @@ jobs: - name: bring up cluster + helm install run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e - name: run go e2e (${{ matrix.bucket }}) + env: + GAMEPLANE_RECOVERY_REPORT_DIR: ${{ github.workspace }}/recovery-reports run: | RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')" cd test/e2e && \ @@ -1727,6 +1734,14 @@ jobs: GAMEPLANE_E2E_CLUSTER="$CLUSTER" \ go test -tags=e2e -timeout ${{ matrix.test_timeout }} -v \ -parallel ${{ matrix.parallel }} -run "$RUN_REGEX" ./... + - name: upload measured panel recovery report + if: always() && matrix.bucket == 'recovery' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: panel-recovery-${{ matrix.arch }}-${{ github.run_attempt }} + path: recovery-reports/*.json + if-no-files-found: warn + retention-days: 7 - name: run go e2e (ratelimit tail) # LAST on purpose: this test deliberately drains the shared per-IP # and per-username login buckets; anything running after it queues @@ -2510,7 +2525,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|recovery) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2663,7 +2678,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|recovery) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/docs/panel-recovery.md b/docs/panel-recovery.md new file mode 100644 index 000000000..719c00af2 --- /dev/null +++ b/docs/panel-recovery.md @@ -0,0 +1,167 @@ +# Recovering the panel while games keep running + +Panel SQL state, Kubernetes desired state and game files are separate recovery +domains. A game-volume backup cannot reconstruct panel users, password hashes, +roles/grants, sessions, share links, configuration, audit history or standalone +cluster registrations and encrypted credentials. Recover the failed domain; +restoring the panel does not require deleting or recreating live games. + +This runbook extends [standalone storage and key recovery](standalone-panel.md#storage-and-recovery). +Standalone SQL storage was introduced in PR [#630](https://github.com/GameplanePanel/Gameplane/pull/630). +PR [#707](https://github.com/GameplanePanel/Gameplane/pull/707) added PostgreSQL-backed +E2E coverage; it does not provide PostgreSQL backup, PITR or failover automation. + +## Backup inventory and targets + +Record the deployment mode, software/schema version, database location and +backup completion time. Keep the following inventory outside the failed panel: + +| Domain | Retain and verify | +| --- | --- | +| SQL, both modes | Consistent SQLite database/WAL snapshot or external PostgreSQL backups and WAL/PITR; users, role bindings, configuration, sessions/shares/API tokens and audit chain. Standalone also stores management objects and encrypted credentials here. | +| Kubernetes | Cluster/etcd recovery procedure, CRDs, GameServer/Module/ModuleSource/Backup/Restore resources, namespaces/RBAC, management/config/agent/registry Secrets, certificates and external provider credentials. Combined mode stores management resources and remote credentials here, rather than in standalone SQL. | +| Workload data | Primary, extra and mod PVCs, PV identities, storage classes/topology, retention policies, Restic repositories/passwords, CSI snapshot classes/content and backend access. Include game-specific save/restore consistency requirements. | +| Key custody | The exact standalone 32-byte management key, separately protected from database archives, plus keys needed by retained older backups after rotation. Preserve relevant certificate/private-key and secret-manager recovery procedures. | +| Authorization evidence | Intended users, roles, per-cluster/namespace grants, owner/collaborator associations, credentials revoked since the recovery point and audit exports. Record Kubernetes UIDs as well as names. | + +Set targets for each deployment before a drill. Fill in the maximum acceptable +age of restored management state (**RPO**), maximum time from loss until verified +service (**RTO**), responsible operator, backup schedule and restoration location. +Check measured results against those targets and adjust the backup schedule, +storage, tooling or staffing when they are missed. Chart defaults are not a +universal RPO/RTO promise. Measure game-data recovery separately. + +For SQLite, stop every API process and use a SQLite backup tool or copy the +database **and its WAL state** consistently. Never copy only a live `.db` file. +Run `PRAGMA integrity_check` against the restored copy. The CI drill uses SQLite's +backup API with the panel stopped and converts the archive to DELETE journal mode +so it is self-contained. Keep the archive read-only and restore to a separate +working copy. Use the ownership/key permission rules in the standalone guide. + +For PostgreSQL, assign backup/PITR/failover to the database platform: schedule and +test base backups plus WAL retention, protect credentials and select an explicit +recovery timestamp/LSN. Restore into an isolated instance, validate schema and +data, and prevent split-brain writes before switching the API's DSN. Stop all API +writers for offline credential invalidation and keep one API process after +recovery. The SQLite CI drill does not certify a PostgreSQL service's PITR/RTO. + +## Database outage + +Stop or restrict panel traffic and identify whether the database is unreachable, +out of space, locked or corrupt. Preserve logs and the failed storage before +changing it. Restore database connectivity/capacity first when the original +database is intact; do not initialize an empty database or bootstrap new users +as a connectivity repair. Verify readiness, login, scoped access and registered +cluster operations before reopening access. + +Game Pods and the Kubernetes operator can continue independently of the panel. +Do not assume this proves game health: check Kubernetes events, game process +state and saves through the operational access available during the outage. + +## Database loss and recovery order + +1. Restrict external panel access and stop **all** API writers. Fence the old + database instance so it cannot resume writing. Record incident/recovery start + time and live GameServer/PVC/Pod UIDs, container IDs and restart counts. Keep + the operator and healthy game Pods running when only panel SQL was lost. +2. Preserve the failed database and audit evidence. Select a verified backup or + PostgreSQL recovery point, its software/schema version, and the matching + standalone key version. Copy the backup into a new working location; keep + the original immutable. Restore database file ownership/permissions and + verify integrity before starting the API. +3. With every API stopped, reconcile authorization on the **working copy**. + An old backup can resurrect users, grants and credentials revoked later. + Compare the external authorization/revocation inventory, disable/remove stale + accounts and grants, and replace compromised credentials. Review SSO/provider + access too. Preserve original audit rows and record recovery actions in a + separate incident log rather than rewriting the audit chain. +4. Invalidate restored bearer access according to the recovery policy. The drill + deletes all restored `sessions` and `api_tokens`, and sets `share_links.revoked_at` + for every active link. Do this offline in one transaction, on the working copy: + + ```sql + BEGIN; + DELETE FROM sessions; + DELETE FROM api_tokens; + UPDATE share_links SET revoked_at = '' + WHERE revoked_at IS NULL; + COMMIT; + ``` + + Replace the timestamp placeholder before execution. Retain the archive as + evidence, protect access to it, and tell users that they must log in again and + recreate shares/tokens. Do not run cleanup against the evidence archive. +5. Mount the original standalone management key read-only from separate custody. + In combined mode, restore/validate required Kubernetes management objects and + Secrets through the cluster's recovery process instead. Start one API using + compatible software and the restored database; inspect migration/startup + results before exposing it. A wrong/missing key is a hard failure, not a reason + to replace the key or discard encrypted rows. +6. Reconcile the panel's inventory against each target cluster. Check the actual + registered cluster endpoint/identity and credential-backed API operations. + Compare GameServer, PVC and Pod UIDs with the recorded inventory; validate + owner/collaborator user IDs against intended accounts. Names alone do not + establish identity or ownership. Resolve deleted/recreated resources, dangling + references and reused numeric user IDs before granting access. Do not recreate + live workloads merely to make stale panel records appear consistent. +7. Verify restored administrators and restricted users, role definitions, scoped + allow/deny behavior, old-cookie/share denial, cluster credentials and audit + history/integrity. Confirm game Pod/container identities and persistent save + markers remain intact and games continue progressing. Reopen access only + after these checks; record completion time and any lost post-backup changes. +8. Record achieved recovery-point age and RTO, compare with the selected targets, + and retain the drill/incident report without passwords, keys, kubeconfigs, + session cookies or share tokens. Take a new protected backup after the intended + authorization and credential state has been restored. + +## Missing or wrong management key + +Keep the panel stopped. Recover the exact key version paired with the chosen +database backup from separate custody. Existing encrypted credentials cannot be +recovered by generating a new random key. Current startup rejects a missing key +with `management encryption key is missing`; an inactive/wrong key for a recorded +key identity reports `management encryption key is inactive`. Other corruption +may fail authentication of encrypted records. None is permission to serve partial +management state. + +If the matching key is irretrievably lost, retain the database/audit archive and +treat the encrypted credentials as lost. Recover them from their original issuers +or reissue/re-register them under an explicitly reviewed rebuild plan. Reconcile +users, grants and Kubernetes identities before exposing a rebuilt panel. A game +backup cannot recover this key or the encrypted management credentials. + +## Automated CI drill and measured results + +`TestPanel_OfflineRecovery` runs in the amd64 `recovery` E2E bucket. It uses a +unique namespace, its own standalone API from the built image, separate database +and archive PVCs, separate original/wrong key Secrets and a private login budget. +The API has no Kubernetes service-account token: reading the existing GameServer +through a registered cluster proves the restored encrypted kubeconfig works. + +The fixture seeds a custom role, a user with a scoped grant, a registration, +sessions, a share and audit events. It stops the API, takes a consistent archive, +mounts the archive read-only, creates a post-backup user, then deletes the working +database. It restores a copy, invalidates restored bearers while preserving audit +rows and the archive digest, and checks wrong-key startup exits without a successful +HTTP response during the startup probe. With the original key it checks login, +allow/deny permissions, registration-backed reads, old-cookie/share denial and +loss of the deliberately post-backup user. A busybox workload retains its Pod UID, +container ID/restart count and persistent marker while its counter advances. + +The `panel-recovery--` CI artifact contains JSON with: + +- `recovery_point_age_at_loss_seconds`: age of the completed fixture backup when + the working database was lost. This measures the fixture's recovery point; + post-backup changes are deliberately absent after restore. +- `rto_seconds`: time from simulated loss through restoration and acceptance + checks, **including** the negative wrong-key trial. Cluster/bootstrap setup and + backup creation precede this clock. +- Check results, archive/audit digests and row counts, and game identity evidence. + +The report records actual run values; no benchmark values are hard-coded here. +A failed run writes a partial report with `passed: false`; missing timing fields +mean that stage was not reached. The artifact contains no keys or raw database +rows. Use repeated passing runs and your own production-shaped drills to select +and demonstrate targets. This fixture covers SQLite panel recovery, not storage +cluster loss, PostgreSQL failover, a particular game's save format, or a complete +Kubernetes disaster recovery exercise. diff --git a/docs/standalone-panel.md b/docs/standalone-panel.md index c405b996b..d1deab8eb 100644 --- a/docs/standalone-panel.md +++ b/docs/standalone-panel.md @@ -66,6 +66,11 @@ registered clusters. Gateway credentials are configured through the API. ## Storage and recovery +For database outage/loss, key loss, authorization reconciliation and restoring +the panel while games continue running, follow the [panel recovery runbook](panel-recovery.md). +It builds on the storage/key procedures below and describes the CI recovery +drill's measured recovery-point age and recovery time. + The Compose volume `panel-data` contains `/data/gameplane.db`; a separate `panel-keys` volume contains `/keys/panel.key`. Standalone settings and remote credentials are stored in SQL; credential values are authenticated-encrypted diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d60a26f14..d21411ea1 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -379,6 +379,13 @@ TestOptionalComponents_Lifecycle EOF } +# recovery owns a separate standalone API/database/key in a unique namespace. +# Zero shared panel logins; the fixture has its own login limiter and storage. +bucket_recovery() { cat <<'EOF' +TestPanel_OfflineRecovery +EOF +} + # capture-overhead is kept OUT of bot-fast on purpose: bot-fast's matrix legs # are blocking, while this bucket feeds the non-blocking e2e-capture-overhead # job (continue-on-error, never blocks a merge — spec 007 OD-1) and runs only @@ -395,7 +402,7 @@ EOF unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components recovery } list_bucket() { @@ -414,6 +421,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + recovery) bucket_recovery ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go new file mode 100644 index 000000000..45a54bee7 --- /dev/null +++ b/test/e2e/panel_recovery_e2e_test.go @@ -0,0 +1,526 @@ +//go:build e2e + +package e2e + +import ( + "context" + "crypto/rand" + _ "embed" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + clientcmdapi "k8s.io/client-go/tools/clientcmd/api" +) + +//go:embed testdata/recovery/maintenance.py +var recoveryMaintenance string + +type recoveryFixture struct { + t *testing.T + ns, image, node, allowedCIDR string +} + +// TestPanel_OfflineRecovery owns its API, database, archive and login budget. +// It never stops the installed API/operator or restores shared cluster state. +func TestPanel_OfflineRecovery(t *testing.T) { + t.Parallel() + e := envInstance + ctx := t.Context() + suffix := strconv.FormatInt(time.Now().UnixNano(), 36) + f := &recoveryFixture{t: t, ns: "e2e-recovery-" + suffix} + report := map[string]any{"fixture": "standalone-sqlite-kind", "namespace": f.ns, "started_utc": time.Now().UTC(), "stage": "bootstrap", "passed": false} + t.Cleanup(func() { + report["passed"], report["finished_utc"] = !t.Failed(), time.Now().UTC() + dir := os.Getenv("GAMEPLANE_RECOVERY_REPORT_DIR") + if dir == "" { + dir = t.TempDir() + } + if err := os.MkdirAll(dir, 0o700); err != nil { + t.Errorf("create recovery report directory: %v", err) + return + } + data, err := json.MarshalIndent(report, "", " ") + if err != nil { + t.Errorf("encode recovery report: %v", err) + return + } + if err := os.WriteFile(filepath.Join(dir, f.ns+".json"), data, 0o600); err != nil { + t.Errorf("write recovery report: %v", err) + } + }) + if _, err := e.K8s.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: f.ns}}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = e.K8s.CoreV1().Namespaces().Delete(context.Background(), f.ns, metav1.DeleteOptions{}) }) + deploy, err := e.K8s.AppsV1().Deployments("gameplane-system").Get(ctx, "gameplane-api", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for _, c := range deploy.Spec.Template.Spec.Containers { + if c.Name == "api" { + f.image = c.Image + } + } + if f.image == "" { + t.Fatal("installed API image unavailable") + } + kubernetes, err := e.K8s.CoreV1().Services("default").Get(ctx, "kubernetes", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + f.allowedCIDR = kubernetes.Spec.ClusterIP + "/32" + if strings.Contains(kubernetes.Spec.ClusterIP, ":") { + f.allowedCIDR = kubernetes.Spec.ClusterIP + "/128" + } + for _, name := range []string{"data", "archive"} { + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: f.ns}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}}}} + if _, err := e.K8s.CoreV1().PersistentVolumeClaims(f.ns).Create(ctx, pvc, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + password := recoveryRandom(t) + for name, data := range map[string]map[string][]byte{ + "original-key": {"panel.key": recoveryKey(t)}, "wrong-key": {"panel.key": recoveryKey(t)}, "bootstrap": {"password": []byte(password)}, + } { + if _, err := e.K8s.CoreV1().Secrets(f.ns).Create(ctx, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name}, Data: data}, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + } + service := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: "panel", Namespace: f.ns}, Spec: corev1.ServiceSpec{Selector: map[string]string{"app": "recovery-panel"}, PublishNotReadyAddresses: true, Ports: []corev1.ServicePort{{Port: 8000, TargetPort: intstr.FromInt32(8000)}}}} + if _, err := e.K8s.CoreV1().Services(f.ns).Create(ctx, service, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + f.startAPI("initial", "original-key", true) + pod := f.waitPodReady("initial") + f.node = pod.Spec.NodeName + f.startMaintenance(false) + probeURL := "http://panel." + f.ns + ".svc:8000/healthz" + // Positive control: a blocked or misrouted Service must not make the later + // no-HTTP negative case appear to pass. + e.Eventually(t, 30*time.Second, func() (bool, string) { + result := f.maintenance("http-check", probeURL) + return result["status"] == float64(http.StatusOK), "waiting for recovery HTTP monitor positive control" + }) + report["http_monitor_positive_control"] = true + admin := f.login("initial", "admin", password) + defer admin.Close() + + // Register the existing kind cluster using stored, encrypted credentials. + // The panel has no service-account token, so successful dispatch depends on + // decrypting this registration after recovery. + clusterID := "fixture-cluster" + f.request(admin, http.MethodPost, "/clusters", map[string]any{"name": clusterID, "displayName": "Recovery fixture", "kubeconfig": recoveryKubeconfig(t)}, http.StatusCreated) + roleName := "recovery-reader" + f.request(admin, http.MethodPost, "/roles", map[string]any{"name": roleName, "description": "Recovery fixture read only", "permissions": []string{"servers:read"}}, http.StatusCreated) + userPassword := recoveryRandom(t) + created := f.request(admin, http.MethodPost, "/users", map[string]any{"username": "reader", "password": userPassword, "role": roleName}, http.StatusOK) + userID := extractIntField(string(created), "id") + if userID == "" { + t.Fatal("created reader has no ID") + } + f.request(admin, http.MethodPost, "/users/"+userID+"/bindings", map[string]any{"roleName": roleName, "cluster": clusterID, "namespace": "gameplane-games"}, http.StatusCreated) + reader := f.login("initial", "reader", userPassword) + defer reader.Close() + gameName, gameUID := recoveryGame(t, suffix) + serversPath := "/servers?cluster=" + clusterID + "&namespace=gameplane-games" + f.waitServer(reader, serversPath, gameUID) + shareBody := f.request(admin, http.MethodPost, "/servers/"+gameName+":shares?cluster="+clusterID+"&namespace=gameplane-games", map[string]any{"canStart": false, "neverExpires": true}, http.StatusOK) + var share struct { + Token string `json:"token"` + } + if err := json.Unmarshal(shareBody, &share); err != nil || share.Token == "" { + t.Fatal("share token missing") + } + public := &APIClient{BaseURL: admin.BaseURL, HTTP: &http.Client{Timeout: 10 * time.Second}} + f.request(public, http.MethodGet, "/shares/"+share.Token, nil, http.StatusOK) + beforeGame := recoveryGameIdentity(t, gameName) + beforeCounter := recoveryCounter(t, gameName) + report["game_before"] = beforeGame + report["counter_before"] = beforeCounter + + // Stop the sole API before SQLite's online-backup API takes the archive. + admin.Close() + reader.Close() + f.stopPod("initial") + report["stage"] = "backup" + backup := f.maintenance("backup") + report["backup"] = backup + backupTime, err := time.Parse(time.RFC3339Nano, backup["backup_utc"].(string)) + if err != nil { + t.Fatal("invalid archive recovery-point timestamp") + } + if backup["sessions"].(float64) < 2 || backup["active_shares"].(float64) < 1 || backup["audit_rows"].(float64) < 1 { + t.Fatal("backup lacks session/share/audit fixture evidence") + } + // From here on the archive is mounted read-only even in the maintenance Pod. + f.stopPod("maintenance") + f.startMaintenance(true) + f.startAPI("after-backup", "original-key", false) + f.waitPodReady("after-backup") + afterBackup := f.login("after-backup", "admin", password) + f.request(afterBackup, http.MethodPost, "/users", map[string]any{"username": "after-backup", "password": recoveryRandom(t), "role": "viewer"}, http.StatusOK) + afterBackup.Close() + f.stopPod("after-backup") + lossTime := time.Now() + report["stage"] = "loss-and-restore" + report["loss_utc"] = lossTime.UTC() + report["recovery_point_age_at_loss_seconds"] = lossTime.Sub(backupTime).Seconds() + loss := f.maintenance("loss") + if loss["working_database_absent"] != true || loss["archive_digest"] != backup["archive_digest"] { + t.Fatal("loss simulation did not preserve separate archive") + } + restored := f.maintenance("restore") + report["restored_copy"] = restored + if restored["archive_digest"] != backup["archive_digest"] || restored["audit_digest"] != backup["audit_digest"] || restored["audit_rows"] != backup["audit_rows"] { + t.Fatal("restore changed original archive or audit evidence") + } + if recoveryCounter(t, gameName) <= beforeCounter { + t.Fatal("game did not continue across panel downtime") + } + + // Start an HTTP monitor before the wrong-key process. A startup error alone + // is insufficient evidence if a listener briefly served partial state first. + report["stage"] = "wrong-key" + probeCtx, cancelProbe := context.WithCancel(ctx) + defer cancelProbe() + type probeResult struct { + output string + err error + } + probeDone := make(chan probeResult, 1) + go func() { + out, err := f.execMaintenance(probeCtx, "probe", probeURL) + probeDone <- probeResult{out, err} + }() + e.Eventually(t, 30*time.Second, func() (bool, string) { + result := f.maintenance("probe-ready") + return result["ready"] == true, "waiting for wrong-key HTTP monitor" + }) + f.startAPI("wrong-key", "wrong-key", false) + e.Eventually(t, 90*time.Second, func() (bool, string) { + p, err := e.K8s.CoreV1().Pods(f.ns).Get(ctx, "wrong-key", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, status := range p.Status.ContainerStatuses { + if status.Name == "api" && status.State.Terminated != nil { + return status.State.Terminated.ExitCode != 0, "wrong-key process exited successfully" + } + } + return false, "waiting for wrong-key startup rejection" + }) + logs, err := e.K8s.CoreV1().Pods(f.ns).GetLogs("wrong-key", &corev1.PodLogOptions{Container: "api"}).DoRaw(ctx) + if err != nil || !strings.Contains(string(logs), "management encryption key is inactive") { + t.Fatal("wrong-key startup did not report the expected inactive-key error (logs withheld)") + } + f.maintenance("probe-stop") + var probe map[string]any + select { + case result := <-probeDone: + if result.err != nil || json.Unmarshal([]byte(result.output), &probe) != nil { + t.Fatal("wrong-key HTTP monitor failed") + } + case <-ctx.Done(): + t.Fatal("wrong-key HTTP monitor did not stop") + } + if probe["responses"] != float64(0) || probe["stopped"] != true || probe["attempts"].(float64) < 1 { + t.Fatal("wrong-key process served HTTP or monitor did not finish") + } + report["wrong_key_probe"] = probe + f.stopPod("wrong-key") + report["stage"] = "restored-acceptance" + f.startAPI("restored", "original-key", false) + f.waitPodReady("restored") + recovered := f.login("restored", "admin", password) + defer recovered.Close() + // Reuse the original cookie jar against the recovered endpoint, without login. + stale := &APIClient{BaseURL: recovered.BaseURL, HTTP: reader.HTTP, CSRF: reader.CSRF} + f.request(stale, http.MethodGet, "/auth/me", nil, http.StatusUnauthorized) + public.BaseURL = recovered.BaseURL + f.request(public, http.MethodGet, "/shares/"+share.Token, nil, http.StatusNotFound) + recoveredReader := f.login("restored", "reader", userPassword) + defer recoveredReader.Close() + f.waitServer(recoveredReader, serversPath, gameUID) + f.request(recoveredReader, http.MethodPost, "/servers?cluster="+clusterID+"&namespace=gameplane-games", map[string]any{"metadata": map[string]any{"name": "forbidden"}}, http.StatusForbidden) + users := f.request(recovered, http.MethodGet, "/users", nil, http.StatusOK) + if strings.Contains(string(users), `"after-backup"`) || !strings.Contains(string(users), `"reader"`) { + t.Fatal("restored users do not match the recovery point") + } + roles := f.request(recovered, http.MethodGet, "/roles", nil, http.StatusOK) + if !strings.Contains(string(roles), `"`+roleName+`"`) { + t.Fatal("custom role missing after recovery") + } + bindings := f.request(recovered, http.MethodGet, "/users/"+userID+"/bindings", nil, http.StatusOK) + if !strings.Contains(string(bindings), `"`+clusterID+`"`) { + t.Fatal("scoped grant missing after recovery") + } + f.request(recovered, http.MethodGet, "/admin/audit", nil, http.StatusOK) + verification := f.request(recovered, http.MethodGet, "/admin/audit/verify", nil, http.StatusOK) + var integrity struct { + OK bool `json:"ok"` + Checked int64 `json:"checked"` + } + if json.Unmarshal(verification, &integrity) != nil || !integrity.OK || integrity.Checked < 1 { + t.Fatal("restored audit chain failed verification") + } + report["audit_chain_verified_rows"] = integrity.Checked + if f.maintenance("archive")["archive_digest"] != backup["archive_digest"] { + t.Fatal("original archive mutated during recovery") + } + afterGame := recoveryGameIdentity(t, gameName) + if afterGame != beforeGame { + t.Fatalf("game identity changed during panel recovery: before=%+v after=%+v", beforeGame, afterGame) + } + e.Eventually(t, 15*time.Second, func() (bool, string) { + return recoveryCounter(t, gameName) > beforeCounter, "game counter did not advance" + }) + report["game_after"], report["credential_backed_dispatch_verified"], report["restored_bearers_invalidated"] = afterGame, true, true + report["counter_after"] = recoveryCounter(t, gameName) + report["rto_seconds"] = time.Since(lossTime).Seconds() + report["rto_scope"] = "loss through restore, auth invalidation, wrong-key trial, and credential-backed acceptance checks" + report["stage"] = "complete" +} + +func recoveryKey(t *testing.T) []byte { + t.Helper() + key := make([]byte, 32) + if _, err := rand.Read(key); err != nil { + t.Fatal(err) + } + return key +} + +func recoveryRandom(t *testing.T) string { return hex.EncodeToString(recoveryKey(t)) } + +func (f *recoveryFixture) request(c *APIClient, method, path string, body any, want int) []byte { + f.t.Helper() + resp, data, err := c.Do(method, path, body) + if err != nil { + f.t.Fatalf("recovery HTTP request failed (details withheld): %s", method) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != want { + f.t.Fatalf("recovery HTTP %s: status=%d want=%d (path/body withheld)", method, resp.StatusCode, want) + } + return data +} + +func (f *recoveryFixture) login(pod, user, password string) *APIClient { + f.t.Helper() + port, stop := envInstance.PortForward(f.t, f.ns, "pod/"+pod, 8000) + c := &APIClient{BaseURL: fmt.Sprintf("http://127.0.0.1:%d", port), HTTP: &http.Client{Jar: newInsecureCookieJar(), Timeout: 20 * time.Second}, stop: stop} + f.t.Cleanup(stop) + data := f.request(c, http.MethodPost, "/auth/login", map[string]string{"username": user, "password": password}, http.StatusOK) + var login struct { + CSRF string `json:"csrf"` + } + if err := json.Unmarshal(data, &login); err != nil || login.CSRF == "" { + f.t.Fatal("login response lacked CSRF token") + } + c.CSRF = login.CSRF + return c +} + +func recoverySecurity() (*corev1.PodSecurityContext, *corev1.SecurityContext) { + uid, yes, no := int64(65532), true, false + return &corev1.PodSecurityContext{RunAsUser: &uid, RunAsGroup: &uid, FSGroup: &uid, RunAsNonRoot: &yes, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, &corev1.SecurityContext{AllowPrivilegeEscalation: &no, ReadOnlyRootFilesystem: &yes, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}} +} + +func (f *recoveryFixture) startAPI(name, key string, bootstrap bool) { + f.t.Helper() + psc, sc := recoverySecurity() + no, mode := false, int32(0o440) + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: f.ns, Labels: map[string]string{"app": "recovery-panel"}}, Spec: corev1.PodSpec{RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &no, SecurityContext: psc, Containers: []corev1.Container{{Name: "api", Image: f.image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/api"}, Args: []string{"--standalone", "--panel-key-file=/keys/panel.key", "--panel-key-provisioned", "--db-driver=sqlite", "--db-dsn=file:/data/gameplane.db?_pragma=journal_mode(WAL)", "--remote-allowed-cidrs=" + f.allowedCIDR}, SecurityContext: sc, VolumeMounts: []corev1.VolumeMount{{Name: "data", MountPath: "/data"}, {Name: "key", MountPath: "/keys", ReadOnly: true}}, ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(8000)}}, PeriodSeconds: 1}, Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m"), corev1.ResourceMemory: resource.MustParse("128Mi")}, Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("512Mi")}}}}, Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}}, {Name: "key", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: key, DefaultMode: &mode}}}}}} + if bootstrap { + pod.Spec.InitContainers = []corev1.Container{{Name: "bootstrap", Image: f.image, ImagePullPolicy: corev1.PullIfNotPresent, Command: []string{"/api", "bootstrap-admin", "--username", "admin"}, SecurityContext: sc, Env: []corev1.EnvVar{{Name: "GAMEPLANE_DB_DSN", Value: "file:/data/gameplane.db?_pragma=journal_mode(WAL)"}, {Name: "GAMEPLANE_ADMIN_PASSWORD", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "bootstrap"}, Key: "password"}}}}, VolumeMounts: []corev1.VolumeMount{{Name: "data", MountPath: "/data"}}}} + } + if _, err := envInstance.K8s.CoreV1().Pods(f.ns).Create(f.t.Context(), pod, metav1.CreateOptions{}); err != nil { + f.t.Fatal(err) + } +} + +func (f *recoveryFixture) waitPodReady(name string) *corev1.Pod { + f.t.Helper() + var pod *corev1.Pod + envInstance.Eventually(f.t, 3*time.Minute, func() (bool, string) { + var err error + pod, err = envInstance.K8s.CoreV1().Pods(f.ns).Get(f.t.Context(), name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, c := range pod.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return true, "" + } + } + return false, "waiting for isolated fixture Pod " + name + " (phase=" + string(pod.Status.Phase) + ")" + }) + return pod +} + +func (f *recoveryFixture) stopPod(name string) { + f.t.Helper() + if err := envInstance.K8s.CoreV1().Pods(f.ns).Delete(f.t.Context(), name, metav1.DeleteOptions{}); err != nil { + f.t.Fatal(err) + } + envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { + _, err := envInstance.K8s.CoreV1().Pods(f.ns).Get(f.t.Context(), name, metav1.GetOptions{}) + return apierrors.IsNotFound(err), "waiting for API termination before offline database operation" + }) +} + +func (f *recoveryFixture) startMaintenance(archiveReadOnly bool) { + psc, sc := recoverySecurity() + no := false + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "maintenance", Namespace: f.ns}, Spec: corev1.PodSpec{RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &no, SecurityContext: psc, Affinity: &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{f.node}}}}}}}}, Containers: []corev1.Container{{Name: "maintenance", Image: "python:3.12-alpine", Command: []string{"python3", "-c", "import time; time.sleep(1800)"}, SecurityContext: sc, VolumeMounts: []corev1.VolumeMount{{Name: "data", MountPath: "/data"}, {Name: "archive", MountPath: "/archive"}}}}, Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "data"}}}, {Name: "archive", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "archive"}}}}}} + pod.Spec.Containers[0].VolumeMounts[1].ReadOnly = archiveReadOnly + pod.Spec.Volumes[1].PersistentVolumeClaim.ReadOnly = archiveReadOnly + if _, err := envInstance.K8s.CoreV1().Pods(f.ns).Create(f.t.Context(), pod, metav1.CreateOptions{}); err != nil { + f.t.Fatal(err) + } + f.waitPodReady("maintenance") +} + +func (f *recoveryFixture) execMaintenance(ctx context.Context, args ...string) (string, error) { + command := []string{"exec", "-n", f.ns, "maintenance", "--", "python3", "-c", recoveryMaintenance} + return envInstance.Kubectl(ctx, append(command, args...)...) +} + +func (f *recoveryFixture) maintenance(op string, args ...string) map[string]any { + f.t.Helper() + out, err := f.execMaintenance(f.t.Context(), append([]string{op}, args...)...) + if err != nil { + f.t.Fatalf("offline recovery operation %s failed (output withheld)", op) + } + var result map[string]any + if err := json.Unmarshal([]byte(out), &result); err != nil { + f.t.Fatalf("decode offline recovery operation %s", op) + } + return result +} + +func recoveryKubeconfig(t *testing.T) string { + t.Helper() + cfg := rest.CopyConfig(envInstance.Cfg) + if err := rest.LoadTLSFiles(cfg); err != nil { + t.Fatal("could not load kind TLS fixture") + } + if len(cfg.CAData) == 0 || len(cfg.CertData) == 0 || len(cfg.KeyData) == 0 { + t.Fatal("recovery drill requires kind client certificate credentials") + } + kubeconfig := clientcmdapi.Config{Clusters: map[string]*clientcmdapi.Cluster{"kind": {Server: "https://kubernetes.default.svc", CertificateAuthorityData: cfg.CAData}}, AuthInfos: map[string]*clientcmdapi.AuthInfo{"kind": {ClientCertificateData: cfg.CertData, ClientKeyData: cfg.KeyData}}, Contexts: map[string]*clientcmdapi.Context{"kind": {Cluster: "kind", AuthInfo: "kind"}}, CurrentContext: "kind"} + data, err := clientcmd.Write(kubeconfig) + if err != nil { + t.Fatal("could not encode kind TLS fixture") + } + return string(data) +} + +func (f *recoveryFixture) waitServer(c *APIClient, path, uid string) { + f.t.Helper() + envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { + resp, data, err := c.Get(path) + if err != nil { + return false, "waiting for credential-backed cluster dispatch" + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return false, "waiting for credential-backed cluster dispatch" + } + var list struct { + Items []unstructured.Unstructured `json:"items"` + } + if json.Unmarshal(data, &list) != nil { + return false, "invalid server inventory" + } + for _, item := range list.Items { + if string(item.GetUID()) == uid { + return true, "" + } + } + return false, "expected live GameServer UID absent from registered cluster" + }) +} + +func recoveryGame(t *testing.T, suffix string) (string, string) { + t.Helper() + name := "recovery-game-" + suffix + applyBusyboxTemplate(t, name) + tmpl, err := envInstance.Dyn.Resource(gameTemplateGVR).Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := unstructured.SetNestedStringSlice(tmpl.Object, []string{"sh", "-c", "echo retained > /data/recovery-marker; n=0; while true; do n=$((n+1)); echo $n > /data/recovery-counter.next; mv /data/recovery-counter.next /data/recovery-counter; sleep 1; done"}, "spec", "command"); err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Update(t.Context(), tmpl, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + applyBusyboxGameServer(t, "gameplane-games", name, name) + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods("gameplane-games").Get(t.Context(), name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, c := range pod.Status.ContainerStatuses { + if c.Name == "game" && c.State.Running != nil { + return true, "" + } + } + return false, "waiting for persistent game fixture" + }) + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace("gameplane-games").Get(t.Context(), name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return name, string(gs.GetUID()) +} + +type recoveryGameState struct { + PodUID, ContainerID string + Restarts int32 +} + +func recoveryGameIdentity(t *testing.T, name string) recoveryGameState { + t.Helper() + pod, err := envInstance.K8s.CoreV1().Pods("gameplane-games").Get(t.Context(), name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + for _, c := range pod.Status.ContainerStatuses { + if c.Name == "game" && c.State.Running != nil && c.ContainerID != "" { + return recoveryGameState{string(pod.UID), c.ContainerID, c.RestartCount} + } + } + t.Fatal("game container is not running") + return recoveryGameState{} +} + +func recoveryCounter(t *testing.T, name string) int { + t.Helper() + out, err := envInstance.Kubectl(t.Context(), "exec", "-n", "gameplane-games", name+"-0", "-c", "game", "--", "sh", "-c", "test \"$(cat /data/recovery-marker)\" = retained && cat /data/recovery-counter") + if err != nil { + t.Fatal("read persistent game marker/counter") + } + n, err := strconv.Atoi(strings.TrimSpace(out)) + if err != nil { + t.Fatal("invalid game counter") + } + return n +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index abf7cda91..aaf952839 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -19,6 +19,20 @@ It was one of three `go.work` modules without CI lint coverage until this featur - Validate multi-cluster dispatch, upgrade-from-previous-release, wake-on-connect, module signing/SSRF rejection, and backup/restore round trips end to end. - Enforce, via `buckets.sh`, that every test in the suite is assigned to exactly one CI bucket so nothing silently drops out of coverage. +### Panel recovery drill + +The amd64 `recovery` bucket runs `TestPanel_OfflineRecovery` in a unique namespace +with its own standalone API, SQL database/archive PVCs, separately mounted keys +and private login budget. It exercises stopped SQLite backup, working database +loss, restore and offline bearer invalidation. It verifies correct-key users, +custom roles/scoped grants, encrypted-registration-backed GameServer UID reads, +wrong-key startup failure/no HTTP response, old session/share rejection, retained +audit evidence and an unchanged continuously running game Pod/container. The +read-only original archive is never changed by invalidation. Actual fixture +recovery-point age/RTO and acceptance evidence are uploaded as JSON, with no key, +credential, cookie, share token or raw SQL rows. It does not stop the installed +panel/operator or claim PostgreSQL PITR coverage. See `docs/panel-recovery.md`. + ### Remote gateway parity acceptance The multicluster bucket runs `TestMultiCluster_GatewayParity` against two real diff --git a/test/e2e/testdata/recovery/maintenance.py b/test/e2e/testdata/recovery/maintenance.py new file mode 100644 index 000000000..9c7d65711 --- /dev/null +++ b/test/e2e/testdata/recovery/maintenance.py @@ -0,0 +1,116 @@ +"""Offline SQLite maintenance for the isolated recovery fixture, never production. + +The API must be stopped for backup/loss/restore. No key is mounted here. Output +contains only counts, timestamps and digests, never rows, credentials or tokens. +""" +import datetime +from contextlib import closing +import hashlib +import json +import pathlib +import sqlite3 +import sys +import time +import urllib.error +import urllib.request + +DATA = pathlib.Path("/data/gameplane.db") +ARCHIVE = pathlib.Path("/archive/panel-backup.db") + + +def digest(path): + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def readonly(path): + return sqlite3.connect("file:" + str(path) + "?mode=ro", uri=True) + + +def inventory(db): + assert db.execute("PRAGMA integrity_check").fetchall() == [("ok",)] + rows = db.execute("SELECT * FROM audit_events ORDER BY id").fetchall() + return { + "audit_rows": len(rows), + "audit_digest": hashlib.sha256(json.dumps(rows, separators=(",", ":")).encode()).hexdigest(), + "users": db.execute("SELECT COUNT(*) FROM users").fetchone()[0], + "sessions": db.execute("SELECT COUNT(*) FROM sessions").fetchone()[0], + "active_shares": db.execute("SELECT COUNT(*) FROM share_links WHERE revoked_at IS NULL").fetchone()[0], + } + + +def remove_working_database(): + for suffix in ("", "-wal", "-shm"): + pathlib.Path(str(DATA) + suffix).unlink(missing_ok=True) + + +op = sys.argv[1] +if op == "backup": + assert not ARCHIVE.exists(), "archive must not be overwritten" + with closing(readonly(DATA)) as source, closing(sqlite3.connect(ARCHIVE)) as dest: + source.backup(dest) + # Make the archive self-contained; do not retain a destination WAL. + dest.execute("PRAGMA journal_mode=DELETE") + result = inventory(dest) + # The original archive is never modified again, including by auth cleanup. + ARCHIVE.chmod(0o400) + result.update(archive_digest=digest(ARCHIVE), backup_utc=datetime.datetime.now(datetime.timezone.utc).isoformat()) +elif op == "loss": + assert ARCHIVE.is_file(), "loss requires a separate archive" + remove_working_database() + result = {"working_database_absent": not DATA.exists(), "archive_digest": digest(ARCHIVE)} +elif op == "restore": + assert not DATA.exists(), "restore only into the deleted working database" + before = digest(ARCHIVE) + with closing(readonly(ARCHIVE)) as source, closing(sqlite3.connect(DATA)) as dest: + source.backup(dest) + original = inventory(source) + assert inventory(dest) == original + # Invalidate restored bearer credentials on the working copy, preserving + # the original archive and audit chain for incident investigation. + dest.execute("DELETE FROM sessions") + dest.execute("DELETE FROM api_tokens") + dest.execute("UPDATE share_links SET revoked_at=? WHERE revoked_at IS NULL", (datetime.datetime.now(datetime.timezone.utc).isoformat(),)) + dest.commit() + result = inventory(dest) + assert result["audit_digest"] == original["audit_digest"] + assert result["sessions"] == 0 and result["active_shares"] == 0 + DATA.chmod(0o600) + assert digest(ARCHIVE) == before + result["archive_digest"] = before +elif op == "archive": + result = {"archive_digest": digest(ARCHIVE)} +elif op == "http-check": + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(sys.argv[2], timeout=2) as response: + result = {"status": response.status} + except (urllib.error.URLError, TimeoutError, OSError): + result = {"status": 0} +elif op == "probe": + stop = pathlib.Path("/data/probe-stop") + stop.unlink(missing_ok=True) + pathlib.Path("/data/probe-ready").write_text("ready") + attempts, responses = 0, 0 + deadline = time.monotonic() + 120 + # Bypass proxy environment settings: this is an in-cluster Service probe. + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + while not stop.exists() and time.monotonic() < deadline: + attempts += 1 + try: + with opener.open(sys.argv[2], timeout=0.5) as response: + response.read(1) + responses += 1 + except urllib.error.HTTPError: + responses += 1 # Any HTTP response is a failure for wrong-key startup. + except (urllib.error.URLError, TimeoutError, OSError): + pass + time.sleep(0.05) + result = {"attempts": attempts, "responses": responses, "stopped": stop.exists()} +elif op == "probe-ready": + result = {"ready": pathlib.Path("/data/probe-ready").exists()} +elif op == "probe-stop": + pathlib.Path("/data/probe-stop").write_text("stop") + result = {"stopped": True} +else: + raise ValueError("unknown maintenance operation") +print(json.dumps(result, sort_keys=True)) From 51c63720147492bd88308ef62a44a93e7bd5051e Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:32:23 -0400 Subject: [PATCH 2/8] test: bind recovery requests to the fixture context Signed-off-by: Baylem --- test/e2e/env.go | 7 ++++++- test/e2e/panel_recovery_e2e_test.go | 4 ++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/e2e/env.go b/test/e2e/env.go index 35e3da008..abd376d19 100644 --- a/test/e2e/env.go +++ b/test/e2e/env.go @@ -571,6 +571,11 @@ func (c *APIClient) Close() { // marshalled as JSON and the Content-Type header is set. CSRF header is // attached for mutating methods. func (c *APIClient) Do(method, path string, body any) (*http.Response, []byte, error) { + return c.DoContext(context.Background(), method, path, body) +} + +// DoContext performs an authenticated request within the caller's lifetime. +func (c *APIClient) DoContext(ctx context.Context, method, path string, body any) (*http.Response, []byte, error) { var br io.Reader if body != nil { b, err := json.Marshal(body) @@ -579,7 +584,7 @@ func (c *APIClient) Do(method, path string, body any) (*http.Response, []byte, e } br = bytes.NewReader(b) } - req, err := http.NewRequestWithContext(context.Background(), method, c.BaseURL+path, br) + req, err := http.NewRequestWithContext(ctx, method, c.BaseURL+path, br) if err != nil { return nil, nil, fmt.Errorf("new request: %w", err) } diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index 45a54bee7..6fae31437 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -310,7 +310,7 @@ func recoveryRandom(t *testing.T) string { return hex.EncodeToString(recoveryKey func (f *recoveryFixture) request(c *APIClient, method, path string, body any, want int) []byte { f.t.Helper() - resp, data, err := c.Do(method, path, body) + resp, data, err := c.DoContext(f.t.Context(), method, path, body) if err != nil { f.t.Fatalf("recovery HTTP request failed (details withheld): %s", method) } @@ -435,7 +435,7 @@ func recoveryKubeconfig(t *testing.T) string { func (f *recoveryFixture) waitServer(c *APIClient, path, uid string) { f.t.Helper() envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { - resp, data, err := c.Get(path) + resp, data, err := c.DoContext(f.t.Context(), http.MethodGet, path, nil) if err != nil { return false, "waiting for credential-backed cluster dispatch" } From f12d902659b812e85124a1d6f6ff33c9b737b629 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 17:11:13 -0400 Subject: [PATCH 3/8] test: specialize recovery user ID parsing Signed-off-by: Baylem --- test/e2e/api_auth_e2e_test.go | 4 ++-- test/e2e/api_session_e2e_test.go | 2 +- test/e2e/env.go | 12 ++++++------ test/e2e/panel_recovery_e2e_test.go | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/e2e/api_auth_e2e_test.go b/test/e2e/api_auth_e2e_test.go index fafac66e1..510d277ec 100644 --- a/test/e2e/api_auth_e2e_test.go +++ b/test/e2e/api_auth_e2e_test.go @@ -187,7 +187,7 @@ func TestAPI_AuditEmitsOnMutation(t *testing.T) { } // Extract our id without pulling in encoding/json: the response // shape is small and we only need the leading "id":N field. - id := extractIntField(string(body), "id") + id := extractUserID(string(body)) if id == "" { t.Fatalf("could not parse id from /users/me: %s", string(body)) } @@ -224,7 +224,7 @@ func TestAPI_AuditEmitsOnMutation(t *testing.T) { // the *passed-down* context, which is out of scope for this test. } -// extractIntField has been promoted to env.go so non-test helper code +// extractUserID has been promoted to env.go so non-test helper code // (Env.CreateUser) can use it. Test files in this package still call // it through the package-level scope. diff --git a/test/e2e/api_session_e2e_test.go b/test/e2e/api_session_e2e_test.go index 5110beba0..603b8d69f 100644 --- a/test/e2e/api_session_e2e_test.go +++ b/test/e2e/api_session_e2e_test.go @@ -168,7 +168,7 @@ func TestAPI_AuditPaginationAndFilter(t *testing.T) { t.Fatalf("/users/me %d: %s", getMe.StatusCode, string(body)) } defer func() { _ = getMe.Body.Close() }() - id := extractIntField(string(body), "id") + id := extractUserID(string(body)) if id == "" { t.Fatalf("could not parse id from /users/me: %s", string(body)) } diff --git a/test/e2e/env.go b/test/e2e/env.go index abd376d19..4f5cb055d 100644 --- a/test/e2e/env.go +++ b/test/e2e/env.go @@ -696,21 +696,21 @@ func (e *Env) CreateUser(t *testing.T, admin *APIClient, role, prefix string) (u if resp.StatusCode != http.StatusOK { t.Fatalf("CreateUser %s/%s %d: %s", role, prefix, resp.StatusCode, string(body)) } - id = extractIntField(string(body), "id") + id = extractUserID(string(body)) if id == "" { t.Fatalf("CreateUser: could not parse id from response: %s", string(body)) } return username, userPhrase, id } -// extractIntField is a tiny scanner for `"":` inside a JSON -// blob. Lets the e2e tests parse the one or two integer fields they -// care about (user id, mostly) without pulling in encoding/json. +// extractUserID is a tiny scanner for `"id":` inside a JSON +// blob. It reads the user ID returned by the API without adding a JSON +// decoding dependency to callers that only need that field. // // Lives on env.go (not in a _test.go) so non-test helper code like // CreateUser can use it. -func extractIntField(body, key string) string { - needle := `"` + key + `":` +func extractUserID(body string) string { + const needle = `"id":` i := strings.Index(body, needle) if i < 0 { return "" diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index 6fae31437..d970c5717 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -130,7 +130,7 @@ func TestPanel_OfflineRecovery(t *testing.T) { f.request(admin, http.MethodPost, "/roles", map[string]any{"name": roleName, "description": "Recovery fixture read only", "permissions": []string{"servers:read"}}, http.StatusCreated) userPassword := recoveryRandom(t) created := f.request(admin, http.MethodPost, "/users", map[string]any{"username": "reader", "password": userPassword, "role": roleName}, http.StatusOK) - userID := extractIntField(string(created), "id") + userID := extractUserID(string(created)) if userID == "" { t.Fatal("created reader has no ID") } From fa7a28502d917dcbdea325fb85ec68a77fb39f13 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 17:15:52 -0400 Subject: [PATCH 4/8] fix(agent): bound NuclearOption request writes Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption.go | 28 ++++++----- .../rcon/nuclearoption_deadline_test.go | 46 +++++++++++++++++++ agent/internal/rcon/nuclearoption_test.go | 2 +- agent/specs.md | 2 +- 4 files changed, 64 insertions(+), 14 deletions(-) create mode 100644 agent/internal/rcon/nuclearoption_deadline_test.go diff --git a/agent/internal/rcon/nuclearoption.go b/agent/internal/rcon/nuclearoption.go index 7b8f2b1f2..001135354 100644 --- a/agent/internal/rcon/nuclearoption.go +++ b/agent/internal/rcon/nuclearoption.go @@ -112,14 +112,21 @@ func (c *NuclearOption) Close() error { // If the response has no body (body length = 0), an empty string is returned. // Non-2000 status codes produce an error that includes the status code and, // if present, the error detail from the response body. +// One deadline bounds the complete exchange, including dialing and request writes. func (c *NuclearOption) Exec(cmd string) (string, error) { c.mu.Lock() defer c.mu.Unlock() + d := c.execDeadline + if d <= 0 { + d = defaultNuclearOptionExecDeadline + } + exchangeDeadline := time.Now().Add(d) + // Ensure connection is open. if c.conn == nil { d := net.Dialer{Timeout: c.dialTimeout} - ctx, cancel := context.WithTimeout(context.Background(), c.dialTimeout) + ctx, cancel := context.WithDeadline(context.Background(), exchangeDeadline) defer cancel() conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(c.host, fmt.Sprint(c.port))) if err != nil { @@ -144,6 +151,14 @@ func (c *NuclearOption) Exec(cmd string) (string, error) { return "", fmt.Errorf("nuclearoption rcon: marshal request: %w", err) } + // Bound writes as well as reads. Arm this before sending anything: a + // server that stops reading must not retain the shared client forever. + if err := c.conn.SetDeadline(exchangeDeadline); err != nil { + _ = c.conn.Close() + c.conn = nil + return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err) + } + // Write request: 4-byte LE length + body. reqLen := len(reqBody) if reqLen < 0 || reqLen > math.MaxUint32 { @@ -162,17 +177,6 @@ func (c *NuclearOption) Exec(cmd string) (string, error) { return "", fmt.Errorf("nuclearoption rcon: write body: %w", err) } - // Set deadline for the entire response read. - d := c.execDeadline - if d <= 0 { - d = defaultNuclearOptionExecDeadline - } - if err := c.conn.SetDeadline(time.Now().Add(d)); err != nil { - _ = c.conn.Close() - c.conn = nil - return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err) - } - // Read response: 4-byte status + 4-byte body length + body. statusBuf := make([]byte, 4) if _, err := io.ReadFull(c.conn, statusBuf); err != nil { diff --git a/agent/internal/rcon/nuclearoption_deadline_test.go b/agent/internal/rcon/nuclearoption_deadline_test.go new file mode 100644 index 000000000..9cae0e073 --- /dev/null +++ b/agent/internal/rcon/nuclearoption_deadline_test.go @@ -0,0 +1,46 @@ +package rcon + +import ( + "io" + "net" + "strings" + "testing" + "time" +) + +func TestNuclearOptionDeadlineIncludesRequestWrites(t *testing.T) { + t.Parallel() + for _, readHeader := range []bool{false, true} { + stage := "write length" + if readHeader { + stage = "write body" + } + t.Run(stage, func(t *testing.T) { + t.Parallel() + local, remote := net.Pipe() + defer local.Close() + defer remote.Close() + client := NewNuclearOption("ignored", 1, nil) + client.conn = local + client.execDeadline = 100 * time.Millisecond + if readHeader { + go func() { _, _ = io.CopyN(io.Discard, remote, 4) }() + } + done := make(chan error, 1) + go func() { _, err := client.Exec("get-server-id"); done <- err }() + select { + case err := <-done: + if err == nil || !isTimeout(err) || !strings.Contains(err.Error(), stage) { + t.Fatalf("stalled %s must time out: %v", stage, err) + } + case <-time.After(2 * time.Second): + _ = remote.Close() + <-done + t.Fatalf("stalled %s exceeded the command budget", stage) + } + if client.conn != nil { + t.Fatal("timed-out partial request must discard the connection") + } + }) + } +} diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 248c6a7fb..cb46a4e21 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -740,7 +740,7 @@ func TestNuclearOptionWriteBodyError(t *testing.T) { } // TestNuclearOptionSetDeadlineError verifies that a failure setting the -// response read deadline (after the request was fully written) is reported +// exchange deadline (before sending the request) is reported // and the connection is reset. func TestNuclearOptionSetDeadlineError(t *testing.T) { local, remote := net.Pipe() diff --git a/agent/specs.md b/agent/specs.md index af1f57529..05d4bcbb9 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -244,7 +244,7 @@ The agent and operator modules use `k8s.io/apimachinery`/`k8s.io/client-go` v0.3 - `battleye`: UDP-based CRC32 packet framing. - `satisfactory`: HTTPS JSON function-call API with session token. - `palworld`: HTTP Basic REST admin API. - - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. + - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. One 30-second deadline bounds dialing, request writes, and response reads; responses are capped at 10 MiB. A timed-out partial exchange closes the connection before the next command. - `rest`: Generic HTTP/JSON REST console client with lazy auth (`PassFn`), 1 MiB bounded body read, configurable timeouts, auth failure cooldown, and loopback TLS protection. Provides adapters for FiveM (txAdmin: `POST /fxserver/commands`, bearer/X-TxAdmin-Token, `{"action":"console","parameter":""}`), Farming Simulator 25 (`POST /api/console`, Basic auth, `{"command":""}`), and generic REST (`POST /api/command`). - `cli`: Container stdin/PTY / local execution client (Option A in OPEN-DECISIONS.md). Drives commands via named FIFO pipe (`GAMEPLANE_CLI_PIPE`) or local process shell execution without requiring remote TCP networking or password secrets, while web console continues to attach via pod-attach PTY. From 56ba5910db4f77915f729d0f30f8c0e8f2135cf9 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 22:46:23 -0400 Subject: [PATCH 5/8] fix(e2e): wait for recovery cluster registration Signed-off-by: Baylem --- test/e2e/panel_recovery_e2e_test.go | 15 ++++++++------- test/e2e/specs.md | 4 +++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index d970c5717..fcd2965d1 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -22,6 +22,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -134,11 +135,14 @@ func TestPanel_OfflineRecovery(t *testing.T) { if userID == "" { t.Fatal("created reader has no ID") } + // Registration persists before the asynchronous registry watcher loads its + // client. Prove dispatch works before assigning a binding to that cluster. + gameName, gameUID := recoveryGame(t, suffix) + serversPath := "/servers?cluster=" + clusterID + "&namespace=gameplane-games" + f.waitServer(admin, serversPath, gameUID) f.request(admin, http.MethodPost, "/users/"+userID+"/bindings", map[string]any{"roleName": roleName, "cluster": clusterID, "namespace": "gameplane-games"}, http.StatusCreated) reader := f.login("initial", "reader", userPassword) defer reader.Close() - gameName, gameUID := recoveryGame(t, suffix) - serversPath := "/servers?cluster=" + clusterID + "&namespace=gameplane-games" f.waitServer(reader, serversPath, gameUID) shareBody := f.request(admin, http.MethodPost, "/servers/"+gameName+":shares?cluster="+clusterID+"&namespace=gameplane-games", map[string]any{"canStart": false, "neverExpires": true}, http.StatusOK) var share struct { @@ -462,14 +466,11 @@ func recoveryGame(t *testing.T, suffix string) (string, string) { t.Helper() name := "recovery-game-" + suffix applyBusyboxTemplate(t, name) - tmpl, err := envInstance.Dyn.Resource(gameTemplateGVR).Get(t.Context(), name, metav1.GetOptions{}) + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"command": []string{"sh", "-c", "echo retained > /data/recovery-marker; n=0; while true; do n=$((n+1)); echo $n > /data/recovery-counter.next; mv /data/recovery-counter.next /data/recovery-counter; sleep 1; done"}}}) if err != nil { t.Fatal(err) } - if err := unstructured.SetNestedStringSlice(tmpl.Object, []string{"sh", "-c", "echo retained > /data/recovery-marker; n=0; while true; do n=$((n+1)); echo $n > /data/recovery-counter.next; mv /data/recovery-counter.next /data/recovery-counter; sleep 1; done"}, "spec", "command"); err != nil { - t.Fatal(err) - } - if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Update(t.Context(), tmpl, metav1.UpdateOptions{}); err != nil { + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(t.Context(), name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { t.Fatal(err) } applyBusyboxGameServer(t, "gameplane-games", name, name) diff --git a/test/e2e/specs.md b/test/e2e/specs.md index aaf952839..091552018 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -31,7 +31,9 @@ audit evidence and an unchanged continuously running game Pod/container. The read-only original archive is never changed by invalidation. Actual fixture recovery-point age/RTO and acceptance evidence are uploaded as JSON, with no key, credential, cookie, share token or raw SQL rows. It does not stop the installed -panel/operator or claim PostgreSQL PITR coverage. See `docs/panel-recovery.md`. +panel/operator or claim PostgreSQL PITR coverage. Initial scoped grants wait for +the asynchronous cluster registry to load the connection and permit dispatch. +See `docs/panel-recovery.md`. ### Remote gateway parity acceptance From 19b7932ea1be69d9d1bd627134c29430c4054146 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:25:54 -0400 Subject: [PATCH 6/8] test: authorize recovery fixture cluster access Signed-off-by: Baylem --- test/e2e/panel_recovery_e2e_test.go | 71 +++++++++++++++++++++++++---- test/e2e/specs.md | 6 ++- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/test/e2e/panel_recovery_e2e_test.go b/test/e2e/panel_recovery_e2e_test.go index fcd2965d1..4f9156663 100644 --- a/test/e2e/panel_recovery_e2e_test.go +++ b/test/e2e/panel_recovery_e2e_test.go @@ -119,14 +119,30 @@ func TestPanel_OfflineRecovery(t *testing.T) { return result["status"] == float64(http.StatusOK), "waiting for recovery HTTP monitor positive control" }) report["http_monitor_positive_control"] = true - admin := f.login("initial", "admin", password) - defer admin.Close() + bootstrapAdmin := f.login("initial", "admin", password) + defer bootstrapAdmin.Close() + adminID := extractUserID(string(f.request(bootstrapAdmin, http.MethodGet, "/users/me", nil, http.StatusOK))) + if adminID == "" { + t.Fatal("bootstrap admin has no ID") + } // Register the existing kind cluster using stored, encrypted credentials. // The panel has no service-account token, so successful dispatch depends on // decrypting this registration after recovery. clusterID := "fixture-cluster" - f.request(admin, http.MethodPost, "/clusters", map[string]any{"name": clusterID, "displayName": "Recovery fixture", "kubeconfig": recoveryKubeconfig(t)}, http.StatusCreated) + f.request(bootstrapAdmin, http.MethodPost, "/clusters", map[string]any{"name": clusterID, "displayName": "Recovery fixture", "kubeconfig": recoveryKubeconfig(t)}, http.StatusCreated) + // Bootstrap grants control-plane administration through the local cluster, + // but remote server access requires an explicit target namespace binding. + // Discovery is safe before that grant; Healthy also proves the asynchronous + // registry has loaded the client needed to accept the binding. + f.waitClusterHealthy(bootstrapAdmin, clusterID) + f.request(bootstrapAdmin, http.MethodPost, "/users/"+adminID+"/bindings", map[string]any{"roleName": "admin", "cluster": clusterID, "namespace": "gameplane-games"}, http.StatusCreated) + // A binding change revokes the affected user's sessions. The new cookie, + // acquired after the grant, must remain live until the backup is taken. + f.request(bootstrapAdmin, http.MethodGet, "/users/me", nil, http.StatusUnauthorized) + bootstrapAdmin.Close() + admin := f.login("initial", "admin", password) + defer admin.Close() roleName := "recovery-reader" f.request(admin, http.MethodPost, "/roles", map[string]any{"name": roleName, "description": "Recovery fixture read only", "permissions": []string{"servers:read"}}, http.StatusCreated) userPassword := recoveryRandom(t) @@ -135,8 +151,8 @@ func TestPanel_OfflineRecovery(t *testing.T) { if userID == "" { t.Fatal("created reader has no ID") } - // Registration persists before the asynchronous registry watcher loads its - // client. Prove dispatch works before assigning a binding to that cluster. + // Prove credential-backed dispatch with the scoped administrator, then + // repeat it using only the reader's explicit target namespace grant. gameName, gameUID := recoveryGame(t, suffix) serversPath := "/servers?cluster=" + clusterID + "&namespace=gameplane-games" f.waitServer(admin, serversPath, gameUID) @@ -153,6 +169,9 @@ func TestPanel_OfflineRecovery(t *testing.T) { } public := &APIClient{BaseURL: admin.BaseURL, HTTP: &http.Client{Timeout: 10 * time.Second}} f.request(public, http.MethodGet, "/shares/"+share.Token, nil, http.StatusOK) + for _, original := range []*APIClient{admin, reader} { + f.request(original, http.MethodGet, "/users/me", nil, http.StatusOK) + } beforeGame := recoveryGameIdentity(t, gameName) beforeCounter := recoveryCounter(t, gameName) report["game_before"] = beforeGame @@ -253,9 +272,12 @@ func TestPanel_OfflineRecovery(t *testing.T) { f.waitPodReady("restored") recovered := f.login("restored", "admin", password) defer recovered.Close() - // Reuse the original cookie jar against the recovered endpoint, without login. - stale := &APIClient{BaseURL: recovered.BaseURL, HTTP: reader.HTTP, CSRF: reader.CSRF} - f.request(stale, http.MethodGet, "/auth/me", nil, http.StatusUnauthorized) + // Reuse both pre-backup cookie jars against the recovered endpoint, without + // login. Each was accepted immediately before the original API stopped. + for _, original := range []*APIClient{admin, reader} { + stale := &APIClient{BaseURL: recovered.BaseURL, HTTP: original.HTTP, CSRF: original.CSRF} + f.request(stale, http.MethodGet, "/users/me", nil, http.StatusUnauthorized) + } public.BaseURL = recovered.BaseURL f.request(public, http.MethodGet, "/shares/"+share.Token, nil, http.StatusNotFound) recoveredReader := f.login("restored", "reader", userPassword) @@ -436,16 +458,45 @@ func recoveryKubeconfig(t *testing.T) string { return string(data) } +func (f *recoveryFixture) waitClusterHealthy(c *APIClient, clusterID string) { + f.t.Helper() + envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { + resp, data, err := c.DoContext(f.t.Context(), http.MethodGet, "/clusters", nil) + if err != nil { + return false, fmt.Sprintf("cluster discovery request failed (error type=%T; details withheld)", err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != http.StatusOK { + return false, fmt.Sprintf("cluster discovery: HTTP status=%d (body withheld)", resp.StatusCode) + } + var discovery struct { + Items []struct { + Name string `json:"name"` + Phase string `json:"phase"` + } `json:"items"` + } + if json.Unmarshal(data, &discovery) != nil { + return false, "invalid cluster discovery response (body withheld)" + } + for _, item := range discovery.Items { + if item.Name == clusterID && item.Phase == "Healthy" { + return true, "" + } + } + return false, "registered cluster connection is not Healthy yet" + }) +} + func (f *recoveryFixture) waitServer(c *APIClient, path, uid string) { f.t.Helper() envInstance.Eventually(f.t, 90*time.Second, func() (bool, string) { resp, data, err := c.DoContext(f.t.Context(), http.MethodGet, path, nil) if err != nil { - return false, "waiting for credential-backed cluster dispatch" + return false, fmt.Sprintf("credential-backed cluster dispatch failed (error type=%T; details withheld)", err) } defer func() { _ = resp.Body.Close() }() if resp.StatusCode != http.StatusOK { - return false, "waiting for credential-backed cluster dispatch" + return false, fmt.Sprintf("credential-backed cluster dispatch: HTTP status=%d (path/body withheld)", resp.StatusCode) } var list struct { Items []unstructured.Unstructured `json:"items"` diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 091552018..a514b2303 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -32,7 +32,11 @@ read-only original archive is never changed by invalidation. Actual fixture recovery-point age/RTO and acceptance evidence are uploaded as JSON, with no key, credential, cookie, share token or raw SQL rows. It does not stop the installed panel/operator or claim PostgreSQL PITR coverage. Initial scoped grants wait for -the asynchronous cluster registry to load the connection and permit dispatch. +Healthy cluster discovery so the asynchronous registry has loaded the connection. +The locally bootstrapped administrator receives an explicit fixture-cluster/gameplane-games +grant and logs in again after that change invalidates its session. Both administrator +and reader cookies are accepted before backup and rejected through `/users/me` after +offline recovery; remote server UID and share checks retain their scoped authorization. See `docs/panel-recovery.md`. ### Remote gateway parity acceptance From a5c6d5352df4a6fd629f355d1dfc3bb61a79e4cc Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:19:17 -0400 Subject: [PATCH 7/8] test: synchronize closed RCON response fixtures Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption_test.go | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index cb46a4e21..ccde83c53 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1275,18 +1275,47 @@ type pipeFrameCase struct { forbidErr string // substring the error must NOT contain (e.g. a detail suffix) } +type responseDeadlineConn struct { + net.Conn + deadlineSet chan<- struct{} +} + +func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error { + if err := c.Conn.SetDeadline(deadline); err != nil { + return err + } + select { + case c.deadlineSet <- struct{}{}: + default: + } + return nil +} + // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe // and checks the body or the error. func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + deadlineSet := make(chan struct{}, 1) client := newPipeClient(t, func(remote net.Conn) { if _, _, err := readNuclearOptionRequest(remote); err != nil { return } + if len(tt.frame) == 0 { + // Exec sets its response deadline after sending the request. + // Closing first can fail SetDeadline instead of the status read. + select { + case <-deadlineSet: + case <-t.Context().Done(): + return + } + } writeFrameSegments(remote, tt.frame, tt.oneByte) }) + if len(tt.frame) == 0 { + client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet} + } body, err := client.Exec("test-command") if tt.wantErr != "" { From 2e8989ee6dd8c868d28cc24beeaa30c6deddfeea Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:26:24 -0400 Subject: [PATCH 8/8] test: clarify closed response deadline synchronization Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index ccde83c53..d03e76514 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1303,8 +1303,8 @@ func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { return } if len(tt.frame) == 0 { - // Exec sets its response deadline after sending the request. - // Closing first can fail SetDeadline instead of the status read. + // Wait until the deadline is installed before closing so the + // intended error comes from the status read, not SetDeadline. select { case <-deadlineSet: case <-t.Context().Done():