diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 26d9ee778..bb40e2dca 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -439,7 +439,8 @@ jobs: needs: [changes] if: (needs.changes.outputs.images == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'build-images') runs-on: ubuntu-latest - timeout-minutes: 15 + # Include all image builds, cache exports, artifact upload and cleanup. + timeout-minutes: 30 permissions: contents: read steps: @@ -1232,6 +1233,8 @@ jobs: run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders run: python3 charts/tests/check_standalone.py + - name: workload security chart contract + run: python3 charts/tests/check_workload_security.py - name: sanity-check rendered manifests run: | grep -q 'kind: Deployment' /tmp/rendered.yaml @@ -1793,7 +1796,7 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads] arch: [amd64] include: - bucket: operator @@ -1829,6 +1832,10 @@ jobs: parallel: 1 test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: restricted-workloads + parallel: 1 + test_timeout: 25m # Includes real projected-token rotation and node capture cleanup. + job_timeout: 60 env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e @@ -1900,7 +1907,7 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # Mirrors e2e-go's steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads] arch: [arm64] include: - bucket: operator @@ -1936,6 +1943,10 @@ jobs: parallel: 1 test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite. + - bucket: restricted-workloads + parallel: 1 + test_timeout: 25m # Includes real projected-token rotation and node capture cleanup. + job_timeout: 60 env: CLUSTER: gameplane-e2e CTX: kind-gameplane-e2e @@ -2660,8 +2671,8 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '), - 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), 'e2e-upgrade': (n) => n.startsWith('e2e upgrade /') && n.includes('/ amd64 '), @@ -2813,7 +2824,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 248c6a7fb..9061d2c94 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1275,18 +1275,47 @@ type pipeFrameCase struct { forbidErr string // substring the error must NOT contain (e.g. a detail suffix) } +type responseDeadlineConn struct { + net.Conn + deadlineSet chan<- struct{} +} + +func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error { + if err := c.Conn.SetDeadline(deadline); err != nil { + return err + } + select { + case c.deadlineSet <- struct{}{}: + default: + } + return nil +} + // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe // and checks the body or the error. func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + deadlineSet := make(chan struct{}, 1) client := newPipeClient(t, func(remote net.Conn) { if _, _, err := readNuclearOptionRequest(remote); err != nil { return } + if len(tt.frame) == 0 { + // Exec sets its response deadline after sending the request. + // Closing first can fail SetDeadline instead of the status read. + select { + case <-deadlineSet: + case <-t.Context().Done(): + return + } + } writeFrameSegments(remote, tt.frame, tt.oneByte) }) + if len(tt.frame) == 0 { + client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet} + } body, err := client.Exec("test-command") if tt.wantErr != "" { diff --git a/agent/specs.md b/agent/specs.md index af1f57529..027bcbfa9 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -5,6 +5,16 @@ ## Purpose +### Kubernetes token projection (#714) + +Game Pods disable automatic credentials. The agent alone mounts a rotating +projected token plus kube-root CA and namespace at client-go's standard path; +`rest.InClusterConfig` preserves `BearerTokenFile` refresh. No token is mounted +into game/config-init. Shared PID remains intentional for process metrics and +is residual in-Pod trust; mount isolation does not make the agent safe from +all processes in its Pod. CI exercises live projection rotation and a heartbeat +after the original short token expires. + The agent is a per-pod HTTP/HTTPS sidecar that runs inside every game pod to expose the game server's operations to the control plane and dashboard. It translates dashboard requests into game-protocol actions (RCON, file I/O, container logs, player queries) and reports liveness, resource usage, and player metrics back to the operator via Kubernetes API patches, decoupling game-specific behavior from the cluster control plane. ## Responsibilities diff --git a/api/internal/gateway/capture.go b/api/internal/gateway/capture.go index 9ead2ed3e..7ecf476d6 100644 --- a/api/internal/gateway/capture.go +++ b/api/internal/gateway/capture.go @@ -52,6 +52,12 @@ func (h *handler) serveCapture(w http.ResponseWriter, req *http.Request, target } defer transport.CloseIdleConnections() host := target.Name + "-agent." + target.Namespace + ".svc.cluster.local:9091" + if name, namespace, node, err := kube.NodeCaptureService(nc); err != nil { + http.NotFound(w, req) + return + } else if node { + host = name + "-agent." + namespace + ".svc.cluster.local:9091" + } proxy := &httputil.ReverseProxy{ Transport: transport, Rewrite: func(pr *httputil.ProxyRequest) { diff --git a/api/internal/handlers/capture.go b/api/internal/handlers/capture.go index 1f3171bcf..36583adf2 100644 --- a/api/internal/handlers/capture.go +++ b/api/internal/handlers/capture.go @@ -72,6 +72,8 @@ func MountCapture(r chi.Router, reg *kube.Registry, auditor *audit.Auditor, cfg } r.Post("/servers/{name}:capture-enable", h.captureEnable) r.Post("/servers/{name}:capture-disable", h.captureDisable) + r.Post("/servers/{name}:capture-node-enable", h.captureNodeEnable) + r.Post("/servers/{name}:capture-node-disable", h.captureNodeDisable) r.Post("/servers/{name}:capture-start", h.captureStart) r.Post("/servers/{name}:capture-stop", h.captureStop) r.Get("/servers/{name}:captures", h.captureList) @@ -957,7 +959,9 @@ func (h *captureHandler) captureDelete(w http.ResponseWriter, req *http.Request) } deleteErr = deleteCaptureWithUID(req.Context(), k, nc) } else { - h.deleteSidecarCaptureFile(req.Context(), gs.Name, gs.Namespace, captureID) + if nc.Annotations[kube.NodeCaptureNamespaceAnnotation] == "" { + h.deleteSidecarCaptureFile(req.Context(), gs.Name, gs.Namespace, captureID) + } deleteErr = k.DeleteNetworkCapture(req.Context(), ns, captureID) } @@ -1099,7 +1103,11 @@ func (h *captureHandler) captureDownload(w http.ResponseWriter, req *http.Reques if isRemoteCluster(req) { status = h.remoteCaptureDownload(w, req, gs, nc) } else { - status = h.proxyFileDownload(w, req, host, captureID) + if _, _, node, _ := kube.NodeCaptureService(nc); node { + status = h.proxyNodeCaptureDownload(w, req, gs, nc) + } else { + status = h.proxyFileDownload(w, req, host, captureID) + } } if status < http.StatusOK || (status >= http.StatusMultipleChoices && status != http.StatusNotModified) { // A disconnect must not also erase the corrective audit outcome. diff --git a/api/internal/handlers/capture_node.go b/api/internal/handlers/capture_node.go new file mode 100644 index 000000000..6025037ab --- /dev/null +++ b/api/internal/handlers/capture_node.go @@ -0,0 +1,111 @@ +package handlers + +import ( + "encoding/json" + "errors" + "net/http" + "net/url" + + "github.com/GameplanePanel/gameplane/api/internal/auth" + "github.com/GameplanePanel/gameplane/api/internal/httperr" + "github.com/GameplanePanel/gameplane/api/internal/kube" + "github.com/GameplanePanel/gameplane/api/internal/scope" + "github.com/go-chi/chi/v5" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +func (h *captureHandler) captureNodeEnable(w http.ResponseWriter, r *http.Request) { + h.captureNodeOptIn(w, r, true) +} +func (h *captureHandler) captureNodeDisable(w http.ResponseWriter, r *http.Request) { + h.captureNodeOptIn(w, r, false) +} + +func (h *captureHandler) captureNodeOptIn(w http.ResponseWriter, req *http.Request, enabled bool) { + k, ok := resolveCluster(w, req, h.reg) + if !ok { + return + } + ns, ok := resolveNS(w, req) + if !ok { + return + } + name := chi.URLParam(req, "name") + u := auth.UserFromContext(req.Context()) + if u == nil || (u.Role != "admin" && !u.Can("*", true, scope.RequestedCluster(req), "*")) { + if h.auditWriteOrFail(w, req, req.Method, req.URL.Path, name, "node_diagnostics_admin_required", http.StatusForbidden) { + httperr.WriteCode(w, req, http.StatusForbidden, errors.New("node diagnostic exception requires administrator privileges")) + } + return + } + gs, err := k.GetServer(req.Context(), ns, name) + if err != nil { + httperr.Write(w, req, err) + return + } + if gs.GetDeletionTimestamp() != nil { + httperr.WriteCode(w, req, http.StatusConflict, errors.New("server is terminating")) + return + } + reason := "node_diagnostics_opt_out" + if enabled { + reason = "node_diagnostics_opt_in_host_privileges" + } + // Record intent synchronously before granting host capabilities. A failed + // audit write prevents the exception; Kubernetes status remains separate. + if !h.auditWriteOrFail(w, req, req.Method, req.URL.Path, name+":"+string(gs.GetUID()), reason, http.StatusOK) { + return + } + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"uid": string(gs.GetUID()), "resourceVersion": gs.GetResourceVersion()}, "spec": map[string]any{"capture": map[string]any{"nodeDiagnostic": enabled}}}) + if err != nil { + httperr.Write(w, req, err) + return + } + _, err = k.Dynamic.Resource(kube.GVRs["servers"]).Namespace(ns).Patch(req.Context(), name, types.MergePatchType, patch, metav1.PatchOptions{}) + if err != nil { + if h.auditWriteOrFail(w, req, req.Method, req.URL.Path, name, "node_diagnostics_update_failed", http.StatusInternalServerError) { + httperr.Write(w, req, err) + } + return + } + if !enabled { + if err := h.stopActiveCaptures(req.Context(), k, ns, name); err != nil { + httperr.Write(w, req, err) + return + } + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]bool{"nodeDiagnostic": enabled}) +} + +func (h *captureHandler) proxyNodeCaptureDownload(w http.ResponseWriter, req *http.Request, gs *kube.GameServer, nc *kube.NetworkCapture) int { + bound, err := captureTarget(scope.RequestedCluster(req), gs, nc) + if err != nil { + http.NotFound(w, req) + return http.StatusNotFound + } + name, ns, _, err := kube.NodeCaptureService(nc) + if err != nil { + httperr.Write(w, req, err) + return http.StatusInternalServerError + } + host, ok := newValidatedHost(name, ns) + if !ok || h.tlsClient == nil { + http.Error(w, "capture artifacts unavailable", http.StatusServiceUnavailable) + return http.StatusServiceUnavailable + } + u := url.URL{Scheme: "https", Host: string(host), Path: "/v1/targets/" + bound.UID + "/captures/" + bound.Capture + "/uids/" + bound.CaptureUID + "/file"} + up, err := http.NewRequestWithContext(req.Context(), http.MethodGet, u.String(), nil) + if err != nil { + httperr.Write(w, req, err) + return http.StatusInternalServerError + } + copyProxyHeaders(up.Header, req.Header) + resp, err := h.tlsClient.Do(up) + if err != nil { + return writeUpstreamError(w, req, err) + } + defer func() { _ = resp.Body.Close() }() + return streamCaptureResponse(w, req, resp, nc.Name) +} diff --git a/api/internal/handlers/capture_node_test.go b/api/internal/handlers/capture_node_test.go new file mode 100644 index 000000000..28063868c --- /dev/null +++ b/api/internal/handlers/capture_node_test.go @@ -0,0 +1,188 @@ +package handlers + +import ( + "net/http" + "net/url" + "strings" + "testing" + + "github.com/GameplanePanel/gameplane/api/internal/audit" + "github.com/GameplanePanel/gameplane/api/internal/auth" + "github.com/GameplanePanel/gameplane/api/internal/kube" + "github.com/GameplanePanel/gameplane/api/internal/scope" + "github.com/go-chi/chi/v5" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestNodeCaptureExceptionRequiresAuditedAdministrator(t *testing.T) { + for _, admin := range []bool{false, true} { + t.Run(map[bool]string{false: "capture-manager", true: "administrator"}[admin], func(t *testing.T) { + gs := newCaptureServerObj("game", true) + gs.SetUID("server-uid") + k := fakeCaptureClient(gs) + store := newTestStore(t) + router := mountCaptureTestRouter(k, CaptureConfig{FeatureEnabled: true}, audit.New(store)) + user := &auth.User{Username: "manager", Role: "custom", Perms: map[string]map[string]map[string]struct{}{"local": {"gameplane-games": {"captures:manage": {}}}}} + want := http.StatusForbidden + if admin { + user = testAdminUser() + want = http.StatusOK + } + rr := doWithUser(t, router, http.MethodPost, "/servers/game:capture-node-enable", nil, user) + if rr.Code != want { + t.Fatalf("status %d %s", rr.Code, rr.Body) + } + live, err := k.GetServer(t.Context(), scope.DefaultNamespace, "game") + if err != nil { + t.Fatal(err) + } + enabled, _, _ := unstructured.NestedBool(live.Object, "spec", "capture", "nodeDiagnostic") + if enabled != admin { + t.Fatalf("opt-in=%t", enabled) + } + if admin { + var count int + if err := store.DB.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM audit_events WHERE reason = ?", "node_diagnostics_opt_in_host_privileges").Scan(&count); err != nil || count != 1 { + t.Fatalf("missing explicit audit event count=%d err=%v", count, err) + } + } + }) + } +} + +func TestNodeExceptionRequiresAdministratorOnSelectedCluster(t *testing.T) { + for _, test := range []struct { + name, target, grantCluster, grantNamespace string + allowed bool + }{ + {"other-cluster", "cluster-b", "cluster-a", "*", false}, + {"namespace-only", "cluster-b", "cluster-b", "gameplane-games", false}, + {"selected-cluster", "cluster-b", "cluster-b", "*", true}, + {"default-local", "", "local", "*", true}, + {"explicit-local", "local", "local", "*", true}, + {"normalized-local", " local ", "local", "*", true}, + {"other-cluster-local-target", "local", "cluster-a", "*", false}, + } { + t.Run(test.name, func(t *testing.T) { + gs := newCaptureServerObj("game", true) + gs.SetUID("server-uid") + k := fakeCaptureClient(gs) + reg := kube.NewRegistry(scope.DefaultCluster) + reg.Set("local", k) + reg.Set("cluster-b", k) + router := chi.NewRouter() + MountCapture(router, reg, audit.New(newTestStore(t)), CaptureConfig{FeatureEnabled: true}, "", "", "") + user := &auth.User{Username: "custom-admin", Role: "custom", Perms: map[string]map[string]map[string]struct{}{test.grantCluster: {test.grantNamespace: {"*": {}}}}} + target := strings.TrimSpace(test.target) + if target == "" { + target = "local" + } + if user.Perms[target] == nil { + user.Perms[target] = map[string]map[string]struct{}{} + } + if user.Perms[target][scope.DefaultNamespace] == nil { + user.Perms[target][scope.DefaultNamespace] = map[string]struct{}{} + } + user.Perms[target][scope.DefaultNamespace]["captures:manage"] = struct{}{} + path := "/servers/game:capture-node-enable" + if test.target != "" { + path += "?cluster=" + url.QueryEscape(test.target) + } + rr := doWithUser(t, router, http.MethodPost, path, nil, user) + want := http.StatusForbidden + if test.allowed { + want = http.StatusOK + } + if rr.Code != want { + t.Fatalf("status %d want %d: %s", rr.Code, want, rr.Body) + } + live, err := k.GetServer(t.Context(), scope.DefaultNamespace, "game") + if err != nil { + t.Fatal(err) + } + enabled, _, _ := unstructured.NestedBool(live.Object, "spec", "capture", "nodeDiagnostic") + if enabled != test.allowed { + t.Fatal("wrong scope granted host exception") + } + }) + } +} + +func TestNodeExceptionCannotBeGrantedByGenericEditOrClone(t *testing.T) { + src := newServerObj("gameplane-games", "alpha") + if err := unstructured.SetNestedField(src.Object, true, "spec", "capture", "nodeDiagnostic"); err != nil { + t.Fatal(err) + } + k := fakeKubeClient(src) + err := validateAndProtectGameServer(auth.WithUser(t.Context(), testAdminUser()), k, "local", "gameplane-games", "new", src, nil) + if err == nil || !strings.Contains(err.Error(), "capture-node-enable") { + t.Fatalf("generic create granted exception: %v", err) + } + rr := doWithUser(t, mountLifecycleRouter(k), http.MethodPost, "/servers/alpha:clone", map[string]any{"newName": "beta"}, testAdminUser()) + if rr.Code != http.StatusOK { + t.Fatalf("clone: %d %s", rr.Code, rr.Body) + } + clone, err := k.GetServer(t.Context(), "gameplane-games", "beta") + if err != nil { + t.Fatal(err) + } + if enabled, _, _ := unstructured.NestedBool(clone.Object, "spec", "capture", "nodeDiagnostic"); enabled { + t.Fatal("clone inherited node exception") + } +} + +func TestNodeExceptionGenericEditPreservesOmittedOptIn(t *testing.T) { + manager := &auth.User{Username: "manager", Role: "custom", Perms: map[string]map[string]map[string]struct{}{"local": {"gameplane-games": {"captures:manage": {}}}}} + for _, user := range []*auth.User{testAdminUser(), manager} { + for _, enabled := range []bool{false, true} { + live := newServerObj("gameplane-games", "alpha") + if err := unstructured.SetNestedField(live.Object, enabled, "spec", "capture", "nodeDiagnostic"); err != nil { + t.Fatal(err) + } + for _, omitCapture := range []bool{false, true} { + desired := live.DeepCopy() + unstructured.RemoveNestedField(desired.Object, "spec", "capture", "nodeDiagnostic") + if omitCapture { + unstructured.RemoveNestedField(desired.Object, "spec", "capture") + } + if err := validateAndProtectGameServer(auth.WithUser(t.Context(), user), fakeKubeClient(live), "local", "gameplane-games", "alpha", desired, live); err != nil { + t.Fatalf("omitted exception rejected for %s, enabled=%t: %v", user.Role, enabled, err) + } + got, _, _ := unstructured.NestedBool(desired.Object, "spec", "capture", "nodeDiagnostic") + if got != enabled { + t.Fatal("generic edit changed an omitted exception") + } + } + desired := live.DeepCopy() + if err := unstructured.SetNestedField(desired.Object, !enabled, "spec", "capture", "nodeDiagnostic"); err != nil { + t.Fatal(err) + } + if err := validateAndProtectGameServer(auth.WithUser(t.Context(), user), fakeKubeClient(live), "local", "gameplane-games", "alpha", desired, live); err == nil || !strings.Contains(err.Error(), "capture-node-enable") { + t.Fatalf("generic edit changed explicit exception for %s: %v", user.Role, err) + } + } + } +} + +func TestNodeCaptureExceptionAuditFailureDoesNotGrant(t *testing.T) { + gs := newCaptureServerObj("game", true) + gs.SetUID("server-uid") + k := fakeCaptureClient(gs) + store := newTestStore(t) + auditor := audit.New(store) + if err := store.Close(); err != nil { + t.Fatal(err) + } + rr := doWithUser(t, mountCaptureTestRouter(k, CaptureConfig{FeatureEnabled: true}, auditor), http.MethodPost, "/servers/game:capture-node-enable", nil, testAdminUser()) + if rr.Code != http.StatusInternalServerError { + t.Fatalf("status %d", rr.Code) + } + live, err := k.GetServer(t.Context(), scope.DefaultNamespace, "game") + if err != nil { + t.Fatal(err) + } + enabled, _, _ := unstructured.NestedBool(live.Object, "spec", "capture", "nodeDiagnostic") + if enabled { + t.Fatal("granted exception without audit") + } +} diff --git a/api/internal/handlers/lifecycle.go b/api/internal/handlers/lifecycle.go index 5d10e517a..ebe1fe9b2 100644 --- a/api/internal/handlers/lifecycle.go +++ b/api/internal/handlers/lifecycle.go @@ -229,6 +229,9 @@ func cloneHandler(reg *kube.Registry) http.HandlerFunc { clone.SetFinalizers(nil) clone.SetManagedFields(nil) clone.SetCreationTimestamp(metav1.Time{}) + // A node diagnostic exception belongs to one server identity. Cloning + // never grants host-level diagnostic privileges to the new server. + unstructured.RemoveNestedField(clone.Object, "spec", "capture", "nodeDiagnostic") stampOwner(clone, req) cl := req.URL.Query().Get("cluster") diff --git a/api/internal/handlers/resources.go b/api/internal/handlers/resources.go index 2e892ff5d..8c16dd7f0 100644 --- a/api/internal/handlers/resources.go +++ b/api/internal/handlers/resources.go @@ -581,6 +581,27 @@ func validateAndProtectGameServer( } // 2. Validate spec.serviceAccountName + // A host-capable diagnostic exception must be a separate, synchronously + // audited administrator action. Generic edits may preserve an existing + // opt-in, but cannot grant/revoke it or inherit it on a clone. + nodeDiagnostic, hasNodeDiagnostic, err := unstructured.NestedBool(desired.Object, "spec", "capture", "nodeDiagnostic") + if err != nil { + return errors.New("spec.capture.nodeDiagnostic must be a boolean") + } + liveNodeDiagnostic := false + if live != nil { + liveNodeDiagnostic, _, _ = unstructured.NestedBool(live.Object, "spec", "capture", "nodeDiagnostic") + if !hasNodeDiagnostic && liveNodeDiagnostic { + if err := unstructured.SetNestedField(desired.Object, true, "spec", "capture", "nodeDiagnostic"); err != nil { + return fmt.Errorf("preserve node diagnostic exception: %w", err) + } + nodeDiagnostic = true + } + } + if nodeDiagnostic != liveNodeDiagnostic { + return errors.New("change the node diagnostic exception through the administrator capture-node-enable/capture-node-disable action") + } + sa, hasSA, _ := unstructured.NestedString(desired.Object, "spec", "serviceAccountName") if !isAdmin { if live == nil { diff --git a/api/internal/kube/capture.go b/api/internal/kube/capture.go index 7fe33ba4d..9b6d04f26 100644 --- a/api/internal/kube/capture.go +++ b/api/internal/kube/capture.go @@ -100,7 +100,8 @@ type NetworkCapture struct { // CaptureConfiguration mirrors operator/api/v1alpha1.CaptureConfiguration // (only the fields the API needs to read). type CaptureConfiguration struct { - Enabled bool `json:"enabled,omitempty"` + Enabled bool `json:"enabled,omitempty"` + NodeDiagnostic bool `json:"nodeDiagnostic,omitempty"` // RetentionSeconds is the per-GameServer override of the cluster-default // retention window applied when a capture-start request omits its own // ttlSecondsAfterFinished. Still bounded by the cluster maximum, the same diff --git a/api/internal/kube/capture_node.go b/api/internal/kube/capture_node.go new file mode 100644 index 000000000..038fad49b --- /dev/null +++ b/api/internal/kube/capture_node.go @@ -0,0 +1,27 @@ +package kube + +import ( + "errors" + "regexp" + + "k8s.io/apimachinery/pkg/util/validation" +) + +// NodeCaptureNamespaceAnnotation is written only by the operator when it +// binds a NetworkCapture to a dedicated diagnostic artifact service. +const NodeCaptureNamespaceAnnotation = "gameplane.local/capture-node-namespace" + +var nodeCaptureUID = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) + +// NodeCaptureService derives a fixed in-cluster service from the immutable +// capture UID. An annotation can never supply a URL, arbitrary host or path. +func NodeCaptureService(nc *NetworkCapture) (name, namespace string, node bool, err error) { + namespace = nc.Annotations[NodeCaptureNamespaceAnnotation] + if namespace == "" { + return "", "", false, nil + } + if !nodeCaptureUID.MatchString(string(nc.UID)) || len(validation.IsDNS1123Label(namespace)) != 0 { + return "", "", true, errors.New("invalid diagnostic artifact identity") + } + return "capture-" + string(nc.UID), namespace, true, nil +} diff --git a/api/internal/kube/capture_node_test.go b/api/internal/kube/capture_node_test.go new file mode 100644 index 000000000..332c03c31 --- /dev/null +++ b/api/internal/kube/capture_node_test.go @@ -0,0 +1,26 @@ +package kube + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestNodeCaptureServiceDerivesOnlyBoundInternalName(t *testing.T) { + nc := &NetworkCapture{ObjectMeta: metav1.ObjectMeta{UID: "11111111-2222-3333-4444-555555555555", Annotations: map[string]string{NodeCaptureNamespaceAnnotation: "diagnostics"}}} + name, ns, node, err := NodeCaptureService(nc) + if err != nil || !node || ns != "diagnostics" || name != "capture-11111111-2222-3333-4444-555555555555" { + t.Fatalf("unexpected target: %s %s %t %v", name, ns, node, err) + } + for _, invalid := range []string{"https://host", "../games", "host:9091", "UPPER"} { + nc.Annotations[NodeCaptureNamespaceAnnotation] = invalid + if _, _, _, err := NodeCaptureService(nc); err == nil { + t.Fatalf("accepted namespace %q", invalid) + } + } + nc.Annotations[NodeCaptureNamespaceAnnotation] = "diagnostics" + nc.UID = "../../host" + if _, _, _, err := NodeCaptureService(nc); err == nil { + t.Fatal("accepted invalid capture UID") + } +} diff --git a/api/internal/rbac/rbac.go b/api/internal/rbac/rbac.go index 3d5563cfa..9cff2bd14 100644 --- a/api/internal/rbac/rbac.go +++ b/api/internal/rbac/rbac.go @@ -246,6 +246,8 @@ var rules = []rule{ // that moves these rules after the servers catch-all silently grants // capture access to the operator role, defeating FR-005/SC-005. {method: "POST", segment: "servers", suffix: ":capture-enable", perm: "captures:manage"}, + {method: "POST", segment: "servers", suffix: ":capture-node-enable", perm: "captures:manage"}, + {method: "POST", segment: "servers", suffix: ":capture-node-disable", perm: "captures:manage"}, {method: "POST", segment: "servers", suffix: ":capture-disable", perm: "captures:manage"}, {method: "POST", segment: "servers", suffix: ":capture-start", perm: "captures:manage"}, {method: "POST", segment: "servers", suffix: ":capture-stop", perm: "captures:manage"}, diff --git a/api/specs.md b/api/specs.md index 1dccc8987..ad20f6f22 100644 --- a/api/specs.md +++ b/api/specs.md @@ -5,6 +5,22 @@ ## Purpose +### Node diagnostic exception (#714) + +`POST /servers/{name}:capture-node-enable` and `:capture-node-disable` require +cluster-wide administrator authority in the selected cluster in addition to +capture-management routing; a namespace-only wildcard grant is insufficient. +Enable synchronously audits the server UID and +host-privilege intent before a UID/resourceVersion-fenced patch; audit failure +grants nothing. Generic create/update cannot change `capture.nodeDiagnostic`, +and clones clear it. Generic updates that omit the field preserve its live +value, including clients that omit the entire capture object. Explicit changes +still require the dedicated audited action. Disable requests cancellation of active captures. Other +capture operations retain capture permissions and existing scope/provenance +checks. A node capture's operator annotation and immutable capture UID derive +its diagnostic Service; local and gateway file downloads use UID-bound mTLS +routes. Capture deletion delegates node artifact cleanup to the CR finalizer. + The api module is the REST + WebSocket gateway for Gameplane, serving the web dashboard and external integrations. It exposes the Kubernetes game-server CRDs (GameServer, GameTemplate, Backup, BackupSchedule, Restore, Module, ModuleSource, Cluster) and operator-reconciled state through a type-safe HTTP surface with multi-cluster support, role-based access control, and audit logging. ## Responsibilities diff --git a/capture-sidecar/Dockerfile b/capture-sidecar/Dockerfile index 05ecbfc50..a97d12ba7 100644 --- a/capture-sidecar/Dockerfile +++ b/capture-sidecar/Dockerfile @@ -28,11 +28,14 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \ # only mechanism that survives a non-root execve. The final stage relies on this # grant and runs as nonroot, accepting allowPrivilegeEscalation: true to permit # file capabilities to take effect (they are ignored under no_new_privs). -RUN setcap cap_net_raw+ep /out/capture-sidecar +RUN cp /out/capture-sidecar /out/capture-tool && setcap cap_net_raw+ep /out/capture-sidecar FROM gcr.io/distroless/static:nonroot WORKDIR / COPY --from=build /out/capture-sidecar /capture-sidecar +# No file capabilities: the artifact server is Restricted; the node worker +# receives its explicit root capability set from its dedicated Job spec. +COPY --from=build /out/capture-tool /capture-tool USER 65532:65532 EXPOSE 9091 ENTRYPOINT ["/capture-sidecar"] diff --git a/capture-sidecar/cmd/main.go b/capture-sidecar/cmd/main.go index 337f157d9..c1219037c 100644 --- a/capture-sidecar/cmd/main.go +++ b/capture-sidecar/cmd/main.go @@ -12,6 +12,7 @@ package main import ( "context" + "encoding/json" "errors" "flag" "log/slog" @@ -24,6 +25,7 @@ import ( "github.com/GameplanePanel/gameplane/capture-sidecar/internal/auth" "github.com/GameplanePanel/gameplane/capture-sidecar/internal/httpserver" + "github.com/GameplanePanel/gameplane/capture-sidecar/internal/nodecapture" ) // Version is overridden at build time via -ldflags (see Dockerfile). @@ -68,6 +70,8 @@ func main() { keyFile := flag.String("tls-key", envOrDefault("TLS_KEY_FILE", "/etc/tls/tls.key"), "Server key file") caFile := flag.String("tls-client-ca", envOrDefault("TLS_CA_FILE", "/etc/tls/ca.crt"), "Client CA certificate file") captureDataDir := flag.String("capture-dir", "/tmp/captures", "Directory for capture files") + nodeWorker := flag.Bool("node-worker", false, "Run one operator-bound node diagnostic capture and exit") + artifactOnly := flag.Bool("artifacts-only", false, "Serve completed node capture artifacts without capture controls") // The operator sets CAPTURE_VOLUME_BUDGET_BYTES on the capture ephemeral // container (see buildCaptureEphemeralContainer in @@ -89,6 +93,18 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() + if *nodeWorker { + var cfg nodecapture.Config + if *artifactOnly || json.Unmarshal([]byte(os.Getenv("NODE_CAPTURE_CONFIG")), &cfg) != nil { + slog.Error("invalid node capture configuration") + os.Exit(1) + } + if err := nodecapture.Run(ctx, cfg, *captureDataDir); err != nil { + slog.Error("node capture failed", "err", err) + os.Exit(1) + } + return + } // Create capture server. ctx (cancelled on SIGTERM/SIGINT) is the parent // for every capture goroutine, so shutdown cancels in-flight captures @@ -111,6 +127,9 @@ func main() { // (see HandleHealthz's doc comment for why nothing actually calls this // route today; F-193). mux := captureServer.Routes(auth.Middleware) + if *artifactOnly { + mux = captureServer.ArtifactRoutes(auth.Middleware) + } server := &http.Server{ Addr: *listenAddr, diff --git a/capture-sidecar/go.mod b/capture-sidecar/go.mod index 3815530bf..2834a12db 100644 --- a/capture-sidecar/go.mod +++ b/capture-sidecar/go.mod @@ -8,4 +8,4 @@ require ( golang.org/x/net v0.61.0 ) -require golang.org/x/sys v0.49.0 // indirect +require golang.org/x/sys v0.49.0 diff --git a/capture-sidecar/internal/httpserver/artifacts.go b/capture-sidecar/internal/httpserver/artifacts.go new file mode 100644 index 000000000..8eeafa1b7 --- /dev/null +++ b/capture-sidecar/internal/httpserver/artifacts.go @@ -0,0 +1,57 @@ +package httpserver + +import ( + "encoding/json" + "io" + "net/http" + + "github.com/GameplanePanel/gameplane/capture-sidecar/internal/nodecapture" +) + +// ArtifactRoutes exposes retained node-worker results without any operation +// that can open a raw socket or start privileged work. The Pod is Restricted. +func (s *Server) ArtifactRoutes(mw func(http.Handler) http.Handler) *http.ServeMux { + mux := http.NewServeMux() + if mw == nil { + mw = func(h http.Handler) http.Handler { return h } + } + mux.HandleFunc("GET /healthz", HandleHealthz) + mux.Handle("GET /captures/{id}/status", mw(http.HandlerFunc(s.handleArtifactStatus))) + mux.Handle("GET /v1/targets/{serverUID}/captures/{id}/uids/{captureUID}/file", mw(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + result, ok := s.artifactResult(r.PathValue("id")) + if !ok || result.Status != "completed" { + http.NotFound(w, r) + return + } + s.HandleBoundDownload(w, r) + }))) + mux.Handle("DELETE /v1/targets/{serverUID}/captures/{id}/uids/{captureUID}/file", mw(http.HandlerFunc(s.HandleBoundDelete))) + return mux +} + +func (s *Server) artifactResult(id string) (nodecapture.Result, bool) { + var result nodecapture.Result + if !validCaptureID(id) { + return result, false + } + f, err := s.openCaptureRegular("capture-" + id + ".result.json") + if err != nil { + return result, false + } + defer func() { _ = f.Close() }() + raw, err := io.ReadAll(io.LimitReader(f, 4097)) + if err != nil || len(raw) > 4096 || json.Unmarshal(raw, &result) != nil || result.CaptureID != id || result.ServerUID != s.serverUID || !captureUIDPattern.MatchString(result.CaptureUID) { + return result, false + } + return result, true +} + +func (s *Server) handleArtifactStatus(w http.ResponseWriter, r *http.Request) { + result, ok := s.artifactResult(r.PathValue("id")) + if !ok || r.Header.Get("X-Gameplane-Server-UID") != result.ServerUID || r.Header.Get("X-Gameplane-Capture-UID") != result.CaptureUID { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(result) +} diff --git a/capture-sidecar/internal/httpserver/artifacts_test.go b/capture-sidecar/internal/httpserver/artifacts_test.go new file mode 100644 index 000000000..959f4d66d --- /dev/null +++ b/capture-sidecar/internal/httpserver/artifacts_test.go @@ -0,0 +1,31 @@ +package httpserver + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" +) + +func TestArtifactServerCannotStartCaptureAndBindsResult(t *testing.T) { + dir := t.TempDir() + s := NewServer(t.Context(), dir, 0, "server-uid") + routes := s.ArtifactRoutes(nil) + if err := os.WriteFile(filepath.Join(dir, "capture-cap-one.result.json"), []byte(`{"serverUID":"server-uid","captureUID":"capture-uid","captureId":"cap-one","status":"completed"}`), 0o600); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + method, path, uid string + want int + }{{http.MethodPost, "/captures/cap-one/start", "capture-uid", http.StatusNotFound}, {http.MethodGet, "/captures/cap-one/status", "other-uid", http.StatusNotFound}, {http.MethodGet, "/captures/cap-one/status", "capture-uid", http.StatusOK}} { + req := httptest.NewRequestWithContext(t.Context(), test.method, test.path, nil) + req.Header.Set("X-Gameplane-Server-UID", "server-uid") + req.Header.Set("X-Gameplane-Capture-UID", test.uid) + rr := httptest.NewRecorder() + routes.ServeHTTP(rr, req) + if rr.Code != test.want { + t.Fatalf("%s %s: %d %s", test.method, test.path, rr.Code, rr.Body) + } + } +} diff --git a/capture-sidecar/internal/nodecapture/worker.go b/capture-sidecar/internal/nodecapture/worker.go new file mode 100644 index 000000000..4a8e8acaf --- /dev/null +++ b/capture-sidecar/internal/nodecapture/worker.go @@ -0,0 +1,277 @@ +// Package nodecapture runs one bounded capture against an operator-selected +// containerd game process. It never accepts a host PID or filesystem path. +package nodecapture + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + "github.com/GameplanePanel/gameplane/capture-sidecar/internal/capture" + "golang.org/x/sys/unix" +) + +var identityPattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,128}$`) +var podPattern = regexp.MustCompile(`^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`) +var containerPattern = regexp.MustCompile(`^containerd://[0-9a-f]{64}$`) +var capturePattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,64}$`) + +// The separate artifact server reads finalized files through the dedicated PVC group. +const artifactMode os.FileMode = 0o640 + +// Config contains immutable identities copied from live Kubernetes objects. +type Config struct { + ServerUID string `json:"serverUID"` + CaptureUID string `json:"captureUID"` + CaptureID string `json:"captureID"` + PodUID string `json:"podUID"` + ContainerID string `json:"containerID"` + Node string `json:"node"` + Filter string `json:"filter"` + DurationSeconds int64 `json:"durationSeconds"` + MaxSizeBytes int64 `json:"maxSizeBytes"` +} + +// Validate rejects unsupported runtimes, unconstrained captures and paths. +func (c Config) Validate() error { + if !identityPattern.MatchString(c.ServerUID) || !identityPattern.MatchString(c.CaptureUID) || !capturePattern.MatchString(c.CaptureID) || !podPattern.MatchString(c.PodUID) || !containerPattern.MatchString(c.ContainerID) || c.Node == "" || c.Filter == "" || len(c.Filter) > 1024 || c.DurationSeconds < 1 || c.DurationSeconds > 3600 || c.MaxSizeBytes < 1 || c.MaxSizeBytes > 900*1024*1024 { + return errors.New("invalid node capture identity, runtime, filter or limits") + } + return nil +} + +// Result is written atomically only after the PCAP has been closed. +type Result struct { + ServerUID string `json:"serverUID"` + CaptureUID string `json:"captureUID"` + CaptureID string `json:"captureId"` + Status string `json:"status"` + PacketsWritten int64 `json:"packetsWritten"` + BytesWritten int64 `json:"bytesWritten"` + Message string `json:"message"` +} + +func matchesCgroup(body, pod, id string) bool { + for _, line := range strings.Split(body, "\n") { + parts := strings.SplitN(line, ":", 3) + if len(parts) != 3 { + continue + } + p := parts[2] + podMatch := strings.Contains(p, "/pod"+pod+"/") || strings.Contains(p, "-pod"+strings.ReplaceAll(pod, "-", "_")+".slice/") + containerMatch := strings.HasSuffix(p, "/"+id) || strings.HasSuffix(p, "/cri-containerd-"+id+".scope") + if podMatch && containerMatch { + return true + } + } + return false +} + +type targetProcess struct { + root *os.Root + namespace *os.File + pod, id string +} + +func (t *targetProcess) close() { _ = t.namespace.Close(); _ = t.root.Close() } +func (t *targetProcess) verify() error { + body, err := t.root.ReadFile("cgroup") + if err != nil || !matchesCgroup(string(body), t.pod, t.id) { + return errors.New("target container identity changed") + } + return nil +} + +func findTarget(c Config) (*targetProcess, error) { + entries, err := os.ReadDir("/proc") + if err != nil { + return nil, fmt.Errorf("read host processes: %w", err) + } + id := strings.TrimPrefix(c.ContainerID, "containerd://") + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if _, err := strconv.Atoi(entry.Name()); err != nil { + continue + } + root, err := os.OpenRoot(filepath.Join("/proc", entry.Name())) + if err != nil { + continue + } + body, err := root.ReadFile("cgroup") + if err != nil || !matchesCgroup(string(body), c.PodUID, id) { + _ = root.Close() + continue + } + // /proc namespace descriptors are magic symlinks, so os.Root correctly + // refuses Open on them. Open through this already-verified numeric PID, + // then verify the pinned proc directory again before and after setns. + ns, err := os.Open(filepath.Join("/proc", entry.Name(), "ns/net")) + if err != nil { + _ = root.Close() + return nil, fmt.Errorf("open target network namespace: %w", err) + } + t := &targetProcess{root: root, namespace: ns, pod: c.PodUID, id: id} + if err := t.verify(); err != nil { + t.close() + return nil, err + } + return t, nil + } + return nil, errors.New("containerd target process not found on selected node") +} + +func (t *targetProcess) source(filter string) (capture.PacketSource, error) { + type opened struct { + s capture.PacketSource + err error + } + done := make(chan opened, 1) + go func() { + // Never return a thread in the target netns to Go's thread pool. + // A goroutine exiting with its OS thread locked terminates that thread; + // the AF_PACKET descriptor remains bound to the original target netns. + runtime.LockOSThread() + if err := t.verify(); err != nil { + done <- opened{err: err} + return + } + if err := unix.Setns(int(t.namespace.Fd()), unix.CLONE_NEWNET); err != nil { + done <- opened{err: fmt.Errorf("enter target network namespace: %w", err)} + return + } + if err := t.verify(); err != nil { + done <- opened{err: err} + return + } + s, err := capture.NewAFPacketSource("eth0", 0, filter) + done <- opened{s: s, err: err} + }() + r := <-done + return r.s, r.err +} + +// Run captures once and exits. SIGTERM cancels ctx and finalizes the file; +// controller completion waits for worker Pods to disappear before publishing. +func Run(ctx context.Context, c Config, dir string) error { + if err := c.Validate(); err != nil { + return err + } + target, err := findTarget(c) + if err != nil { + return err + } + defer target.close() + source, err := target.source(c.Filter) + if err != nil { + return err + } + defer func() { _ = source.Close() }() + return runCapture(ctx, c, dir, source, target.verify) +} + +func runCapture(ctx context.Context, c Config, dir string, source capture.PacketSource, verify func() error) error { + identity, err := json.Marshal(map[string]string{"serverUID": c.ServerUID, "captureUID": c.CaptureUID}) + if err != nil { + return err + } + output, err := os.OpenRoot(dir) + if err != nil { + return fmt.Errorf("open capture output directory: %w", err) + } + defer func() { _ = output.Close() }() + identityName := "capture-" + c.CaptureID + ".identity.json" + f, err := output.OpenFile(identityName, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + if err != nil { + return fmt.Errorf("reserve capture identity: %w", err) + } + if _, err = f.Write(identity); err != nil { + _ = f.Close() + return err + } + if err = f.Close(); err != nil { + return err + } + // fsGroup grants the nonroot artifact server access after the worker exits. + if err = output.Chmod(identityName, artifactMode); err != nil { + return err + } + writer, err := capture.NewWriter(filepath.Join(dir, "capture-"+c.CaptureID+".pcapng"), c.DurationSeconds, c.MaxSizeBytes, 0, c.Filter) + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, time.Duration(c.DurationSeconds)*time.Second) + defer cancel() + identityChanged := make(chan struct{}, 1) + go func() { + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + if verify() != nil { + identityChanged <- struct{}{} + cancel() + return + } + } + } + }() + result := Result{ServerUID: c.ServerUID, CaptureUID: c.CaptureUID, CaptureID: c.CaptureID, Status: "completed"} + for { + packet, err := source.ReadPacket(ctx) + if err != nil { + if ctx.Err() == nil { + result.Status = "failed" + result.Message = "packet source failed" + } + break + } + if err := writer.WritePacket(packet); err != nil { + if !errors.Is(err, capture.ErrLimitReached) { + result.Status = "failed" + result.Message = "capture write failed" + } + break + } + } + select { + case <-identityChanged: + result.Status = "failed" + result.Message = "target container identity changed" + default: + } + if verify() != nil { + result.Status = "failed" + result.Message = "target container identity changed" + } + if err := writer.Close(); err != nil { + result.Status = "failed" + result.Message = "capture finalization failed" + } + result.PacketsWritten = writer.PacketsWritten() + result.BytesWritten = writer.BytesWritten() + if err := output.Chmod("capture-"+c.CaptureID+".pcapng", artifactMode); err != nil { + return err + } + raw, err := json.Marshal(result) + if err != nil { + return err + } + path := "capture-" + c.CaptureID + ".result.json" + if err := output.WriteFile(path+".tmp", raw, artifactMode); err != nil { + return err + } + return output.Rename(path+".tmp", path) +} diff --git a/capture-sidecar/internal/nodecapture/worker_test.go b/capture-sidecar/internal/nodecapture/worker_test.go new file mode 100644 index 000000000..ccb349f13 --- /dev/null +++ b/capture-sidecar/internal/nodecapture/worker_test.go @@ -0,0 +1,97 @@ +package nodecapture + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/GameplanePanel/gameplane/capture-sidecar/internal/capture" +) + +func TestCgroupIdentityMatchesExactPodAndContainer(t *testing.T) { + pod := "11111111-2222-3333-4444-555555555555" + id := strings.Repeat("a", 64) + for _, test := range []struct { + name, cgroup string + want bool + }{ + {"systemd", "0::/kubepods.slice/kubepods-burstable-pod" + strings.ReplaceAll(pod, "-", "_") + ".slice/cri-containerd-" + id + ".scope\n", true}, + {"cgroupfs", "0::/kubepods/burstable/pod" + pod + "/" + id + "\n", true}, + {"other-pod", "0::/kubepods/pod99999999-2222-3333-4444-555555555555/" + id + "\n", false}, + {"container-prefix", "0::/kubepods/pod" + pod + "/" + id + "a\n", false}, + {"host", "0::/system.slice/containerd.service\n", false}, + } { + t.Run(test.name, func(t *testing.T) { + if got := matchesCgroup(test.cgroup, pod, id); got != test.want { + t.Fatalf("identity match=%t want %t", got, test.want) + } + }) + } +} + +type endingSource struct{ err error } + +func (s endingSource) ReadPacket(context.Context) (*capture.RawPacket, error) { return nil, s.err } +func (endingSource) Close() error { return nil } + +func TestRunCapturePublishesOnlyFinalizedIdentityBoundResult(t *testing.T) { + for _, changed := range []bool{false, true} { + t.Run(map[bool]string{false: "canceled", true: "identity-changed"}[changed], func(t *testing.T) { + dir := t.TempDir() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + c := Config{ServerUID: "server", CaptureUID: "capture", CaptureID: "cap", DurationSeconds: 1, MaxSizeBytes: 4096, Filter: "tcp port 25565"} + verify := func() error { + if changed { + return errors.New("replaced") + } + return nil + } + if err := runCapture(ctx, c, dir, endingSource{context.Canceled}, verify); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(dir, "capture-cap.result.json")) + if err != nil { + t.Fatal(err) + } + var result Result + if err := json.Unmarshal(raw, &result); err != nil { + t.Fatal(err) + } + want := "completed" + if changed { + want = "failed" + } + if result.Status != want || result.ServerUID != "server" || result.CaptureUID != "capture" || result.BytesWritten <= 0 { + t.Fatalf("invalid result: %+v", result) + } + for _, name := range []string{"capture-cap.identity.json", "capture-cap.pcapng", "capture-cap.result.json"} { + stat, err := os.Stat(filepath.Join(dir, name)) + if err != nil || stat.Mode().Perm() != 0o640 { + t.Fatalf("artifact %s mode: %v %v", name, stat, err) + } + } + if err := runCapture(ctx, c, dir, endingSource{context.Canceled}, verify); err == nil { + t.Fatal("reused existing capture identity") + } + }) + } +} + +func TestWorkerRejectsUnsafeConfiguration(t *testing.T) { + c := Config{ServerUID: "server-uid", CaptureUID: "capture-uid", CaptureID: "cap-test", PodUID: "11111111-2222-3333-4444-555555555555", ContainerID: "containerd://" + strings.Repeat("a", 64), Node: "node", Filter: "tcp port 25565", DurationSeconds: 60, MaxSizeBytes: 1024} + if err := c.Validate(); err != nil { + t.Fatal(err) + } + for _, mutate := range []func(*Config){func(c *Config) { c.ContainerID = "docker://" + strings.Repeat("a", 64) }, func(c *Config) { c.CaptureID = "../../host" }, func(c *Config) { c.PodUID = "" }, func(c *Config) { c.DurationSeconds = 3601 }, func(c *Config) { c.MaxSizeBytes = 1 << 30 }, func(c *Config) { c.Filter = "" }} { + bad := c + mutate(&bad) + if bad.Validate() == nil { + t.Fatalf("accepted unsafe config: %+v", bad) + } + } +} diff --git a/capture-sidecar/specs.md b/capture-sidecar/specs.md index 5485d2def..10d8bb994 100644 --- a/capture-sidecar/specs.md +++ b/capture-sidecar/specs.md @@ -6,6 +6,27 @@ ## Purpose +### Separate node diagnostic worker (#714) + +The image also contains `/capture-tool` without file capabilities. Its +`--node-worker` mode accepts only an operator-generated JSON identity tuple +(GameServer/NetworkCapture/Pod UIDs, node, exact containerd ID), filter and +bounded limits. It discovers a process through exact Pod/container cgroup +matching, pins proc/network namespace descriptors, verifies cgroups immediately +before/after setns on a dedicated locked OS thread, then opens an eth0 AF_PACKET +socket. The locked thread exits; it is never returned to the thread pool. +Ongoing identity checks cancel capture, including idle packet streams. No host +PID/path, socket, mount or Kubernetes token is supplied. Linux/containerd, +systemd/cgroupfs paths are supported; other runtimes and host networking fail. + +PCAP, identity and atomically published result files are group-readable only +after finalization. A separate nonroot `--artifacts-only` process exposes +UID-bound download/delete and header-bound status, never start/stop/raw socket +operations. The controller removes the privileged Job/Pods before publishing +completion. SIGTERM finalizes a partial capture; changed identity publishes a +failed result. Host capability blast radius remains node-wide despite the +per-server authorization and exact target selection. + Optional packet capture sidecar injected into game pods to record network traffic for protocol reverse-engineering. The sidecar will use AF_PACKET live capture to collect packets matching a BPF filter expression, write them to a PCAPNG file on a pre-provisioned emptyDir volume, and expose a mTLS-authenticated HTTP control endpoint (`:9091`) for the operator and API to start/stop/monitor captures and download the resulting files. > **Current state:** Full implementation with AF_PACKET socket handling, PCAPNG writing, BPF filter compilation, mTLS authentication, and HTTP control endpoint. Real dependencies (gopacket, go-pcap) are pinned in go.mod. Unit tests for capture, filter, writer, and handlers are co-located with their implementations. TTL-based expiry reconciliation (`NetworkCaptureReconciler.expireCapture`, `operator/internal/controller/networkcapture_controller.go`) and the dashboard UI (`web/src/components/CaptureWidget.tsx`, `web/src/routes/tabs/settings/NetworkCapture.tsx`) are both implemented — see "Planned" below for the one remaining gap (F-190). diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index d902ea995..9bbd901d2 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index d85406090..90170c818 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 4d6aadcbb..425e70175 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index 95178e56e..673ea8588 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1040,10 +1040,18 @@ spec: description: |- Enabled: opt-in flag for packet-capture capability on this GameServer. If false or omitted, no capture sidecar is injected; the game container is unchanged. - If true, the operator injects the capture sidecar as an ephemeral container via + In the legacy profile, the operator injects the capture sidecar via pods/ephemeralcontainers, live, without restarting the game container. Disabling (true→false) does not remove the sidecar from a running pod — Kubernetes has no API to remove ephemeral containers — but stops accepting new captures. + Restricted workloads require NodeDiagnostic instead and never inject a sidecar. + type: boolean + nodeDiagnostic: + description: |- + NodeDiagnostic permits the separately deployed node capture worker for + this server. Application administrators must explicitly opt in. The + worker has host-level capabilities; it never weakens the game Pod. + Required for capture on Restricted workloads. Off by default. type: boolean retentionSeconds: description: |- diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index e5e3e1e3b..cced1885f 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 5ade28cee..593b822c4 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 6c5dcbf36..47d542ac8 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index cc113e7e8..b04079982 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index b77cd5c25..fbcdb337c 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index d902ea995..9bbd901d2 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index d85406090..90170c818 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 4d6aadcbb..425e70175 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index 95178e56e..673ea8588 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: @@ -1040,10 +1040,18 @@ spec: description: |- Enabled: opt-in flag for packet-capture capability on this GameServer. If false or omitted, no capture sidecar is injected; the game container is unchanged. - If true, the operator injects the capture sidecar as an ephemeral container via + In the legacy profile, the operator injects the capture sidecar via pods/ephemeralcontainers, live, without restarting the game container. Disabling (true→false) does not remove the sidecar from a running pod — Kubernetes has no API to remove ephemeral containers — but stops accepting new captures. + Restricted workloads require NodeDiagnostic instead and never inject a sidecar. + type: boolean + nodeDiagnostic: + description: |- + NodeDiagnostic permits the separately deployed node capture worker for + this server. Application administrators must explicitly opt in. The + worker has host-level capabilities; it never weakens the game Pod. + Required for capture on Restricted workloads. Off by default. type: boolean retentionSeconds: description: |- diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index e5e3e1e3b..cced1885f 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 5ade28cee..593b822c4 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 6c5dcbf36..47d542ac8 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index cc113e7e8..b04079982 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index b77cd5c25..fbcdb337c 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 181aeee1d52a9cb178b0f8a8a862dc5959e772facf11817d5e70e5196574cd4c + gameplane.local/crd-bundle-sha256: 2a90e7f1b247291f31d6bed22f2fd59af08f0a15aa779836d7afff235c98ba0c controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index bec3e1ffb..f2da8ef4a 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -135,3 +135,9 @@ true true {{- end -}} {{- end -}} +{{- /* Resolve new diagnostics defaults for upgrades using older reuse-values. +mergeOverwrite preserves explicit zero/empty values so validation can reject them. */ -}} +{{- define "gameplane.nodeDiagnostics" -}} +{{- $configured := (.Values.capture | default dict).nodeDiagnostics | default dict -}} +{{- mergeOverwrite (dict "enabled" false "namespace" "gameplane-diagnostics" "maxCaptures" 10 "storageQuota" "10Gi" "storageClass" "") $configured | toJson -}} +{{- end -}} diff --git a/charts/gameplane/templates/diagnostics.yaml b/charts/gameplane/templates/diagnostics.yaml new file mode 100644 index 000000000..2f886951e --- /dev/null +++ b/charts/gameplane/templates/diagnostics.yaml @@ -0,0 +1,122 @@ +{{- $d := include "gameplane.nodeDiagnostics" . | fromJson }} +{{- if $d.enabled }} +# This namespace is an explicit host-privilege exception. Never share it with +# games or the control plane; the worker has no Kubernetes token or host mount. +apiVersion: v1 +kind: Namespace +metadata: + name: {{ $d.namespace }} + labels: + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/enforce-version: {{ .Values.podSecurity.version | default "v1.34" | quote }} +--- +apiVersion: v1 +kind: ResourceQuota +metadata: + name: gameplane-diagnostics + namespace: {{ $d.namespace }} +spec: + hard: + persistentvolumeclaims: {{ $d.maxCaptures | quote }} + requests.storage: {{ $d.storageQuota | quote }} + pods: {{ mul (int $d.maxCaptures) 2 | quote }} + count/jobs.batch: {{ $d.maxCaptures | quote }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ .Release.Name }}-diagnostics-namespace +rules: + - apiGroups: [""] + resources: [namespaces] + resourceNames: [{{ $d.namespace | quote }}] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ .Release.Name }}-diagnostics-namespace +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .Release.Name }}-diagnostics-namespace +subjects: + - kind: ServiceAccount + name: gameplane-operator + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gameplane-diagnostics + namespace: {{ $d.namespace }} +rules: + - apiGroups: [""] + resources: [pods, services, persistentvolumeclaims, configmaps, secrets] + verbs: [get, list, create, update, patch, delete] + - apiGroups: ["batch"] + resources: [jobs] + verbs: [get, list, create, update, patch, delete] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: gameplane-diagnostics + namespace: {{ $d.namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: gameplane-diagnostics +subjects: + - kind: ServiceAccount + name: gameplane-operator + namespace: {{ .Release.Namespace }} +--- +# Files remain behind mTLS and UID-bound routes. Workers need no network +# egress: their AF_PACKET socket receives traffic in the selected game netns. +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gameplane-diagnostics + namespace: {{ $d.namespace }} +spec: + podSelector: {} + policyTypes: [Ingress, Egress] + egress: [] + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Release.Namespace }} + podSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: [gameplane-operator, gameplane-api, gameplane-gateway] + ports: + - { protocol: TCP, port: 9091 } +{{- if (.Values.gateway | default dict).enabled }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: gameplane-gateway-diagnostics + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gameplane-gateway + app.kubernetes.io/instance: {{ .Release.Name }} + policyTypes: [Egress] + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ $d.namespace }} + podSelector: + matchLabels: + gameplane.local/capture-component: artifacts + ports: + - { protocol: TCP, port: 9091 } +{{- end }} +{{- end }} diff --git a/charts/gameplane/templates/namespaces.yaml b/charts/gameplane/templates/namespaces.yaml index 8642b33dd..b73cce0bd 100644 --- a/charts/gameplane/templates/namespaces.yaml +++ b/charts/gameplane/templates/namespaces.yaml @@ -7,7 +7,7 @@ metadata: {{- include "gameplane.labels" . | nindent 4 }} {{- if .Values.podSecurity.enforceRestricted }} pod-security.kubernetes.io/enforce: restricted - pod-security.kubernetes.io/enforce-version: latest + pod-security.kubernetes.io/enforce-version: {{ .Values.podSecurity.version | default "v1.34" | quote }} {{- end }} annotations: # F-212: exclude the games Namespace from Helm's ownership so diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 772e0b10e..002111f72 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -137,7 +137,7 @@ rules: resources: [gameservers/status, backups/status, backupschedules/status, restores/status, networkcaptures/status] verbs: [get, update, patch] - apiGroups: ["gameplane.local"] - resources: [gameservers/finalizers, backups/finalizers, backupschedules/finalizers, restores/finalizers] + resources: [gameservers/finalizers, backups/finalizers, backupschedules/finalizers, restores/finalizers, networkcaptures/finalizers] verbs: [update] - apiGroups: ["apps"] resources: [statefulsets, deployments] @@ -314,9 +314,16 @@ spec: - --agent-ca-secret-name={{ .Values.api.agentMTLS.caSecretRef.name }} - --agent-ca-secret-namespace={{ .Release.Namespace }} - --module-namespace={{ .Release.Namespace }} + - --restricted-workloads={{ .Values.podSecurity.enforceRestricted }} + - --agent-token-expiration-seconds={{ .Values.operator.agentTokenExpirationSeconds | default 3600 }} {{- if and .Values.capture .Values.capture.enabled }} - --capture-enabled=true - --capture-sidecar-image={{ include "gameplane.captureImage" . }} + {{- $diagnostics := include "gameplane.nodeDiagnostics" . | fromJson }} + {{- if $diagnostics.enabled }} + - --diagnostics-namespace={{ $diagnostics.namespace }} + - --diagnostics-storage-class={{ $diagnostics.storageClass | default "" }} + {{- end }} - --capture-default-retention-seconds={{ .Values.capture.defaultRetentionSeconds | int64 }} - --capture-max-retention-seconds={{ .Values.capture.maxRetentionSeconds | int64 }} - --capture-default-max-duration-seconds={{ .Values.capture.defaultMaxDurationSeconds | int64 }} diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index 79b5e17b6..12e28a855 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,3 +1,18 @@ +{{- $diagnostics := include "gameplane.nodeDiagnostics" . | fromJson }} +{{- if $diagnostics.enabled }} +{{- if or (not .Values.capture.enabled) (ne (include "gameplane.operatorEnabled" .) "true") }} +{{- fail "capture.nodeDiagnostics requires capture.enabled and the operator" }} +{{- end }} +{{- if or (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" $diagnostics.namespace)) (gt (len $diagnostics.namespace) 63) (eq $diagnostics.namespace .Values.gamesNamespace) (eq $diagnostics.namespace .Release.Namespace) }} +{{- fail "capture.nodeDiagnostics.namespace must be a separate explicit DNS-label namespace" }} +{{- end }} +{{- if lt (int $diagnostics.maxCaptures) 1 }} +{{- fail "capture.nodeDiagnostics.maxCaptures must be positive" }} +{{- end }} +{{- end }} +{{- if not (regexMatch "^v1\\.[0-9]+$" (.Values.podSecurity.version | default "v1.34")) }} +{{- fail "podSecurity.version must pin a Kubernetes minor such as v1.34" }} +{{- end }} {{- if .Values.api.standalone }} {{- if ne (include "gameplane.apiEnabled" .) "true" }} {{- fail "api.standalone requires api.enabled=true" }} diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 66d95af0b..22f748968 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -50,6 +50,8 @@ crds: pullPolicy: IfNotPresent operator: + # Bound, rotating agent-only token; 600 supports short rotation drills. + agentTokenExpirationSeconds: 3600 # Disable the operator and bundled native CRDs for a standalone central panel. enabled: true replicas: 1 @@ -654,7 +656,8 @@ grafanaDashboards: grafana_dashboard: "1" podSecurity: - enforceRestricted: false # label gamesNamespace with pod-security.kubernetes.io/enforce=restricted + enforceRestricted: false # require compatible nonroot templates and enforce Restricted game pods + version: v1.34 # pinned admission policy; change after testing workload compatibility # Default ModuleSource shipped with the chart. Disable when you manage # ModuleSource resources via GitOps. The operator indexes this source on a @@ -742,6 +745,14 @@ uploadModuleSource: # opt in has no capture sidecar attached. Off by default. capture: enabled: false + # Separate host-capable Linux/containerd worker; each server also needs an + # audited administrator opt-in. Never enables privileges in game Pods. + nodeDiagnostics: + enabled: false + namespace: gameplane-diagnostics + storageClass: "" + maxCaptures: 10 + storageQuota: 10Gi # Default retention window for completed captures (seconds). Per-server overrides # are clamped to the cluster maximum below; this is the default when a server does # not specify a retention policy. diff --git a/charts/tests/check_workload_security.py b/charts/tests/check_workload_security.py new file mode 100644 index 000000000..5718f7ae7 --- /dev/null +++ b/charts/tests/check_workload_security.py @@ -0,0 +1,65 @@ +"""CI-only chart checks for the opt-in Restricted workload and node exception.""" +from pathlib import Path +import shutil +import tempfile + +import yaml + +from check_gateway import CHART, one, render, settings + + +def check_reuse_values(): + with tempfile.TemporaryDirectory() as directory: + chart = Path(directory) / "chart" + shutil.copytree(CHART, chart) + values_file = chart / "values.yaml" + old_values = yaml.safe_load(values_file.read_text()) + old_values["capture"].pop("nodeDiagnostics", None) + values_file.write_text(yaml.safe_dump(old_values)) + enabled = {"capture": {"enabled": True, "nodeDiagnostics": {"enabled": True}}} + objects = render(enabled, upgrade=True, chart=chart) + one(objects, "Namespace", "gameplane-diagnostics") + quota = one(objects, "ResourceQuota", "gameplane-diagnostics")["spec"]["hard"] + assert quota == {"persistentvolumeclaims": "10", "requests.storage": "10Gi", "pods": "20", "count/jobs.batch": "10"} + args = one(objects, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert "--diagnostics-namespace=gameplane-diagnostics" in args + enabled["capture"]["nodeDiagnostics"]["maxCaptures"] = 0 + render(enabled, upgrade=True, chart=chart, failure="maxCaptures must be positive") + enabled["capture"]["nodeDiagnostics"].pop("maxCaptures") + enabled["capture"]["nodeDiagnostics"]["namespace"] = "" + render(enabled, upgrade=True, chart=chart, failure="separate explicit") + + +def main(): + check_reuse_values() + defaults = render() + assert not any(o["metadata"]["name"] == "gameplane-diagnostics" for o in defaults) + objects = render({"podSecurity": {"enforceRestricted": True}}) + games = one(objects, "Namespace", "gameplane-games") + assert games["metadata"]["labels"]["pod-security.kubernetes.io/enforce"] == "restricted" + assert games["metadata"]["labels"]["pod-security.kubernetes.io/enforce-version"] == "v1.34" + args = one(objects, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] + assert "--restricted-workloads=true" in args # even when capture is disabled + assert not any(a.startswith("--diagnostics-namespace=") for a in args) + values = {"podSecurity": {"enforceRestricted": True, "version": "v1.35"}, + "capture": {"enabled": True, "nodeDiagnostics": {"enabled": True}}, + "gateway": settings()} + objects = render(values) + assert one(objects, "Namespace", "gameplane-games")["metadata"]["labels"]["pod-security.kubernetes.io/enforce-version"] == "v1.35" + assert one(objects, "Namespace", "gameplane-diagnostics")["metadata"]["labels"]["pod-security.kubernetes.io/enforce"] == "privileged" + quota = one(objects, "ResourceQuota", "gameplane-diagnostics")["spec"]["hard"] + assert quota["persistentvolumeclaims"] == "10" and quota["requests.storage"] == "10Gi" + role = one(objects, "Role", "gameplane-diagnostics") + assert role["metadata"]["namespace"] == "gameplane-diagnostics" + assert all("*" not in r["resources"] for r in role["rules"]) + policy = one(objects, "NetworkPolicy", "gameplane-diagnostics")["spec"] + assert policy["egress"] == [] and policy["ingress"][0]["ports"] == [{"protocol": "TCP", "port": 9091}] + one(objects, "NetworkPolicy", "gameplane-gateway-diagnostics") + for namespace in ("gameplane-games", "gameplane-system", "../host"): + render({"capture": {"enabled": True, "nodeDiagnostics": {"enabled": True, "namespace": namespace}}}, failure="separate explicit") + render({"capture": {"nodeDiagnostics": {"enabled": True}}}, failure="requires capture.enabled") + render({"podSecurity": {"version": "latest"}}, failure="must pin") + + +if __name__ == "__main__": + main() diff --git a/docs/install.md b/docs/install.md index b4da5a2a3..64efea6ee 100644 --- a/docs/install.md +++ b/docs/install.md @@ -198,7 +198,10 @@ Top-level knobs (see `values.yaml` for the full list): - `mcpServer.enabled` — optional strictly read-only MCP (Model Context Protocol) server [optional] for AI assistants to read cluster state and propose fixes (default off); see [mcp-server/README.md](../mcp-server/README.md) - `mcpServer.replicas` — MCP server replicas (default 1) - `updates.channel` — informational release-channel label (e.g., `stable`, `edge`) shown read-only in the dashboard's Admin Settings → Updates section; purely informational (Gameplane upgrades via Helm, not auto-update) -- `podSecurity.enforceRestricted` — label games namespace for Pod Security Standards +- `podSecurity.enforceRestricted` — enable the Restricted game/config-init profile and namespace admission. Templates require positive `security.runAsUser` and `security.fsGroup`; verify image compatibility and migrate existing file ownership before upgrading. Enabling it rolls existing Pods once. + - `podSecurity.version` — pinned PSA minor, default `v1.34`; administrator override for a tested cluster version. + - `operator.agentTokenExpirationSeconds` — projected agent-only token lifetime, 600–3600 seconds (default 3600); rotation is handled by client-go. + - `capture.nodeDiagnostics.enabled` — off by default. With `capture.enabled`, creates a separate privileged namespace and bounded diagnostic storage. An administrator must additionally opt each server in through `:capture-node-enable`. Linux/containerd only; no capture-related game restart. Read the [host-privilege and cleanup requirements](security.md#network-capture-security-exception) before deployment. - `defaultModuleSource.*` — the official game catalog shipped with the chart (enabled by default) - `defaultModuleSource.enabled` — whether to create the default `ModuleSource` (default `true`; disable when managing sources via GitOps) - `defaultModuleSource.name` — name of the `ModuleSource` resource (default `default`) diff --git a/docs/security.md b/docs/security.md index b50a6d01b..d6456ad81 100644 --- a/docs/security.md +++ b/docs/security.md @@ -367,13 +367,32 @@ the image's own default user, which may be root. The operator sets no container runtime's defaults. This keeps arbitrary third-party game images working unchanged. -Game pods are shaped per-template. For a hostile game module, enable -Pod Security Standards `restricted` on the games namespace via -`podSecurity.enforceRestricted=true`. +`podSecurity.enforceRestricted=true` enables a different workload profile as +well as Restricted admission. Game and config-init containers use a positive +template UID/GID, `runAsNonRoot`, no privilege escalation, `drop: [ALL]`, and +`RuntimeDefault` seccomp. The operator rejects templates without positive +`security.runAsUser` and `security.fsGroup` (and a positive `runAsGroup` when +supplied) and Hostport exposure before creating workloads; use NodePort or +LoadBalancer with Restricted admission. Administrators must choose a compatible +image and migrate existing data ownership; numeric identities cannot prove +image compatibility. Enabling the profile rolls existing game Pods once. PSA +is pinned to `v1.34`; `podSecurity.version` accepts another tested minor. +Config bootstrap uses the declared fsGroup as its primary group even on +volumes that do not apply fsGroup ownership automatically. Repeated starts +preserve already-correct group-write permissions on root-owned config files. +This does not repair arbitrary pre-existing data ownership after root access +has been removed; migrate that data before enabling the profile. + +In both profiles, automatic ServiceAccount mounts are disabled. Only the agent +gets a projected, rotating token, CA and namespace at the standard client-go +path. Game and config-init containers have no token mount. Shared PID remains +enabled for metrics: processes in one Pod still share a trust boundary. +`/proc`, matching UIDs, shared files or a compromised agent can undermine mount +isolation. Restricted is not a safe sandbox for hostile modules. ### Network Capture Security Exception -The optional network capture feature [optional] (see +The legacy optional network capture feature [optional] (see [`docs/roadmap.md`](roadmap.md)) adds an ephemeral sidecar container to game pods when capture is enabled. This sidecar requires **`allowPrivilegeEscalation: true`** in its securityContext, which violates the Pod Security Standards `restricted` @@ -404,35 +423,47 @@ file capability to function. The game container does not share this exception: the operator never sets `allowPrivilegeEscalation: true` on it or adds `NET_RAW` to it, and the capture -sidecar is the only container Gameplane grants `CAP_NET_RAW` for capture. The +worker receives explicit `CAP_NET_RAW` for capture. The game container keeps the posture described in [Pod security](#pod-security): the template's uid/gid or the image's default user, and the container runtime's default capability set. Which capabilities the game container holds therefore depends on that runtime default and on the user the image runs as, not on the capture feature. -**Trade-off with PodSecurity `restricted`**: A cluster enforcing the `restricted` -Pod Security Standards profile on the games namespace will reject any pod with -`allowPrivilegeEscalation: true`. If you require `restricted` admission on your -games namespace, you have three options: - -1. **Disable capture** — leave the cluster's capture feature disabled via Helm - value `capture.enabled: false` (default is false). Captures are not required - for normal operation; this is the safest option if you cannot or prefer not to - relax the `restricted` profile. -2. **Exempt the games namespace** — remove or relax the Pod Security Standards - `restricted` enforcement for the games namespace, using `baseline` or - `privileged` instead. The games namespace remains an untrusted environment - (game code can run arbitrary containers), but the admission level permits the - capture sidecar to be injected when needed. -3. **Leave the games-namespace label off** — the chart only adds the - `pod-security.kubernetes.io/enforce: restricted` label to the games namespace - when `podSecurity.enforceRestricted=true`; that value defaults to `false`. - A default install therefore already leaves the label off, and captures work - without any change. If you previously set `podSecurity.enforceRestricted=true` - and accept the operational trade-off, set it back to `false` in the Helm - values to drop the label. This setting only affects the games `Namespace` - object; it has no cluster-wide effect and there is no per-pod opt-in. +**Capture with Restricted games** uses a separate node diagnostic worker. +Enable both `capture.enabled` and `capture.nodeDiagnostics.enabled`, then an +administrator calls `POST /servers/{name}:capture-node-enable` for each allowed +server. The API synchronously audits the host-privilege opt-in before changing +`spec.capture.nodeDiagnostic`; ordinary edits cannot grant it and clones do not +inherit it. `:capture-node-disable` revokes it and drains active workers. Direct +Kubernetes administrators can manage the field through their own audited +GitOps process. The game Pod receives no capture container or restart. + +The chart creates a separate `gameplane-diagnostics` namespace (configurable), +explicitly labeled `privileged`. Deployment policy must permit hostPID, root, +`SYS_ADMIN`, `SYS_PTRACE`, `NET_RAW` and unconfined seccomp there. This is +host-level trust: a compromised worker can affect its node despite per-server +application authorization. It has no Kubernetes token, runtime socket or +hostPath. It accepts live Kubernetes identities, never user-supplied PIDs or +paths; checks Pod/container cgroups immediately before and after entering the +pinned network namespace; and stops when the target changes. Support is +limited to Linux/containerd, cgroupfs or systemd Pod cgroups, and target `eth0`. +Host-network and other runtime targets fail closed. Windows, rootless or user +namespaced runtimes, and nonstandard CNI interfaces are unsupported. +SELinux/AppArmor may deny these capabilities; capture fails rather than +broadening privileges. + +Each capture is bounded to one hour and 900MiB with a 1Gi PVC. The chart defaults +to ten captures and a 10Gi storage quota, including retained files. Privileged +Jobs and Pods are removed on completion, cancellation, revocation or target +identity change. A separate nonroot Pod serves only UID-bound files and status +over mTLS. A capture finalizer deletes its same-namespace anchor and waits for +garbage collection of Pods, PVC, Secret and Service on expiry, capture deletion +or GameServer deletion. Do not force-remove the finalizer without cleaning its +labeled diagnostic resources. Keep the operator and diagnostic namespace +installed until cleanup finishes. Disabling the deployment option deletes the +diagnostics namespace and retained files. No games-namespace or RuntimeClass +exemption is used. **Data sensitivity**: Captures contain binary game protocols, player IP addresses, and may include sensitive data like in-game chat or credentials. An admin with diff --git a/hack/ci_scope.py b/hack/ci_scope.py index 91da39131..9f58861b0 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -29,7 +29,8 @@ FORCE_ALL_PREFIXES = (".github/actions/",) # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml. E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components"] + "telemetry", "optional-components", "restricted-workloads"] +E2E_AMD64_ONLY = set() SUITES_FILE = "test/e2e/suite-components.json" # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. NO_E2E_FILES = { @@ -936,7 +937,8 @@ def e2e_row(suite, gate_on, picked, key, feed): for bucket in E2E_BUCKETS: name = "e2e api-auth + ratelimit tail" if bucket == "api-auth" else f"e2e {bucket}" picked = bucket in e2e_run or (bucket == "api-auth" and "ratelimit" in e2e_run) - e2e_row(f"{name} (amd64, arm64)", gate("e2e"), picked, bucket, e2e_feed()) + arches = "amd64" if bucket in E2E_AMD64_ONLY else "amd64, arm64" + e2e_row(f"{name} ({arches})", gate("e2e"), picked, bucket, e2e_feed()) e2e_row("e2e multicluster (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-multicluster"), "multicluster", e2e_feed()) e2e_row("e2e upgrade (amd64, arm64)", gate("e2e"), flag(e2e, "e2e-upgrade"), "upgrade", diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index c8dd10dc1..e7500ee1f 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -9,7 +9,7 @@ from unittest.mock import patch from subprocess import CompletedProcess -from ci_scope import (E2E_BUCKETS, FORCE_ALL_FILES, ROOT, affected_modules, bot_matrix, +from ci_scope import (E2E_AMD64_ONLY, E2E_BUCKETS, FORCE_ALL_FILES, ROOT, affected_modules, bot_matrix, bucket_lists, changed_paths, e2e_outputs, full_scope_reason, image_inputs, is_unmapped, load_suites, main, publish_paths, select_images, summarize, verify_suites, go_packages, web_tests, CI_FILE, ci_blocks, ci_jobs, @@ -347,10 +347,10 @@ def test_every_e2e_bucket_and_bot_test_has_a_row(self): """Check every e2e bucket and bot test has a selection row.""" rows, _ = self.web_only() suites = {row["suite"] for row in rows} - self.assertEqual(len(E2E_BUCKETS), 8) for bucket in E2E_BUCKETS: name = "e2e api-auth + ratelimit tail" if bucket == "api-auth" else f"e2e {bucket}" - self.assertIn(f"{name} (amd64, arm64)", suites) + arches = "amd64" if bucket in E2E_AMD64_ONLY else "amd64, arm64" + self.assertIn(f"{name} ({arches})", suites) for test in SUMMARY_BOTS: self.assertIn(f"e2e game bot / {test}", suites) @@ -408,6 +408,16 @@ def select(self, *paths): """Return the e2e outputs for a partial-scope change to paths.""" return e2e_outputs(ROOT, list(paths), None) + def test_audit_restricted_workloads_has_a_run_pattern(self): + """The node capture worker and all drill components retain CI coverage.""" + for path in ("operator/internal/controller/workload_security.go", + "api/internal/handlers/capture_node.go", "agent/cmd/main.go", + "capture-sidecar/internal/nodecapture/worker.go", + "test/e2e/restricted_workloads_e2e_test.go"): + with self.subTest(path=path): + self.assertIn("TestRestrictedWorkloads_NodeCaptureAndTokenRotation", + self.select(path)["e2e-run"].get("restricted-workloads", "")) + def test_agent_only_change_selects_agent_tests_and_skips_auth_and_telemetry(self): values = self.select("agent/internal/console/console.go") self.assertEqual(values["reason"], "") @@ -504,6 +514,20 @@ def test_repository_suite_map_is_valid(self): class E2ERunTests(unittest.TestCase): """The e2e-run regexes that replace buckets.sh regex in the e2e jobs (T016).""" + def test_bucket_registry_matches_workflow_matrices(self): + """Every matrix bucket emits a run pattern and reports its real architectures.""" + text = (ROOT / CI_FILE).read_text(encoding="utf-8") + owner, _ = ci_blocks(text) + matrices = {} + for number, line in enumerate(text.splitlines(), 1): + job = owner[number] + if job in {"e2e-go", "e2e-go-arm64"} and line.strip().startswith("bucket: ["): + self.assertNotIn(job, matrices) + matrices[job] = {name.strip() for name in line.split("[", 1)[1].split("]", 1)[0].split(",")} + self.assertEqual(set(matrices), {"e2e-go", "e2e-go-arm64"}) + self.assertEqual(set(E2E_BUCKETS), matrices["e2e-go"] | matrices["e2e-go-arm64"]) + self.assertEqual(E2E_AMD64_ONLY, matrices["e2e-go"] - matrices["e2e-go-arm64"]) + def test_full_scope_regex_equals_buckets_sh_regex(self): values = e2e_outputs(ROOT, None, "master push") self.assertEqual(values["reason"], "master push") diff --git a/operator/api/v1alpha1/gameserver_types.go b/operator/api/v1alpha1/gameserver_types.go index 654cba6ed..7fefd5c16 100644 --- a/operator/api/v1alpha1/gameserver_types.go +++ b/operator/api/v1alpha1/gameserver_types.go @@ -224,15 +224,23 @@ type IdleSpec struct { type CaptureConfiguration struct { // Enabled: opt-in flag for packet-capture capability on this GameServer. // If false or omitted, no capture sidecar is injected; the game container is unchanged. - // If true, the operator injects the capture sidecar as an ephemeral container via + // In the legacy profile, the operator injects the capture sidecar via // pods/ephemeralcontainers, live, without restarting the game container. // Disabling (true→false) does not remove the sidecar from a running pod — Kubernetes // has no API to remove ephemeral containers — but stops accepting new captures. + // Restricted workloads require NodeDiagnostic instead and never inject a sidecar. // +kubebuilder:validation:Optional // +kubebuilder:validation:Type=boolean // +optional Enabled bool `json:"enabled,omitempty"` + // NodeDiagnostic permits the separately deployed node capture worker for + // this server. Application administrators must explicitly opt in. The + // worker has host-level capabilities; it never weakens the game Pod. + // Required for capture on Restricted workloads. Off by default. + // +optional + NodeDiagnostic bool `json:"nodeDiagnostic,omitempty"` + // RetentionSeconds: optional per-GameServer override of the cluster-default retention window // after a capture completes. Unit: seconds. If omitted, the cluster default // (Helm value capture.defaultRetentionSeconds) is used. Must not exceed the cluster diff --git a/operator/cmd/main.go b/operator/cmd/main.go index 85263e901..46b3bd1e2 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -143,6 +143,10 @@ func main() { gameIngressFromCIDR cidrListFlag gameDataStorageClass string captureEnabled bool + restrictedWorkloads bool + agentTokenExpirationSeconds int64 + diagnosticsNamespace string + diagnosticsStorageClass string captureDefaultRetention int64 captureMaxRetention int64 captureSidecarImage string @@ -185,6 +189,10 @@ func main() { flag.StringVar(&tunnelPlayitImage, "tunnel-playit-image", controller.DefaultTunnelPlayitImage, "Image for the Playit tunnel relay pod that routes players through a Playit tunnel. "+ "Point at a private registry mirror for air-gapped installs.") + flag.BoolVar(&restrictedWorkloads, "restricted-workloads", false, "Require compatible nonroot templates and render Restricted game workloads") + flag.Int64Var(&agentTokenExpirationSeconds, "agent-token-expiration-seconds", 3600, "Projected agent token lifetime (600-3600 seconds)") + flag.StringVar(&diagnosticsNamespace, "diagnostics-namespace", "", "Opt in to node diagnostics in this dedicated privileged namespace") + flag.StringVar(&diagnosticsStorageClass, "diagnostics-storage-class", "", "StorageClass for bounded node capture artifact PVCs") flag.BoolVar(&captureEnabled, "capture-enabled", false, "Enable the network capture feature cluster-wide. When false (the default), "+ "the capture capability is disabled and cannot be enabled per-GameServer.") @@ -259,6 +267,14 @@ func main() { opts := zap.Options{Development: true} opts.BindFlags(flag.CommandLine) flag.Parse() + if agentTokenExpirationSeconds < 600 || agentTokenExpirationSeconds > 3600 { + fmt.Fprintln(os.Stderr, "agent-token-expiration-seconds must be between 600 and 3600") + os.Exit(1) + } + if diagnosticsNamespace != "" && diagnosticsNamespace == controlPlaneNamespace { + fmt.Fprintln(os.Stderr, "diagnostics-namespace must be separate from the control-plane namespace") + os.Exit(1) + } ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) setupLog := ctrl.Log.WithName("setup") @@ -394,13 +410,16 @@ func main() { Config: mgr.GetConfig(), Clientset: kubernetes.NewForConfigOrDie(mgr.GetConfig()), }, - AddressManager: addressManager, - MetalLBNamespace: metalLBNamespace, - GameIngressPolicyEnabled: gameIngressPolicy, - GameIngressFromCIDRs: gameIngressFromCIDR, - CaptureEnabled: captureEnabled, - CaptureSidecarImage: captureSidecarImage, - DefaultStorageClassName: gameDataStorageClass, + AddressManager: addressManager, + MetalLBNamespace: metalLBNamespace, + GameIngressPolicyEnabled: gameIngressPolicy, + GameIngressFromCIDRs: gameIngressFromCIDR, + CaptureEnabled: captureEnabled, + RestrictedWorkloads: restrictedWorkloads, + AgentTokenExpirationSeconds: agentTokenExpirationSeconds, + DiagnosticsNamespace: diagnosticsNamespace, + CaptureSidecarImage: captureSidecarImage, + DefaultStorageClassName: gameDataStorageClass, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to set up controller", "controller", "GameServer") os.Exit(1) @@ -451,6 +470,12 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), SidecarClient: captureClient, + APIReader: mgr.GetAPIReader(), + RestrictedWorkloads: restrictedWorkloads, + DiagnosticsNamespace: diagnosticsNamespace, + DiagnosticsStorageClass: diagnosticsStorageClass, + AgentCASecretName: agentCASecretName, + AgentCASecretNamespace: agentCASecretNamespace, CaptureEnabled: captureEnabled, CaptureSidecarImage: captureSidecarImage, CaptureDefaultMaxDurationSeconds: captureDefaultMaxDurationSeconds, diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index 4be670ebc..0f1639e41 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -1039,10 +1039,18 @@ spec: description: |- Enabled: opt-in flag for packet-capture capability on this GameServer. If false or omitted, no capture sidecar is injected; the game container is unchanged. - If true, the operator injects the capture sidecar as an ephemeral container via + In the legacy profile, the operator injects the capture sidecar via pods/ephemeralcontainers, live, without restarting the game container. Disabling (true→false) does not remove the sidecar from a running pod — Kubernetes has no API to remove ephemeral containers — but stops accepting new captures. + Restricted workloads require NodeDiagnostic instead and never inject a sidecar. + type: boolean + nodeDiagnostic: + description: |- + NodeDiagnostic permits the separately deployed node capture worker for + this server. Application administrators must explicitly opt in. The + worker has host-level capabilities; it never weakens the game Pod. + Required for capture on Restricted workloads. Off by default. type: boolean retentionSeconds: description: |- diff --git a/operator/config/rbac/role_namespace.yaml b/operator/config/rbac/role_namespace.yaml index cdbefd6bd..9e4e3e29b 100644 --- a/operator/config/rbac/role_namespace.yaml +++ b/operator/config/rbac/role_namespace.yaml @@ -40,6 +40,7 @@ rules: - backups/finalizers - backupschedules/finalizers - restores/finalizers + - networkcaptures/finalizers verbs: ["update"] # Deployments: the sentinel and tunnel-client Deployments live beside # their GameServer. diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 67e8be4e3..60ff4f392 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -151,6 +151,11 @@ type GameServerReconciler struct { // per-GameServer; when true, it can be toggled on/off per server. CaptureEnabled bool + // RestrictedWorkloads applies the opt-in PSA-compatible game profile. + RestrictedWorkloads bool + AgentTokenExpirationSeconds int64 + DiagnosticsNamespace string + // CaptureSidecarImage is the container image for the network capture sidecar // injected when capture is enabled on a GameServer. Set from the // --capture-sidecar-image operator flag so air-gapped installs can point it @@ -248,6 +253,11 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } else if changed { return ctrl.Result{Requeue: true}, nil } + // Exposure compatibility depends only on the server. Reject it before + // resolving/adopting a template, which may otherwise initiate a rollout. + if r.RestrictedWorkloads && gs.Spec.Networking.Expose == "Hostport" { + return ctrl.Result{}, r.setPhase(ctx, &gs, "restricted workload cannot use networking.expose=Hostport; use NodePort or LoadBalancer") + } // Resolve the template this GameServer points at. Templates are // cluster-scoped so no namespace is needed. @@ -264,6 +274,11 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) // touching any children: invalid config must fail loudly (mirroring // the missing-template path) instead of materializing a pod that // silently ignores what the user asked for. + if r.RestrictedWorkloads { + if err := validateRestrictedTemplate(&tmpl); err != nil { + return ctrl.Result{}, r.setPhase(ctx, &gs, err.Error()) + } + } mc, err := materializeConfig(&gs, &tmpl) if err != nil { return ctrl.Result{}, r.setPhase(ctx, &gs, @@ -1596,6 +1611,15 @@ func (r *GameServerReconciler) reconcileStatefulSet( // Same unconditional-assign reasoning: removing template.spec.security // must clear a previously-set pod-level FSGroup, not leave it pinned. ss.Spec.Template.Spec.SecurityContext = gamePodSecurityContext(tmpl) + if r.RestrictedWorkloads { + if ss.Spec.Template.Annotations == nil { + ss.Spec.Template.Annotations = map[string]string{} + } + ss.Spec.Template.Annotations[workloadSecurityLabel] = "restricted" + } else { + delete(ss.Spec.Template.Annotations, workloadSecurityLabel) + } + applyWorkloadSecurity(&ss.Spec.Template.Spec, tmpl, r.RestrictedWorkloads, r.AgentTokenExpirationSeconds) // Default to the per-GameServer SA so the agent's heartbeat can // patch gameservers/status (see reconcileAgentRBAC); an explicit // spec.serviceAccountName still wins. @@ -1877,6 +1901,10 @@ func (r *GameServerReconciler) reconcileCapture(ctx context.Context, gs *gamepla return fmt.Errorf("get game pod for capture reconciliation: %w", podErr) } + if r.RestrictedWorkloads || pod.Annotations[workloadSecurityLabel] == "restricted" || (gs.Spec.Capture != nil && gs.Spec.Capture.NodeDiagnostic) { + gs.Status.Capture.Ready = r.DiagnosticsNamespace != "" && gs.Spec.Capture.NodeDiagnostic && pod.Status.Phase == corev1.PodRunning + return r.patchCaptureStatus(ctx, gs, base) + } if !hasCaptureEphemeralContainer(&pod) { image := r.CaptureSidecarImage pod.Spec.EphemeralContainers = append(pod.Spec.EphemeralContainers, buildCaptureEphemeralContainer(image, string(gs.UID))) @@ -1987,18 +2015,26 @@ func buildConfigInitContainer(image string, tmpl *gameplanev1alpha1.GameTemplate image = configInitImageOrDefault(image) mountPath := effectiveMountPath(tmpl) cpCmd := "cp -RL " + configFilesStagingPath + "/* '" + mountPath + "/'" + var security *corev1.SecurityContext - // If fsGroup is set, append a chmod step to make each file and directory - // config-init copied group-writable (entries that came from the staging tree; - // other files already on the volume keep their modes). + // Newly created files must inherit the declared shared group even on + // local-path volumes that do not implement kubelet fsGroup ownership. + // Only reconcile copied entries; unrelated server data keeps its modes. if tmpl.Spec.Security != nil && tmpl.Spec.Security.FSGroup != nil { - cpCmd += " && cd " + configFilesStagingPath + " && find ./* -follow \\( -type f -o -type d \\) | while IFS= read -r p; do chmod g+w '" + mountPath + "/'\"$p\"; done" + group := *tmpl.Spec.Security.FSGroup + security = &corev1.SecurityContext{RunAsGroup: ptrTo(group)} + // Restricted init runs as the game UID. It can overwrite an existing + // root-owned, shared-group-writable file, but cannot chmod that file. + // Inspect the current group/mode before mutating so repeat starts and + // profile upgrades do not require ownership of already-correct files. + cpCmd += fmt.Sprintf(" && cd %s && find ./* -follow \\( -type f -o -type d \\) | while IFS= read -r p; do dest='%s/'\"$p\"; group=$(stat -c %%g \"$dest\") || exit 1; [ \"$group\" = %d ] || chgrp %d \"$dest\" || exit 1; mode=$(stat -c %%a \"$dest\") || exit 1; [ $((0$mode & 0020)) -ne 0 ] || chmod g+w \"$dest\" || exit 1; done", configFilesStagingPath, mountPath, group, group) } return corev1.Container{ - Name: "config-init", - Image: image, - Command: []string{"/bin/sh", "-c"}, + Name: "config-init", + Image: image, + Command: []string{"/bin/sh", "-c"}, + SecurityContext: security, // -L dereferences the kubelet's per-key symlinks; the * glob // skips the ..data/.. dot-entries of the Secret mount. Args: []string{cpCmd}, diff --git a/operator/internal/controller/gameserver_envtest_test.go b/operator/internal/controller/gameserver_envtest_test.go index 2b9265365..8c18ad7e3 100644 --- a/operator/internal/controller/gameserver_envtest_test.go +++ b/operator/internal/controller/gameserver_envtest_test.go @@ -978,8 +978,15 @@ func TestGameServer_NoExtraVolumesNoCaptureComponent(t *testing.T) { types.NamespacedName{Namespace: ns, Name: "smp"}, &ss); err != nil { return false, "statefulset: " + err.Error() } - if len(ss.Spec.Template.Spec.Volumes) != 3 { - return false, fmt.Sprintf("pod volumes = %v, want exactly [data agent-tls captures]", sprintArgs(volumeNames(ss.Spec.Template.Spec.Volumes))) + if len(ss.Spec.Template.Spec.Volumes) != 4 { + return false, fmt.Sprintf("pod volumes = %v, want exactly [data agent-tls captures agent-api-token]", sprintArgs(volumeNames(ss.Spec.Template.Spec.Volumes))) + } + token := volumeByName(ss.Spec.Template.Spec.Volumes, agentTokenVolumeName) + if token == nil || token.Projected == nil || len(token.Projected.Sources) != 3 || token.Projected.Sources[0].ServiceAccountToken == nil { + return false, "agent credentials are not a rotating projection" + } + if ss.Spec.Template.Spec.AutomountServiceAccountToken == nil || *ss.Spec.Template.Spec.AutomountServiceAccountToken { + return false, "automatic ServiceAccount credentials are exposed" } captures := volumeByName(ss.Spec.Template.Spec.Volumes, "captures") if captures == nil { @@ -991,11 +998,18 @@ func TestGameServer_NoExtraVolumesNoCaptureComponent(t *testing.T) { if captures.EmptyDir.SizeLimit == nil || captures.EmptyDir.SizeLimit.IsZero() { return false, "captures emptyDir must carry a nonzero sizeLimit" } - for _, c := range ss.Spec.Template.Spec.Containers { + for _, c := range append(ss.Spec.Template.Spec.Containers, ss.Spec.Template.Spec.InitContainers...) { + tokenMounted := false for _, vm := range c.VolumeMounts { if vm.Name == "captures" { return false, "captures volume must not be mounted on container " + c.Name } + if vm.Name == agentTokenVolumeName { + tokenMounted = true + } + } + if tokenMounted != (c.Name == "agent") { + return false, "agent token mount boundary violated by " + c.Name } } if len(ss.Spec.Template.Spec.EphemeralContainers) != 0 { diff --git a/operator/internal/controller/gameserver_files_test.go b/operator/internal/controller/gameserver_files_test.go index 815d7feea..4eb17f521 100644 --- a/operator/internal/controller/gameserver_files_test.go +++ b/operator/internal/controller/gameserver_files_test.go @@ -1,6 +1,10 @@ package controller import ( + "os" + "os/exec" + "path/filepath" + "runtime" "strings" "testing" @@ -95,12 +99,8 @@ func TestBuildConfigInitContainer_GroupWritableWithFSGroup(t *testing.T) { t.Errorf("args should not contain chmod -R, got %q", arg) } - // The whole command must be valid shell: the mount path is single-quoted - // and closed before the loop variable, so $p still expands. - want := "cp -RL " + configFilesStagingPath + "/* '/data/' && cd " + configFilesStagingPath + - " && find ./* -follow \\( -type f -o -type d \\) | while IFS= read -r p; do chmod g+w '/data/'\"$p\"; done" - if arg != want { - t.Errorf("config-init command =\n %q\nwant\n %q", arg, want) + if !strings.Contains(arg, "stat -c %g") || !strings.Contains(arg, "stat -c %a") { + t.Errorf("already-correct ownership and modes must be checked before mutation: %q", arg) } // Verify the command contains "find ./*" @@ -109,6 +109,65 @@ func TestBuildConfigInitContainer_GroupWritableWithFSGroup(t *testing.T) { } } +// The second start models files that are writable through fsGroup but cannot +// be chmodded by the init UID, as occurs on a root-to-Restricted transition. +func TestBuildConfigInitContainer_ReusesSharedFilesWithoutOwnership(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("config-init runs in Linux containers") + } + dir := t.TempDir() + staging, data := filepath.Join(dir, "staging"), filepath.Join(dir, "data") + for _, path := range []string{filepath.Join(staging, "cfg"), data} { + if err := os.MkdirAll(path, 0o755); err != nil { + t.Fatal(err) + } + } + source := filepath.Join(staging, "cfg", "profile.txt") + if err := os.WriteFile(source, []byte("first"), 0o600); err != nil { + t.Fatal(err) + } + unrelated := filepath.Join(data, "world.dat") + if err := os.WriteFile(unrelated, []byte("world"), 0o600); err != nil { + t.Fatal(err) + } + tmpl := &gameplanev1alpha1.GameTemplate{} + tmpl.Spec.Security = &gameplanev1alpha1.GameSecuritySpec{FSGroup: ptrTo(int64(os.Getgid()))} + tmpl.Spec.Storage.MountPath = data + c := buildConfigInitContainer("", tmpl) + if c.SecurityContext == nil || c.SecurityContext.RunAsGroup == nil || *c.SecurityContext.RunAsGroup != *tmpl.Spec.Security.FSGroup { + t.Fatal("config bootstrap must create files in the shared volume group") + } + script := strings.ReplaceAll(c.Args[0], configFilesStagingPath, "'"+staging+"'") + if out, err := exec.CommandContext(t.Context(), "/bin/sh", "-c", script).CombinedOutput(); err != nil { + t.Fatalf("bootstrap: %v %s", err, out) + } + // A non-owner cannot change modes or groups, even when their requested + // values are unchanged. Reject both operations on the second invocation. + bin := filepath.Join(dir, "bin") + if err := os.Mkdir(bin, 0o755); err != nil { + t.Fatal(err) + } + for _, name := range []string{"chmod", "chgrp"} { + if err := os.WriteFile(filepath.Join(bin, name), []byte("#!/bin/sh\necho unexpected ownership mutation >&2\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(source, []byte("updated"), 0o600); err != nil { + t.Fatal(err) + } + command := exec.CommandContext(t.Context(), "/bin/sh", "-c", script) + command.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH")) + if out, err := command.CombinedOutput(); err != nil { + t.Fatalf("repeat copy required ownership of writable files: %v %s", err, out) + } + if got, err := os.ReadFile(filepath.Join(data, "cfg", "profile.txt")); err != nil || string(got) != "updated" { + t.Fatalf("config was not replaced: %q %v", got, err) + } + if info, err := os.Stat(unrelated); err != nil || info.Mode().Perm() != 0o600 { + t.Fatalf("unrelated data permissions changed: %v %v", info, err) + } +} + func TestBuildConfigInitContainer_NoChmodWithoutFSGroup(t *testing.T) { // Test 1: no Security block at all tmpl1 := &gameplanev1alpha1.GameTemplate{} diff --git a/operator/internal/controller/networkcapture_controller.go b/operator/internal/controller/networkcapture_controller.go index 4b1f46447..a7ada7fa4 100644 --- a/operator/internal/controller/networkcapture_controller.go +++ b/operator/internal/controller/networkcapture_controller.go @@ -127,9 +127,15 @@ const captureFileCleanupBudget = 5 * time.Minute // over mTLS through the -agent Service, and monitors completion. type NetworkCaptureReconciler struct { client.Client + APIReader client.Reader Scheme *runtime.Scheme SidecarClient SidecarCaptureClient CaptureEnabled bool + RestrictedWorkloads bool + DiagnosticsNamespace string + DiagnosticsStorageClass string + AgentCASecretName string + AgentCASecretNamespace string CaptureSidecarImage string CaptureDefaultMaxDurationSeconds int64 CaptureDefaultMaxSizeBytes int64 @@ -166,6 +172,11 @@ func (r *NetworkCaptureReconciler) reconcile(ctx context.Context, req ctrl.Reque if err := r.Get(ctx, req.NamespacedName, &nc); err != nil { return ctrl.Result{}, client.IgnoreNotFound(err) } + if use, err := r.useNodeCapture(ctx, &nc); err != nil { + return ctrl.Result{}, err + } else if use { + return r.reconcileNodeCapture(ctx, &nc) + } // A user asked for this capture to stop (the API's StopNetworkCapture // set stopRequestedAnnotation). Stop the sidecar first, then complete diff --git a/operator/internal/controller/networkcapture_node.go b/operator/internal/controller/networkcapture_node.go new file mode 100644 index 000000000..ab6231797 --- /dev/null +++ b/operator/internal/controller/networkcapture_node.go @@ -0,0 +1,570 @@ +package controller + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const nodeCaptureNamespaceAnnotation = "gameplane.local/capture-node-namespace" +const nodeCaptureNodeAnnotation = "gameplane.local/capture-node" +const nodeCaptureContainerAnnotation = "gameplane.local/capture-container-id" +const nodeCaptureFinishedAnnotation = "gameplane.local/capture-worker-finished" +const nodeCaptureFinalizer = "gameplane.local/node-capture-cleanup" +const nodeCaptureUIDLabel = "gameplane.local/capture-uid" +const nodeCaptureComponentLabel = "gameplane.local/capture-component" + +func nodeCaptureName(nc *gameplanev1alpha1.NetworkCapture) string { return "capture-" + string(nc.UID) } +func (r *NetworkCaptureReconciler) nodeReader() client.Reader { + if r.APIReader != nil { + return r.APIReader + } + return r.Client +} + +func (r *NetworkCaptureReconciler) useNodeCapture(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture) (bool, error) { + if nc.Annotations[nodeCaptureNamespaceAnnotation] != "" || controllerutil.ContainsFinalizer(nc, nodeCaptureFinalizer) { + return true, nil + } + // A profile/opt-in change must not reinterpret an already-started legacy + // capture as a new node worker or abandon its stop/retention protocol. + if nc.Annotations[capturePodUIDAnnotation] != "" || nc.Status.Phase == gameplanev1alpha1.CapturePhaseRunning || isTerminalCapturePhase(nc.Status.Phase) { + return false, nil + } + if r.RestrictedWorkloads { + return true, nil + } + var gs gameplanev1alpha1.GameServer + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: nc.Namespace, Name: nc.Spec.ServerRef.Name}, &gs); err != nil { + return false, client.IgnoreNotFound(err) + } + if gs.Spec.Capture != nil && gs.Spec.Capture.NodeDiagnostic { + return true, nil + } + var pod corev1.Pod + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: nc.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + return false, client.IgnoreNotFound(err) + } + return pod.Annotations[workloadSecurityLabel] == "restricted", nil +} + +func nodeGameContainerID(pod *corev1.Pod) (string, error) { + if pod.Spec.NodeName == "" || pod.UID == "" || pod.Spec.HostNetwork || pod.DeletionTimestamp != nil { + return "", errors.New("node capture requires a live scheduled Pod with its own network namespace") + } + for _, c := range pod.Status.ContainerStatuses { + if c.Name == "game" && c.State.Running != nil { + id := strings.TrimPrefix(c.ContainerID, "containerd://") + decoded, err := hex.DecodeString(id) + if err != nil || len(decoded) != 32 || !strings.HasPrefix(c.ContainerID, "containerd://") { + return "", errors.New("node capture currently supports Linux containerd targets only") + } + return c.ContainerID, nil + } + } + return "", errors.New("game container is not running") +} + +func nodeAffinity(name string) *corev1.Affinity { + return &corev1.Affinity{NodeAffinity: &corev1.NodeAffinity{RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{NodeSelectorTerms: []corev1.NodeSelectorTerm{{MatchFields: []corev1.NodeSelectorRequirement{{Key: "metadata.name", Operator: corev1.NodeSelectorOpIn, Values: []string{name}}}}}}}} +} + +func nodeWorkerPodSpec(nc *gameplanev1alpha1.NetworkCapture, gs *gameplanev1alpha1.GameServer, pod *corev1.Pod, image, pvc, filter string, duration, size int64) (corev1.PodSpec, error) { + id, err := nodeGameContainerID(pod) + if err != nil { + return corev1.PodSpec{}, err + } + if duration < 1 || duration > 3600 || size < 1 || size > 900*1024*1024 || filter == "" { + return corev1.PodSpec{}, errors.New("node capture requires a filter, duration <= 3600s and size <= 900Mi") + } + raw, err := json.Marshal(map[string]any{"serverUID": string(gs.UID), "captureUID": string(nc.UID), "captureID": nc.Name, "podUID": string(pod.UID), "containerID": id, "node": pod.Spec.NodeName, "filter": filter, "durationSeconds": duration, "maxSizeBytes": size}) + if err != nil { + return corev1.PodSpec{}, err + } + return corev1.PodSpec{ + HostPID: true, RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: ptrTo(false), Affinity: nodeAffinity(pod.Spec.NodeName), NodeSelector: map[string]string{"kubernetes.io/os": "linux"}, + Tolerations: append([]corev1.Toleration(nil), pod.Spec.Tolerations...), + TerminationGracePeriodSeconds: ptrTo(int64(10)), + SecurityContext: &corev1.PodSecurityContext{RunAsUser: ptrTo(int64(0)), RunAsGroup: ptrTo(int64(65532)), FSGroup: ptrTo(int64(65532))}, + Volumes: []corev1.Volume{{Name: "captures", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvc}}}}, + Containers: []corev1.Container{{Name: "capture", Image: image, Command: []string{"/capture-tool"}, Args: []string{"--node-worker"}, Env: []corev1.EnvVar{{Name: "NODE_CAPTURE_CONFIG", Value: string(raw)}}, VolumeMounts: []corev1.VolumeMount{{Name: "captures", MountPath: "/tmp/captures"}}, + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("50m"), corev1.ResourceMemory: resource.MustParse("64Mi")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1"), corev1.ResourceMemory: resource.MustParse("256Mi")}}, + SecurityContext: &corev1.SecurityContext{RunAsUser: ptrTo(int64(0)), ReadOnlyRootFilesystem: ptrTo(true), AllowPrivilegeEscalation: ptrTo(false), Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}, Add: []corev1.Capability{"SYS_ADMIN", "SYS_PTRACE", "NET_RAW"}}, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeUnconfined}}, + }}, + }, nil +} + +// Each capture gets a same-namespace anchor. Cross-namespace ownerReferences +// cannot link diagnostics to the NetworkCapture; its finalizer deletes this +// anchor in the recorded namespace and waits for foreground GC to finish. +func (r *NetworkCaptureReconciler) createNodeObject(ctx context.Context, obj client.Object, anchor *corev1.ConfigMap, nc *gameplanev1alpha1.NetworkCapture) error { + obj.SetNamespace(anchor.Namespace) + labels := obj.GetLabels() + if labels == nil { + labels = map[string]string{} + } + labels[nodeCaptureUIDLabel] = string(nc.UID) + obj.SetLabels(labels) + if err := controllerutil.SetControllerReference(anchor, obj, r.Scheme); err != nil { + return err + } + if err := r.Create(ctx, obj); err != nil { + if !apierrors.IsAlreadyExists(err) { + return err + } + live, ok := obj.DeepCopyObject().(client.Object) + if !ok { + return errors.New("diagnostic resource is not a Kubernetes object") + } + if err := r.nodeReader().Get(ctx, client.ObjectKeyFromObject(obj), live); err != nil { + return err + } + owner := metav1.GetControllerOf(live) + if owner == nil || owner.UID != anchor.UID || live.GetLabels()[nodeCaptureUIDLabel] != string(nc.UID) { + return errors.New("diagnostic resource identity collision") + } + } + return nil +} + +func (r *NetworkCaptureReconciler) ensureNodeArtifacts(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture, gs *gameplanev1alpha1.GameServer, pod *corev1.Pod) (*corev1.ConfigMap, error) { + ns, name := nc.Annotations[nodeCaptureNamespaceAnnotation], nodeCaptureName(nc) + anchor := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns, Labels: map[string]string{nodeCaptureUIDLabel: string(nc.UID)}}} + if err := r.Create(ctx, anchor); err != nil { + if !apierrors.IsAlreadyExists(err) { + return nil, err + } + if err := r.nodeReader().Get(ctx, client.ObjectKeyFromObject(anchor), anchor); err != nil { + return nil, err + } + if anchor.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return nil, errors.New("diagnostic anchor identity collision") + } + } + pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name}, Spec: corev1.PersistentVolumeClaimSpec{AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, Resources: corev1.VolumeResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}}}} + if r.DiagnosticsStorageClass != "" { + pvc.Spec.StorageClassName = &r.DiagnosticsStorageClass + } + if err := r.createNodeObject(ctx, pvc, anchor, nc); err != nil { + return nil, err + } + if err := r.ensureNodeCertificate(ctx, anchor, nc); err != nil { + return nil, err + } + image := r.CaptureSidecarImage + if image == "" { + image = DefaultCaptureSidecarImage + } + files := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name + "-files", Labels: map[string]string{nodeCaptureComponentLabel: "artifacts"}}, Spec: corev1.PodSpec{ + AutomountServiceAccountToken: ptrTo(false), Affinity: nodeAffinity(pod.Spec.NodeName), + Tolerations: append([]corev1.Toleration(nil), pod.Spec.Tolerations...), + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: ptrTo(true), RunAsUser: ptrTo(int64(65532)), RunAsGroup: ptrTo(int64(65532)), FSGroup: ptrTo(int64(65532)), SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + Volumes: []corev1.Volume{{Name: "captures", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: name}}}, {Name: "tls", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: name + "-tls"}}}}, + Containers: []corev1.Container{{Name: "artifacts", Image: image, Command: []string{"/capture-tool"}, Args: []string{"--artifacts-only"}, Env: []corev1.EnvVar{{Name: "GAMEPLANE_SERVER_UID", Value: string(gs.UID)}}, Ports: []corev1.ContainerPort{{Name: "capture", ContainerPort: 9091}}, VolumeMounts: []corev1.VolumeMount{{Name: "captures", MountPath: "/tmp/captures"}, {Name: "tls", MountPath: "/etc/tls", ReadOnly: true}}, + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("10m"), corev1.ResourceMemory: resource.MustParse("32Mi")}, Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("500m"), corev1.ResourceMemory: resource.MustParse("128Mi")}}, + SecurityContext: &corev1.SecurityContext{RunAsNonRoot: ptrTo(true), ReadOnlyRootFilesystem: ptrTo(true), AllowPrivilegeEscalation: ptrTo(false), Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}}, + }}, + }} + if err := r.createNodeObject(ctx, files, anchor, nc); err != nil { + return nil, err + } + svc := &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: name + "-agent"}, Spec: corev1.ServiceSpec{Selector: map[string]string{nodeCaptureUIDLabel: string(nc.UID), nodeCaptureComponentLabel: "artifacts"}, Ports: []corev1.ServicePort{{Name: "capture", Port: 9091, TargetPort: intstr.FromInt32(9091)}}}} + if err := r.createNodeObject(ctx, svc, anchor, nc); err != nil { + return nil, err + } + return anchor, nil +} + +func (r *NetworkCaptureReconciler) ensureNodeCertificate(ctx context.Context, anchor *corev1.ConfigMap, nc *gameplanev1alpha1.NetworkCapture) error { + var ca corev1.Secret + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: r.AgentCASecretNamespace, Name: r.AgentCASecretName}, &ca); err != nil { + return err + } + fingerprint := caCertFingerprint(ca.Data["ca.crt"]) + key := types.NamespacedName{Namespace: anchor.Namespace, Name: anchor.Name + "-tls"} + var existing corev1.Secret + err := r.nodeReader().Get(ctx, key, &existing) + if err == nil { + owner := metav1.GetControllerOf(&existing) + if owner == nil || owner.UID != anchor.UID { + return errors.New("diagnostic certificate identity collision") + } + if existing.Annotations["gameplane.local/ca-fingerprint"] == fingerprint { + return nil + } + } else if !apierrors.IsNotFound(err) { + return err + } + cert, keyCA, err := parseAgentCA(ca.Data["ca.crt"], ca.Data["ca.key"]) + if err != nil { + return err + } + fake := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: anchor.Name, Namespace: anchor.Namespace}} + crt, private, err := signAgentServerCert(cert, keyCA, fake) + if err != nil { + return err + } + sec := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace, Annotations: map[string]string{"gameplane.local/ca-fingerprint": fingerprint}}, Type: corev1.SecretTypeTLS, Data: map[string][]byte{"tls.crt": crt, "tls.key": private, "ca.crt": ca.Data["ca.crt"]}} + if existing.UID != "" { + existing.Data = sec.Data + existing.Annotations = sec.Annotations + return r.Update(ctx, &existing) + } + return r.createNodeObject(ctx, sec, anchor, nc) +} + +func (r *NetworkCaptureReconciler) deleteNodeWorkers(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture) (bool, error) { + ns := nc.Annotations[nodeCaptureNamespaceAnnotation] + if ns == "" { + return true, nil + } + var job batchv1.Job + err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: nodeCaptureName(nc) + "-worker"}, &job) + if err == nil { + if job.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return false, errors.New("diagnostic worker identity collision") + } + if job.DeletionTimestamp == nil { + if err := r.Delete(ctx, &job, client.PropagationPolicy(metav1.DeletePropagationForeground), client.Preconditions{UID: &job.UID}); err != nil && !apierrors.IsNotFound(err) { + return false, err + } + } + } else if !apierrors.IsNotFound(err) { + return false, err + } + var pods corev1.PodList + if err := r.nodeReader().List(ctx, &pods, client.InNamespace(ns), client.MatchingLabels{nodeCaptureUIDLabel: string(nc.UID), nodeCaptureComponentLabel: "worker"}); err != nil { + return false, err + } + // Independently drain orphan worker Pods too, including a Job deleted by + // another actor with orphan propagation. Never free a capture lock early. + for i := range pods.Items { + p := &pods.Items[i] + if p.DeletionTimestamp == nil { + if err := r.Delete(ctx, p, client.Preconditions{UID: &p.UID}); err != nil && !apierrors.IsNotFound(err) { + return false, err + } + } + } + return apierrors.IsNotFound(err) && len(pods.Items) == 0, nil +} + +func (r *NetworkCaptureReconciler) cleanupNodeCapture(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture) (bool, error) { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil || !drained { + return false, err + } + ns := nc.Annotations[nodeCaptureNamespaceAnnotation] + if ns == "" { + return true, nil + } + var anchor corev1.ConfigMap + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: nodeCaptureName(nc)}, &anchor); err != nil { + return apierrors.IsNotFound(err), client.IgnoreNotFound(err) + } + if anchor.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return false, errors.New("diagnostic anchor identity collision") + } + if anchor.DeletionTimestamp == nil { + if err := r.Delete(ctx, &anchor, client.PropagationPolicy(metav1.DeletePropagationForeground), client.Preconditions{UID: &anchor.UID}); err != nil && !apierrors.IsNotFound(err) { + return false, err + } + } + return false, nil +} + +func (r *NetworkCaptureReconciler) nodeFail(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture, message string) (ctrl.Result, error) { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return r.fail(ctx, nc, message) +} + +func (r *NetworkCaptureReconciler) reconcileNodeCapture(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture) (ctrl.Result, error) { + if nc.DeletionTimestamp != nil { + clean, err := r.cleanupNodeCapture(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !clean { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + if err := r.releaseNodeCaptureLock(ctx, nc, ptrTo(metav1.Now())); err != nil { + return ctrl.Result{}, err + } + controllerutil.RemoveFinalizer(nc, nodeCaptureFinalizer) + return ctrl.Result{}, r.Update(ctx, nc) + } + if isTerminalCapturePhase(nc.Status.Phase) { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + if nc.Status.CompletionTime == nil { + nc.Status.CompletionTime = ptrTo(metav1.Now()) + return ctrl.Result{RequeueAfter: retentionPollInterval}, r.Status().Update(ctx, nc) + } + ttl := effectiveRetentionSeconds(nc.Spec.TTLSecondsAfterFinished, r.CaptureDefaultRetentionSeconds, r.CaptureMaxRetentionSeconds) + if time.Now().Before(nc.Status.CompletionTime.Add(time.Duration(ttl) * time.Second)) { + // Retained artifacts need certificate rotation even after the target + // game Pod has gone away. No target or worker is recreated here. + if ns := nc.Annotations[nodeCaptureNamespaceAnnotation]; ns != "" { + var anchor corev1.ConfigMap + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: ns, Name: nodeCaptureName(nc)}, &anchor); err != nil { + if !apierrors.IsNotFound(err) { + return ctrl.Result{}, err + } + } else if anchor.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return ctrl.Result{}, errors.New("diagnostic anchor identity collision") + } else if err := r.ensureNodeCertificate(ctx, &anchor, nc); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: retentionPollInterval}, nil + } + return ctrl.Result{}, r.Delete(ctx, nc, client.Preconditions{UID: &nc.UID}) + } + var gs gameplanev1alpha1.GameServer + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: nc.Namespace, Name: nc.Spec.ServerRef.Name}, &gs); err != nil { + if apierrors.IsNotFound(err) { + return r.nodeFail(ctx, nc, "target GameServer no longer exists") + } + return ctrl.Result{}, err + } + if owner := metav1.GetControllerOf(nc); owner != nil && owner.UID != gs.UID { + return r.nodeFail(ctx, nc, "target GameServer identity changed") + } + if err := r.ensureOwnerReference(ctx, nc, &gs); err != nil { + return ctrl.Result{}, err + } + if gs.DeletionTimestamp != nil { + return r.nodeFail(ctx, nc, "target GameServer is terminating") + } + if !r.CaptureEnabled || r.DiagnosticsNamespace == "" || gs.Spec.Capture == nil || !gs.Spec.Capture.Enabled || !gs.Spec.Capture.NodeDiagnostic { + return r.nodeFail(ctx, nc, "node diagnostics require cluster deployment opt-in and administrator spec.capture.nodeDiagnostic opt-in on this server") + } + if _, stop := nc.Annotations[stopRequestedAnnotation]; stop { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return r.finishNodeCapture(ctx, nc, true) + } + if nc.Annotations[nodeCaptureFinishedAnnotation] != "" { + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return r.finishNodeCapture(ctx, nc, false) + } + var pod corev1.Pod + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + if apierrors.IsNotFound(err) { + return r.nodeFail(ctx, nc, "target game Pod no longer exists") + } + return ctrl.Result{}, err + } + var ss appsv1.StatefulSet + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: gs.Namespace, Name: gs.Name}, &ss); err != nil { + return ctrl.Result{}, err + } + ssOwner, podOwner := metav1.GetControllerOf(&ss), metav1.GetControllerOf(&pod) + if ssOwner == nil || ssOwner.UID != gs.UID || podOwner == nil || podOwner.UID != ss.UID { + return r.nodeFail(ctx, nc, "target Pod is not controlled by this GameServer") + } + containerID, err := nodeGameContainerID(&pod) + if err != nil { + return r.nodeFail(ctx, nc, err.Error()) + } + if bound := nc.Annotations[capturePodUIDAnnotation]; bound != "" && (bound != string(pod.UID) || nc.Annotations[nodeCaptureNodeAnnotation] != pod.Spec.NodeName || nc.Annotations[nodeCaptureContainerAnnotation] != containerID) { + return r.nodeFail(ctx, nc, "target Pod, node or container identity changed") + } + if nc.Annotations[nodeCaptureNamespaceAnnotation] == "" { + if r.DiagnosticsNamespace == gs.Namespace { + return r.nodeFail(ctx, nc, "node diagnostics namespace must be separate from the games namespace") + } + var namespace corev1.Namespace + if err := r.nodeReader().Get(ctx, types.NamespacedName{Name: r.DiagnosticsNamespace}, &namespace); err != nil { + return ctrl.Result{}, err + } + if namespace.Labels["pod-security.kubernetes.io/enforce"] != "privileged" { + return r.nodeFail(ctx, nc, "diagnostics namespace must explicitly permit privileged node diagnostics") + } + var captures gameplanev1alpha1.NetworkCaptureList + if err := r.nodeReader().List(ctx, &captures, client.InNamespace(nc.Namespace)); err != nil { + return ctrl.Result{}, err + } + for i := range captures.Items { + o := &captures.Items[i] + if o.UID != nc.UID && o.Spec.ServerRef.Name == gs.Name && !isTerminalCapturePhase(o.Status.Phase) && (o.CreationTimestamp.Before(&nc.CreationTimestamp) || (o.CreationTimestamp.Equal(&nc.CreationTimestamp) && o.Name < nc.Name)) { + return r.nodeFail(ctx, nc, "another capture is already in progress on this GameServer") + } + } + if nc.Annotations == nil { + nc.Annotations = map[string]string{} + } + nc.Annotations[nodeCaptureNamespaceAnnotation] = r.DiagnosticsNamespace + nc.Annotations[capturePodUIDAnnotation] = string(pod.UID) + nc.Annotations[nodeCaptureNodeAnnotation] = pod.Spec.NodeName + nc.Annotations[nodeCaptureContainerAnnotation] = containerID + controllerutil.AddFinalizer(nc, nodeCaptureFinalizer) + if err := r.Update(ctx, nc); err != nil { + return ctrl.Result{}, err + } + } + filter, failMessage, err := r.captureFilter(ctx, nc, &gs) + if err != nil { + return ctrl.Result{}, err + } + if failMessage != "" { + return r.nodeFail(ctx, nc, failMessage) + } + duration, size := r.CaptureDefaultMaxDurationSeconds, r.CaptureDefaultMaxSizeBytes + if nc.Spec.MaxDuration != nil { + duration = int64(nc.Spec.MaxDuration.Seconds()) + } + if nc.Spec.MaxSize != nil { + size = nc.Spec.MaxSize.Value() + } + image := r.CaptureSidecarImage + if image == "" { + image = DefaultCaptureSidecarImage + } + spec, err := nodeWorkerPodSpec(nc, &gs, &pod, image, nodeCaptureName(nc), *filter, duration, size) + if err != nil { + return r.nodeFail(ctx, nc, err.Error()) + } + anchor, err := r.ensureNodeArtifacts(ctx, nc, &gs, &pod) + if err != nil { + return ctrl.Result{}, err + } + var job batchv1.Job + jobKey := types.NamespacedName{Namespace: anchor.Namespace, Name: anchor.Name + "-worker"} + err = r.nodeReader().Get(ctx, jobKey, &job) + if apierrors.IsNotFound(err) { + if nc.Status.Phase == gameplanev1alpha1.CapturePhaseRunning { + return r.nodeFail(ctx, nc, "node diagnostic worker disappeared") + } + job = batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: jobKey.Name}, Spec: batchv1.JobSpec{BackoffLimit: ptrTo(int32(0)), ActiveDeadlineSeconds: ptrTo(duration + 30), Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{nodeCaptureUIDLabel: string(nc.UID), nodeCaptureComponentLabel: "worker"}}, Spec: spec}}} + if err := r.createNodeObject(ctx, &job, anchor, nc); err != nil { + return ctrl.Result{}, err + } + } else if err != nil { + return ctrl.Result{}, err + } + if job.Labels[nodeCaptureUIDLabel] != string(nc.UID) { + return ctrl.Result{}, errors.New("diagnostic worker identity collision") + } + if job.Status.Succeeded > 0 || job.Status.Failed > 0 { + if job.Status.Failed > 0 { + return r.nodeFail(ctx, nc, "node diagnostic worker failed; verify Linux containerd cgroups, node capabilities and storage") + } + nc.Annotations[nodeCaptureFinishedAnnotation] = time.Now().UTC().Format(time.RFC3339) + if err := r.Update(ctx, nc); err != nil { + return ctrl.Result{}, err + } + drained, err := r.deleteNodeWorkers(ctx, nc) + if err != nil { + return ctrl.Result{}, err + } + if !drained { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + return r.finishNodeCapture(ctx, nc, false) + } + if nc.Status.Phase != gameplanev1alpha1.CapturePhaseRunning { + nc.Status.Phase = gameplanev1alpha1.CapturePhaseRunning + nc.Status.StartTime = ptrTo(metav1.Now()) + nc.Status.Message = "node diagnostic capture running" + if err := r.patchGameServerActiveCapture(ctx, &gs, nc, nil); err != nil { + return ctrl.Result{}, err + } + if err := r.Status().Update(ctx, nc); err != nil { + return ctrl.Result{}, err + } + } + return ctrl.Result{RequeueAfter: time.Second}, nil +} + +func (r *NetworkCaptureReconciler) finishNodeCapture(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture, stopped bool) (ctrl.Result, error) { + if nc.Annotations[nodeCaptureNamespaceAnnotation] == "" { + return r.fail(ctx, nc, "capture stopped before the node diagnostic worker started") + } + phase, packets, bytes, message, err := r.SidecarClient.GetCaptureStatus(ctx, nc.Annotations[nodeCaptureNamespaceAnnotation], nodeCaptureName(nc), nc.Name, captureOwnerUID(nc.OwnerReferences, nc.Spec.ServerRef.Name), string(nc.UID)) + if err != nil { + if stopped { + return r.fail(ctx, nc, "capture stopped before a completed artifact was available") + } + finished, parseErr := time.Parse(time.RFC3339, nc.Annotations[nodeCaptureFinishedAnnotation]) + if parseErr != nil || time.Since(finished) > 2*time.Minute { + return r.fail(ctx, nc, "node diagnostic worker finished without an available artifact") + } + return ctrl.Result{RequeueAfter: time.Second}, nil + } + nc.Status.Phase = gameplanev1alpha1.CapturePhaseCompleted + if phase != "completed" { + nc.Status.Phase = gameplanev1alpha1.CapturePhaseFailed + } + nc.Status.PacketsWritten = packets + nc.Status.BytesWritten = resource.NewQuantity(bytes, resource.BinarySI) + nc.Status.Message = message + nc.Status.CompletionTime = ptrTo(metav1.Now()) + if err := r.releaseNodeCaptureLock(ctx, nc, nc.Status.CompletionTime); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: retentionPollInterval}, r.Status().Update(ctx, nc) +} + +// A direct CR deletion can arrive before terminal status. Release only after +// worker drain and only for this capture/server incarnation; optimistic locking +// prevents a concurrent successor from losing its newly acquired pointer. +func (r *NetworkCaptureReconciler) releaseNodeCaptureLock(ctx context.Context, nc *gameplanev1alpha1.NetworkCapture, terminalTime *metav1.Time) error { + var live gameplanev1alpha1.NetworkCapture + if err := r.nodeReader().Get(ctx, client.ObjectKeyFromObject(nc), &live); err != nil { + return client.IgnoreNotFound(err) + } + if live.UID != nc.UID { + return nil + } + var gs gameplanev1alpha1.GameServer + if err := r.nodeReader().Get(ctx, types.NamespacedName{Namespace: nc.Namespace, Name: nc.Spec.ServerRef.Name}, &gs); err != nil { + return client.IgnoreNotFound(err) + } + if captureOwnerUID(nc.OwnerReferences, nc.Spec.ServerRef.Name) != string(gs.UID) || gs.Status.Capture == nil || gs.Status.Capture.ActiveCapture == nil || *gs.Status.Capture.ActiveCapture != nc.Name { + return nil + } + base := gs.DeepCopy() + gs.Status.Capture.ActiveCapture = nil + gs.Status.Capture.LastCaptureTime = terminalTime + return r.Status().Patch(ctx, &gs, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) +} diff --git a/operator/internal/controller/networkcapture_node_test.go b/operator/internal/controller/networkcapture_node_test.go new file mode 100644 index 000000000..ddd9a333d --- /dev/null +++ b/operator/internal/controller/networkcapture_node_test.go @@ -0,0 +1,213 @@ +package controller + +import ( + "context" + "reflect" + "strings" + "testing" + "time" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestNodeCaptureWorkerBoundAndNoCredentials(t *testing.T) { + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap-test", UID: "capture-uid"}} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "server-uid"}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{UID: "11111111-2222-3333-4444-555555555555"}, Spec: corev1.PodSpec{NodeName: "target-node"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "game", ContainerID: "containerd://" + strings.Repeat("a", 64), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}} + spec, err := nodeWorkerPodSpec(nc, gs, pod, "image", "pvc", "tcp port 25565", 10, 1024) + if err != nil { + t.Fatal(err) + } + if !spec.HostPID || spec.HostNetwork || spec.AutomountServiceAccountToken == nil || *spec.AutomountServiceAccountToken || spec.RestartPolicy != corev1.RestartPolicyNever { + t.Fatalf("unsafe worker isolation: %+v", spec) + } + if len(spec.Volumes) != 1 || spec.Volumes[0].PersistentVolumeClaim == nil { + t.Fatalf("worker acquired host paths or credentials: %+v", spec.Volumes) + } + if len(spec.Containers) != 1 || *spec.Containers[0].SecurityContext.RunAsUser != 0 || *spec.Containers[0].SecurityContext.AllowPrivilegeEscalation { + t.Fatalf("unexpected worker privileges: %+v", spec.Containers) + } + node := spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchFields[0] + if node.Key != "metadata.name" || len(node.Values) != 1 || node.Values[0] != "target-node" { + t.Fatalf("worker not bound to exact node: %+v", node) + } + pod.Spec.HostNetwork = true + if _, err := nodeWorkerPodSpec(nc, gs, pod, "image", "pvc", "tcp port 25565", 10, 1024); err == nil { + t.Fatal("host network target accepted") + } + pod.Spec.HostNetwork = false + pod.Status.ContainerStatuses[0].ContainerID = "docker://" + strings.Repeat("a", 64) + if _, err := nodeWorkerPodSpec(nc, gs, pod, "image", "pvc", "tcp port 25565", 10, 1024); err == nil { + t.Fatal("unsupported runtime accepted") + } +} + +func nodeReconcilerFixture(t *testing.T, objects ...client.Object) *NetworkCaptureReconciler { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{corev1.AddToScheme, appsv1.AddToScheme, batchv1.AddToScheme, gameplanev1alpha1.AddToScheme} { + if err := add(scheme); err != nil { + t.Fatal(err) + } + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(&gameplanev1alpha1.NetworkCapture{}, &gameplanev1alpha1.GameServer{}).WithObjects(objects...).Build() + return &NetworkCaptureReconciler{Client: c, APIReader: c, Scheme: scheme, CaptureEnabled: true, RestrictedWorkloads: true, DiagnosticsNamespace: "diagnostics"} +} + +type staleCaptureListClient struct{ client.Client } + +func (c staleCaptureListClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if captures, ok := list.(*gameplanev1alpha1.NetworkCaptureList); ok { + captures.Items = nil + return nil + } + return c.Client.List(ctx, list, opts...) +} + +func TestNodeCaptureExclusivityUsesLiveList(t *testing.T) { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "server"}, Spec: gameplanev1alpha1.GameServerSpec{Capture: &gameplanev1alpha1.CaptureConfiguration{Enabled: true, NodeDiagnostic: true}}} + owner := metav1.OwnerReference{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: gs.Name, UID: gs.UID, Controller: ptrTo(true)} + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "newer", Namespace: "games", UID: "newer", CreationTimestamp: metav1.NewTime(time.Now()), OwnerReferences: []metav1.OwnerReference{owner}}} + nc.Spec.ServerRef.Name = gs.Name + older := nc.DeepCopy() + older.Name = "older" + older.UID = "older" + older.CreationTimestamp = metav1.NewTime(time.Now().Add(-time.Minute)) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "statefulset", OwnerReferences: []metav1.OwnerReference{owner}}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "game-0", Namespace: "games", UID: "pod", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: ss.Name, UID: ss.UID, Controller: ptrTo(true)}}}, Spec: corev1.PodSpec{NodeName: "node"}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "game", ContainerID: "containerd://" + strings.Repeat("a", 64), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}} + ns := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "diagnostics", Labels: map[string]string{"pod-security.kubernetes.io/enforce": "privileged"}}} + r := nodeReconcilerFixture(t, gs, nc, older, ss, pod, ns) + r.Client = staleCaptureListClient{r.Client} + if _, err := r.reconcileNodeCapture(t.Context(), nc); err != nil { + t.Fatal(err) + } + var live gameplanev1alpha1.NetworkCapture + if err := r.Get(t.Context(), client.ObjectKeyFromObject(nc), &live); err != nil { + t.Fatal(err) + } + if live.Status.Phase != gameplanev1alpha1.CapturePhaseFailed || !strings.Contains(live.Status.Message, "another capture") { + t.Fatalf("stale cache bypassed exclusivity: %+v", live.Status) + } + if live.Annotations[nodeCaptureNamespaceAnnotation] != "" { + t.Fatal("new worker bound despite older request") + } +} + +func TestDeletingNodeCaptureReleasesOnlyItsOwnLock(t *testing.T) { + for _, test := range []struct { + name, active, serverUID string + clear bool + }{{"matching", "cap", "server", true}, {"other-capture", "other", "server", false}, {"replaced-server", "cap", "replacement", false}} { + t.Run(test.name, func(t *testing.T) { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: types.UID(test.serverUID)}, Status: gameplanev1alpha1.GameServerStatus{Capture: &gameplanev1alpha1.CaptureStatus{ActiveCapture: ptrTo(test.active)}}} + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap", Namespace: "games", UID: "capture", Finalizers: []string{nodeCaptureFinalizer}, DeletionTimestamp: ptrTo(metav1.Now()), Annotations: map[string]string{nodeCaptureNamespaceAnnotation: "diagnostics"}, OwnerReferences: []metav1.OwnerReference{{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: "game", UID: "server", Controller: ptrTo(true)}}}} + nc.Spec.ServerRef.Name = gs.Name + r := nodeReconcilerFixture(t, gs, nc) + if _, err := r.reconcileNodeCapture(t.Context(), nc); err != nil { + t.Fatal(err) + } + var live gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &live); err != nil { + t.Fatal(err) + } + if (live.Status.Capture.ActiveCapture == nil) != test.clear { + t.Fatalf("wrong lock release: %+v", live.Status.Capture) + } + }) + } +} + +func TestNodeWorkerCopiesTargetTolerations(t *testing.T) { + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{UID: "pod"}, Spec: corev1.PodSpec{NodeName: "tainted-node", Tolerations: []corev1.Toleration{{Key: "game-nodes", Operator: corev1.TolerationOpEqual, Value: "dedicated", Effect: corev1.TaintEffectNoSchedule}}}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "game", ContainerID: "containerd://" + strings.Repeat("a", 64), State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}} + spec, err := nodeWorkerPodSpec(&gameplanev1alpha1.NetworkCapture{}, &gameplanev1alpha1.GameServer{}, pod, "image", "pvc", "tcp", 10, 1024) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(spec.Tolerations, pod.Spec.Tolerations) { + t.Fatal("worker lost target tolerations") + } +} + +func TestNodeCaptureRefusesWithoutServerOptIn(t *testing.T) { + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Capture: &gameplanev1alpha1.CaptureConfiguration{Enabled: true}}} + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap", Namespace: "games", UID: "capture-uid"}} + nc.Spec.ServerRef.Name = gs.Name + r := nodeReconcilerFixture(t, gs, nc) + if _, err := r.reconcileNodeCapture(t.Context(), nc); err != nil { + t.Fatal(err) + } + var live gameplanev1alpha1.NetworkCapture + if err := r.Get(t.Context(), client.ObjectKeyFromObject(nc), &live); err != nil { + t.Fatal(err) + } + if live.Status.Phase != gameplanev1alpha1.CapturePhaseFailed || !strings.Contains(live.Status.Message, "administrator") { + t.Fatalf("opt-in refusal missing: %+v", live.Status) + } + var jobs batchv1.JobList + if err := r.List(t.Context(), &jobs); err != nil || len(jobs.Items) != 0 { + t.Fatalf("unauthorized worker: %v %+v", err, jobs) + } +} + +func TestNodeCaptureDrainsOrphanWorkersBeforeCleanup(t *testing.T) { + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap", Namespace: "games", UID: "capture-uid", Annotations: map[string]string{nodeCaptureNamespaceAnnotation: "diagnostics"}}} + labels := map[string]string{nodeCaptureUIDLabel: string(nc.UID), nodeCaptureComponentLabel: "worker"} + anchor := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: nodeCaptureName(nc), Namespace: "diagnostics", UID: "anchor", Labels: labels}} + worker := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "orphan-worker", Namespace: "diagnostics", UID: "worker", Labels: labels}} + other := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "other-worker", Namespace: "diagnostics", UID: "other", Labels: map[string]string{nodeCaptureUIDLabel: "other", nodeCaptureComponentLabel: "worker"}}} + r := nodeReconcilerFixture(t, nc, anchor, worker, other) + ctx := context.Background() + if clean, err := r.cleanupNodeCapture(ctx, nc); err != nil || clean { + t.Fatalf("cleanup freed lock before observed worker deletion: %t %v", clean, err) + } + var live corev1.ConfigMap + if err := r.Get(ctx, client.ObjectKeyFromObject(anchor), &live); err != nil { + t.Fatal("anchor deleted before draining worker") + } + if clean, err := r.cleanupNodeCapture(ctx, nc); err != nil || clean { + t.Fatalf("cleanup did not wait for anchor GC: %t %v", clean, err) + } + if clean, err := r.cleanupNodeCapture(ctx, nc); err != nil || !clean { + t.Fatalf("cleanup failed to finish: %t %v", clean, err) + } + var p corev1.Pod + if err := r.Get(ctx, client.ObjectKeyFromObject(other), &p); err != nil { + t.Fatal("deleted unrelated worker") + } +} + +func TestNodeCaptureDoesNotDeleteNamesakeJob(t *testing.T) { + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Name: "cap", UID: "capture-uid", Annotations: map[string]string{nodeCaptureNamespaceAnnotation: "diagnostics"}}} + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: nodeCaptureName(nc) + "-worker", Namespace: "diagnostics", UID: "foreign"}} + r := nodeReconcilerFixture(t, job) + if _, err := r.deleteNodeWorkers(t.Context(), nc); err == nil { + t.Fatal("accepted foreign job") + } + var live batchv1.Job + if err := r.Get(t.Context(), types.NamespacedName{Namespace: job.Namespace, Name: job.Name}, &live); apierrors.IsNotFound(err) { + t.Fatal("deleted foreign job") + } +} + +func TestNodeOptInPreservesLegacyCaptureLifecycle(t *testing.T) { + r := nodeReconcilerFixture(t) + for _, phase := range []gameplanev1alpha1.CapturePhase{gameplanev1alpha1.CapturePhasePending, gameplanev1alpha1.CapturePhaseRunning, gameplanev1alpha1.CapturePhaseCompleted} { + nc := &gameplanev1alpha1.NetworkCapture{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{capturePodUIDAnnotation: "legacy-pod"}}, Status: gameplanev1alpha1.NetworkCaptureStatus{Phase: phase}} + if node, err := r.useNodeCapture(t.Context(), nc); err != nil || node { + t.Fatalf("legacy phase %s rerouted: %t %v", phase, node, err) + } + nc.Annotations[nodeCaptureNamespaceAnnotation] = "diagnostics" + if node, err := r.useNodeCapture(t.Context(), nc); err != nil || !node { + t.Fatalf("bound node phase %s lost lifecycle: %t %v", phase, node, err) + } + } +} diff --git a/operator/internal/controller/workload_security.go b/operator/internal/controller/workload_security.go new file mode 100644 index 000000000..d1471444c --- /dev/null +++ b/operator/internal/controller/workload_security.go @@ -0,0 +1,67 @@ +package controller + +import ( + "fmt" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + corev1 "k8s.io/api/core/v1" +) + +const agentTokenVolumeName = "agent-api-token" +const workloadSecurityLabel = "gameplane.local/workload-security" + +// Images must declare their compatible numeric identity; guessing from an +// image name or trusting an unknown image USER only moves failure to kubelet. +func validateRestrictedTemplate(tmpl *gameplanev1alpha1.GameTemplate) error { + sec := tmpl.Spec.Security + if sec == nil || sec.RunAsUser == nil || *sec.RunAsUser <= 0 || sec.FSGroup == nil || *sec.FSGroup <= 0 || (sec.RunAsGroup != nil && *sec.RunAsGroup <= 0) { + return fmt.Errorf("restricted workload requires template security.runAsUser and security.fsGroup greater than zero, and a positive security.runAsGroup when specified; use an image compatible with that identity") + } + return nil +} + +func restrictedContainerSecurity(tmpl *gameplanev1alpha1.GameTemplate) *corev1.SecurityContext { + group := tmpl.Spec.Security.RunAsGroup + if group == nil { + group = tmpl.Spec.Security.FSGroup + } + return &corev1.SecurityContext{ + RunAsUser: tmpl.Spec.Security.RunAsUser, RunAsGroup: group, + RunAsNonRoot: ptrTo(true), AllowPrivilegeEscalation: ptrTo(false), + Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } +} + +// All profiles restrict the mounted API credentials to the agent. Shared PID +// remains intentional for metrics: this is mount isolation, not a hard trust +// boundary against a process that can inspect another process through /proc. +func applyWorkloadSecurity(pod *corev1.PodSpec, tmpl *gameplanev1alpha1.GameTemplate, restricted bool, expirationSeconds int64) { + if expirationSeconds < 600 || expirationSeconds > 3600 { + expirationSeconds = 3600 + } + pod.AutomountServiceAccountToken = ptrTo(false) + pod.Volumes = append(pod.Volumes, corev1.Volume{Name: agentTokenVolumeName, VolumeSource: corev1.VolumeSource{Projected: &corev1.ProjectedVolumeSource{ + DefaultMode: ptrTo(int32(0o644)), Sources: []corev1.VolumeProjection{ + {ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Path: "token", ExpirationSeconds: &expirationSeconds}}, + {ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "kube-root-ca.crt"}, Items: []corev1.KeyToPath{{Key: "ca.crt", Path: "ca.crt"}}}}, + {DownwardAPI: &corev1.DownwardAPIProjection{Items: []corev1.DownwardAPIVolumeFile{{Path: "namespace", FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}}, + }, + }}}) + for i := range pod.Containers { + c := &pod.Containers[i] + if c.Name == "agent" { + c.VolumeMounts = append(c.VolumeMounts, corev1.VolumeMount{Name: agentTokenVolumeName, MountPath: "/var/run/secrets/kubernetes.io/serviceaccount", ReadOnly: true}) + } + if restricted && c.Name == "game" { + c.SecurityContext = restrictedContainerSecurity(tmpl) + } + } + if restricted { + pod.SecurityContext.RunAsNonRoot = ptrTo(true) + pod.SecurityContext.SeccompProfile = &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault} + for i := range pod.InitContainers { + pod.InitContainers[i].SecurityContext = restrictedContainerSecurity(tmpl) + } + } +} diff --git a/operator/internal/controller/workload_security_test.go b/operator/internal/controller/workload_security_test.go new file mode 100644 index 000000000..8ba74505b --- /dev/null +++ b/operator/internal/controller/workload_security_test.go @@ -0,0 +1,125 @@ +package controller + +import ( + "strings" + "testing" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestRestrictedHostportFailsBeforeWorkloadCreation(t *testing.T) { + scheme := testScheme(t) + if err := batchv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "hostport", Namespace: "games", UID: "server-uid"}} + gs.Spec.TemplateRef.Name = "compatible" + gs.Spec.Networking.Expose = "Hostport" + tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: gs.Spec.TemplateRef.Name}} + tmpl.Spec.Security = &gameplanev1alpha1.GameSecuritySpec{RunAsUser: ptrTo(int64(1000)), FSGroup: ptrTo(int64(1000))} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(gs, tmpl).WithStatusSubresource(gs).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: scheme, RestrictedWorkloads: true} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if gs.Status.Phase != gameplanev1alpha1.GameServerPhaseFailed || len(gs.Status.Conditions) == 0 || !strings.Contains(gs.Status.Conditions[0].Message, "use NodePort or LoadBalancer") { + t.Fatalf("incompatible exposure did not report an actionable failure: %+v", gs.Status) + } + var workloads appsv1.StatefulSetList + if err := c.List(t.Context(), &workloads); err != nil || len(workloads.Items) != 0 { + t.Fatalf("created a workload before validation: %+v %v", workloads.Items, err) + } + var claims corev1.PersistentVolumeClaimList + if err := c.List(t.Context(), &claims); err != nil || len(claims.Items) != 0 { + t.Fatalf("created storage before validation: %+v %v", claims.Items, err) + } + var configs corev1.ConfigMapList + if err := c.List(t.Context(), &configs); err != nil || len(configs.Items) != 0 { + t.Fatalf("created template/config state before validation: %+v %v", configs.Items, err) + } + // Exposure validation must also precede template lookup. This keeps + // future template admission/rollout work behind the same rejection. + if err := c.Delete(t.Context(), tmpl); err != nil { + t.Fatal(err) + } + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + if len(gs.Status.Conditions) == 0 || !strings.Contains(gs.Status.Conditions[0].Message, "use NodePort or LoadBalancer") { + t.Fatalf("template lookup ran before exposure validation: %+v", gs.Status) + } +} + +func TestRestrictedTemplateIdentity(t *testing.T) { + for _, test := range []struct { + name string + uid, gid, fs *int64 + valid bool + }{ + {"absent", nil, nil, nil, false}, + {"root", ptrTo(int64(0)), nil, ptrTo(int64(1000)), false}, + {"missing-volume-group", ptrTo(int64(1000)), nil, nil, false}, + {"root-group", ptrTo(int64(1000)), ptrTo(int64(0)), ptrTo(int64(1000)), false}, + {"compatible", ptrTo(int64(1000)), ptrTo(int64(1000)), ptrTo(int64(1000)), true}, + {"default-primary-group", ptrTo(int64(1000)), nil, ptrTo(int64(1000)), true}, + } { + t.Run(test.name, func(t *testing.T) { + tmpl := &gameplanev1alpha1.GameTemplate{} + tmpl.Spec.Security = &gameplanev1alpha1.GameSecuritySpec{RunAsUser: test.uid, RunAsGroup: test.gid, FSGroup: test.fs} + if err := validateRestrictedTemplate(tmpl); (err == nil) != test.valid { + t.Fatalf("valid=%t: %v", test.valid, err) + } + }) + } +} + +func TestWorkloadTokenOnlyMountedInAgent(t *testing.T) { + tmpl := &gameplanev1alpha1.GameTemplate{} + tmpl.Spec.Security = &gameplanev1alpha1.GameSecuritySpec{RunAsUser: ptrTo(int64(1000)), FSGroup: ptrTo(int64(1000))} + gs := &gameplanev1alpha1.GameServer{} + pod := corev1.PodSpec{SecurityContext: gamePodSecurityContext(tmpl), Containers: []corev1.Container{{Name: "game"}, buildAgentContainer(gs, tmpl, nil, "agent", "", "")}, InitContainers: []corev1.Container{buildConfigInitContainer("", tmpl)}} + applyWorkloadSecurity(&pod, tmpl, true, 3600) + if pod.AutomountServiceAccountToken == nil || *pod.AutomountServiceAccountToken { + t.Fatal("automatic credentials exposed") + } + for _, c := range append(pod.Containers, pod.InitContainers...) { + sc := c.SecurityContext + if sc == nil || sc.RunAsNonRoot == nil || !*sc.RunAsNonRoot || sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation || sc.SeccompProfile == nil || sc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault || sc.Capabilities == nil || len(sc.Capabilities.Drop) != 1 || sc.Capabilities.Drop[0] != "ALL" || len(sc.Capabilities.Add) != 0 { + t.Fatalf("%s not Restricted: %+v", c.Name, sc) + } + mounted := false + for _, m := range c.VolumeMounts { + if m.Name == agentTokenVolumeName { + mounted = true + if !m.ReadOnly || m.MountPath != "/var/run/secrets/kubernetes.io/serviceaccount" { + t.Fatalf("unsafe token mount: %+v", m) + } + } + } + if mounted != (c.Name == "agent") { + t.Fatalf("%s token mount=%t", c.Name, mounted) + } + } + var projection *corev1.ProjectedVolumeSource + for _, v := range pod.Volumes { + if v.Name == agentTokenVolumeName { + projection = v.Projected + } + } + if projection == nil || len(projection.Sources) != 3 || projection.Sources[0].ServiceAccountToken == nil || *projection.Sources[0].ServiceAccountToken.ExpirationSeconds != 3600 { + t.Fatalf("missing rotating token projection: %+v", projection) + } +} diff --git a/operator/specs.md b/operator/specs.md index c7aea8225..e4d19aca8 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -6,6 +6,35 @@ ## Purpose +### Restricted workloads and node capture (#714) + +`--restricted-workloads` applies nonroot, no escalation, drop ALL and +RuntimeDefault to game/config-init; incompatible numeric identities and Hostport exposure fail with +the existing Ready/Failed condition before workload creation. Every game Pod +disables token automount and projects token/CA/namespace only into agent. +Token lifetime is 600–3600s (default 3600); shared PID remains for metrics. +Use NodePort or LoadBalancer for Restricted workloads. Config-init creates +files with the declared fsGroup as its primary group in both profiles and +reconciles copied entries' group-write permission only when needed, so an +already-correct root-owned file can be reused by the Restricted init UID. +Existing volumes still need compatible ownership before enabling Restricted. + +Restricted Pods never receive the legacy capture ephemeral container. +`spec.capture.nodeDiagnostic` additionally requires cluster capture enablement +and `--diagnostics-namespace`. The dedicated namespace must explicitly enforce +privileged and be separate from games/control plane. Live owner UIDs establish +GameServer→StatefulSet→Pod provenance; Pod UID, node and exact containerd ID are +pinned before worker creation. A target change drains work before releasing +the capture lock. A bounded, no-retry Job has hostPID and only SYS_ADMIN, +SYS_PTRACE/NET_RAW additions, no token/socket/hostPath. Duration <=3600s, bytes +<=900Mi, PVC 1Gi; deployment quota bounds aggregate retention. Linux/containerd +with eth0 is the supported target. Artifact serving uses a separate Restricted +Pod with per-capture mTLS. Direct reads cover diagnostic Secrets and resources. +Cross-namespace owner references are never used: a NetworkCapture finalizer +drains worker Pods, deletes a same-namespace ConfigMap anchor with foreground +GC, then releases finalization. TTL and GameServer owner GC drive this cleanup. +Terminal captures refresh artifact certificates without recreating targets. + The operator is a Kubernetes controller-runtime-based process that reconciles Gameplane Custom Resource Definitions (CRDs) into operational Kubernetes objects. It is the authoritative owner of all state transformations: a user must be able to `kubectl apply` a GameServer and achieve the same outcome as creating it via the dashboard API. The operator drives the full lifecycle of game servers, backups, restores, and module installation through a set of specialized reconcilers watching CRD status. ## Responsibilities diff --git a/test/e2e/api_events_roles_scope_e2e_test.go b/test/e2e/api_events_roles_scope_e2e_test.go index 2ab19b2b4..4c3b388c0 100644 --- a/test/e2e/api_events_roles_scope_e2e_test.go +++ b/test/e2e/api_events_roles_scope_e2e_test.go @@ -12,6 +12,9 @@ import ( "strings" "testing" "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions checks two @@ -64,19 +67,37 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { applyBusyboxTemplate(t, tmpl) createGameServerViaAPI(t, admin, ns, gsName, tmpl) + // /events starts at each collection's current resourceVersion. Drive real + // updates throughout both streams instead of depending on initial replay + // or other parallel tests to happen to mutate a readable resource kind. + changes := func(ctx context.Context) error { + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": map[string]string{ + "gameplane.local/e2e-event-sequence": time.Now().UTC().Format(time.RFC3339Nano), + }}}) + if err != nil { + return fmt.Errorf("encode event trigger: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, tmpl, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger template event: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, gsName, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger server event: %w", err) + } + return nil + } user := envInstance.APIClient(t, userName, userPW) defer user.Close() t.Run("admin event stream includes templates", func(t *testing.T) { - kinds, found := streamEventKinds(t, admin, ns, func(kind, _ string) bool { return kind == "templates" }, 0) + kinds, found := streamEventKinds(t, admin, ns, func(kind, name string) bool { return kind == "templates" && name == tmpl }, 0, changes) if !found { t.Fatalf("admin stream sent no templates event; kinds=%v", kinds) } }) t.Run("event stream sends only kinds the caller may read", func(t *testing.T) { - kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second) + kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second, changes) if !found { t.Fatalf("user stream never sent server %s; kinds=%v", gsName, kinds) } @@ -126,11 +147,41 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { // streamEventKinds reads GET /events as cli and returns the set of event // kinds it received. When match reports true for a frame, it keeps // reading for grace (0 = stop at once); otherwise it stops after 30s. +// changes keeps both permitted and forbidden kinds active while connecting and +// reading; its worker is cancelled and joined before the helper returns. // found reports whether match was ever true. -func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration) (map[string]bool, bool) { +func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration, changes func(context.Context) error) (map[string]bool, bool) { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() + changesDone := make(chan struct{}) + var changesErr error + go func() { + defer close(changesDone) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + if err := changes(ctx); err != nil { + if ctx.Err() == nil { + changesErr = err + } + cancel() + return + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + defer func() { + cancel() + <-changesDone + if changesErr != nil { + t.Errorf("event trigger failed: %v", changesErr) + } + }() req, err := http.NewRequestWithContext(ctx, http.MethodGet, cli.BaseURL+"/events?namespace="+ns, nil) if err != nil { t.Fatalf("new events request: %v", err) diff --git a/test/e2e/api_login_transport_e2e_test.go b/test/e2e/api_login_transport_e2e_test.go new file mode 100644 index 000000000..60205293c --- /dev/null +++ b/test/e2e/api_login_transport_e2e_test.go @@ -0,0 +1,49 @@ +//go:build e2e + +package e2e + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/url" + "os" + "syscall" + "testing" +) + +// This classification uses no cluster requests or login budget. +func TestAPI_LoginTransportRefusalClassification(t *testing.T) { + t.Parallel() + dialRefused := &net.OpError{Op: "dial", Net: "tcp", Err: &os.SyscallError{Syscall: "connect", Err: syscall.ECONNREFUSED}} + cases := []struct { + name string + err error + want bool + }{ + {"dial refused", dialRefused, true}, + {"wrapped dial refused", fmt.Errorf("login: %w", &url.Error{Op: "Post", URL: "http://127.0.0.1/auth/login", Err: dialRefused}), true}, + {"nil", nil, false}, + {"bare refusal", syscall.ECONNREFUSED, false}, + {"untyped text", errors.New("dial tcp: connection refused"), false}, + {"read refusal", &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNREFUSED}, false}, + {"write refusal", &net.OpError{Op: "write", Net: "tcp", Err: syscall.ECONNREFUSED}, false}, + {"dial reset", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ECONNRESET}, false}, + {"read reset", &net.OpError{Op: "read", Net: "tcp", Err: syscall.ECONNRESET}, false}, + {"dial timeout", &net.OpError{Op: "dial", Net: "tcp", Err: syscall.ETIMEDOUT}, false}, + {"deadline", &net.OpError{Op: "dial", Net: "tcp", Err: context.DeadlineExceeded}, false}, + {"canceled", context.Canceled, false}, + {"EOF", io.EOF, false}, + {"unexpected EOF", io.ErrUnexpectedEOF, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := loginDialRefused(tc.err); got != tc.want { + t.Fatalf("loginDialRefused(%v) = %t, want %t", tc.err, got, tc.want) + } + }) + } +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index a32d38dea..c946268a6 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -125,6 +125,7 @@ EOF bucket_api_auth() { cat <<'EOF' TestAPI_BootstrapAndLogin TestAPI_LoginPrivacy +TestAPI_LoginTransportRefusalClassification TestAPI_DynamicAuthProviders TestAPI_AuditEmitsOnMutation TestAPI_AuditPaginationAndFilter @@ -393,6 +394,12 @@ TestOptionalComponents_Lifecycle EOF } +# Dedicated release: enables real PSA enforcement and privileged diagnostics. +bucket_restricted_workloads() { cat <<'EOF' +TestRestrictedWorkloads_NodeCaptureAndTokenRotation +EOF +} + # capture-overhead is kept OUT of bot-fast on purpose: bot-fast's matrix legs # are blocking, while this bucket feeds the non-blocking e2e-capture-overhead # job (continue-on-error, never blocks a merge — spec 007 OD-1) and runs only @@ -409,7 +416,7 @@ EOF unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components restricted-workloads } list_bucket() { @@ -428,6 +435,7 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; + restricted-workloads) bucket_restricted_workloads ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index d2d6edd00..ba17bd931 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", + "telemetry", "optional-components", "restricted-workloads", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. diff --git a/test/e2e/env.go b/test/e2e/env.go index 35e3da008..3c42a862a 100644 --- a/test/e2e/env.go +++ b/test/e2e/env.go @@ -19,6 +19,7 @@ import ( "path/filepath" "strings" "sync" + "syscall" "testing" "time" @@ -523,7 +524,9 @@ func (e *Env) APIClient(t *testing.T, username, password string) *APIClient { var rb []byte delay := 2 * time.Second const maxDelay = 30 * time.Second - for attempt := 0; attempt < 7; attempt++ { + const maxAttempts = 7 + reconnected := false + for attempt := 0; attempt < maxAttempts; attempt++ { req, rerr := http.NewRequestWithContext(t.Context(), http.MethodPost, base+"/auth/login", bytes.NewReader(body)) if rerr != nil { stop() @@ -533,6 +536,15 @@ func (e *Env) APIClient(t *testing.T, username, password string) *APIClient { resp, err = cli.Do(req) if err != nil { stop() + // A vanished local listener cannot have received this request. + // Reopen once within the existing attempt budget; never replay an + // ambiguous read/write failure or reconnect an established session. + if !reconnected && attempt+1 < maxAttempts && t.Context().Err() == nil && loginDialRefused(err) { + local, stop = e.PortForward(t, "gameplane-system", "svc/gameplane-api", 80) + base = fmt.Sprintf("http://127.0.0.1:%d", local) + reconnected = true + continue + } t.Fatalf("login: %v", err) } rb, _ = io.ReadAll(resp.Body) @@ -560,6 +572,11 @@ func (e *Env) APIClient(t *testing.T, username, password string) *APIClient { return &APIClient{BaseURL: base, CSRF: lr.CSRF, HTTP: cli, stop: stop} } +func loginDialRefused(err error) bool { + var opErr *net.OpError + return errors.As(err, &opErr) && opErr.Op == "dial" && !opErr.Timeout() && errors.Is(opErr.Err, syscall.ECONNREFUSED) +} + // Close tears down the port-forward backing this client. func (c *APIClient) Close() { if c.stop != nil { diff --git a/test/e2e/restricted_workloads_e2e_test.go b/test/e2e/restricted_workloads_e2e_test.go new file mode 100644 index 000000000..ed90d0f13 --- /dev/null +++ b/test/e2e/restricted_workloads_e2e_test.go @@ -0,0 +1,330 @@ +//go:build e2e + +package e2e + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os/exec" + "strings" + "testing" + "time" + + "github.com/gopacket/gopacket/pcapgo" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// Dedicated kind bucket on both architectures: it upgrades namespace policy, +// checks real PSA admission, and waits for kubelet to rotate an actual token. +// It runs serially because it upgrades the shared Helm release. +func TestRestrictedWorkloads_NodeCaptureAndTokenRotation(t *testing.T) { + ctx := t.Context() + const ns, diagnosticNS = "gameplane-games", "gameplane-diagnostics" + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + name, template, incompatible := "restricted-game-"+suffix, "restricted-template-"+suffix, "restricted-incompatible-"+suffix + applyBusyboxTemplate(t, template) + // The template controller updates status immediately after creation. A + // field-scoped patch preserves that status without racing resourceVersion. + patch := []byte(`{"spec":{"security":{"runAsUser":1000,"runAsGroup":1000,"fsGroup":1000},"configFiles":[{"path":"cfg/profile.txt","template":"restricted-config\n"}]}}`) + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, template, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + applyBusyboxGameServer(t, ns, name, template) + t.Cleanup(func() { + if !t.Failed() { + return + } + logCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second) + defer cancel() + out, err := envInstance.Kubectl(logCtx, "-n", ns, "logs", "pod/"+name+"-0", "-c", "config-init", "--tail=30") + t.Logf("config-init diagnostic: %v\n%s", err, out) + }) + waitGameContainerReady(t, ns, name+"-0", 3*time.Minute) + before, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if out, err := envInstance.KubectlExec(t, ns, "pod/"+name+"-0", "sh", "-c", "test $(stat -c %u /data/cfg/profile.txt) = 0 && test $(stat -c %g /data/cfg/profile.txt) = 1000 && echo before-upgrade >> /data/cfg/profile.txt"); err != nil { + t.Fatalf("legacy root config must be writable by the game through its shared group: %v %s", err, out) + } + if out, err := envInstance.KubectlExec(t, ns, "pod/"+name+"-0", "sh", "-c", "echo upgrade-marker > /data/marker && sync"); err != nil { + t.Fatalf("write data: %v %s", err, out) + } + helmUpgradeReuse(ctx, t, "podSecurity.enforceRestricted=true", "operator.agentTokenExpirationSeconds=600", "capture.enabled=true", "capture.nodeDiagnostics.enabled=true") + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + p, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return p.UID != before.UID && p.Annotations["gameplane.local/workload-security"] == "restricted", "waiting for profile upgrade" + }) + waitGameContainerReady(t, ns, name+"-0", 3*time.Minute) + requireAgentReady(t, ns, name) + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken || len(pod.Spec.InitContainers) != 1 || len(pod.Spec.EphemeralContainers) != 0 { + t.Fatalf("unexpected restricted pod: %+v", pod.Spec) + } + if pod.Spec.ShareProcessNamespace == nil || !*pod.Spec.ShareProcessNamespace { + t.Fatal("restricted profile must retain shared PID for agent metrics") + } + for _, c := range append(pod.Spec.Containers, pod.Spec.InitContainers...) { + sc := c.SecurityContext + if sc == nil || sc.RunAsNonRoot == nil || !*sc.RunAsNonRoot || sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation || sc.Capabilities == nil || len(sc.Capabilities.Add) != 0 || len(sc.Capabilities.Drop) != 1 || sc.Capabilities.Drop[0] != "ALL" || sc.SeccompProfile == nil || sc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Fatalf("%s is not Restricted", c.Name) + } + tokenMounted := false + for _, mount := range c.VolumeMounts { + if mount.Name == "agent-api-token" { + tokenMounted = true + } + } + if tokenMounted != (c.Name == "agent") { + t.Fatalf("%s token mount=%t", c.Name, tokenMounted) + } + } + if out, err := envInstance.KubectlExec(t, ns, "pod/"+name+"-0", "sh", "-c", "test ! -e /var/run/secrets/kubernetes.io/serviceaccount/token && test $(id -u) = 1000 && grep upgrade-marker /data/marker && grep restricted-config /data/cfg/profile.txt && echo writable >> /data/marker"); err != nil { + t.Fatalf("identity/config/data/token: %v %s", err, out) + } + // A root container is rejected by the real admission controller. + zero := int64(0) + _, err = envInstance.K8s.CoreV1().Pods(ns).Create(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "rejected-root-" + suffix}, Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "root", Image: "busybox:1.36", SecurityContext: &corev1.SecurityContext{RunAsUser: &zero}}}}}, metav1.CreateOptions{}) + if !apierrors.IsForbidden(err) || !strings.Contains(err.Error(), "PodSecurity") { + t.Fatalf("PSA did not reject root: %v", err) + } + // Hash only: never return the bearer token in CI output. This is the kind + // host fixture inspecting kubelet's projection, not an extra game mount. + tokenPath := "/var/lib/kubelet/pods/" + string(pod.UID) + "/volumes/kubernetes.io~projected/agent-api-token/token" + tokenHash := func() (string, error) { + cctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + out, err := exec.CommandContext(cctx, "docker", "exec", pod.Spec.NodeName, "sha256sum", tokenPath).CombinedOutput() + if err != nil { + return "", fmt.Errorf("hash projected token: %w", err) + } + fields := strings.Fields(string(out)) + if len(fields) == 0 { + return "", fmt.Errorf("empty token hash") + } + return fields[0], nil + } + initialHash, err := tokenHash() + if err != nil { + t.Fatal(err) + } + initialTokenObserved := time.Now() + applyBusyboxTemplate(t, incompatible) + applyBusyboxGameServer(t, ns, incompatible, incompatible) + envInstance.Eventually(t, time.Minute, func() (bool, string) { + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, incompatible, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + phase, _, _ := unstructured.NestedString(gs.Object, "status", "phase") + if phase != "Failed" { + return false, "incompatible template not rejected" + } + _, err = envInstance.K8s.AppsV1().StatefulSets(ns).Get(ctx, incompatible, metav1.GetOptions{}) + return apierrors.IsNotFound(err), "incompatible template reached workload admission" + }) + envInstance.BootstrapAdmin(t, adminUsername, adminPassword) + cli := envInstance.APIClient(t, adminUsername, adminPassword) + defer cli.Close() + post := func(path string, body any, want int) []byte { + resp, raw, err := cli.Post(path, body) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode != want { + t.Fatalf("POST %s: %d %s", path, resp.StatusCode, raw) + } + return raw + } + // Setting an opt-in is idempotent, but its UID/resourceVersion fence may + // conflict with a controller status update. Retry only that explicit + // response, never capture creation or a transport/authorization failure. + setCapture := func(action string) { + envInstance.Eventually(t, time.Minute, func() (bool, string) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, cli.BaseURL+"/servers/"+name+":"+action, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Gameplane-CSRF", cli.CSRF) + resp, err := cli.HTTP.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode == http.StatusConflict { + return false, action + ": conflicting server update" + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("POST %s: %d", action, resp.StatusCode) + } + return true, "" + }) + } + setCapture("capture-enable") + setCapture("capture-node-enable") + envInstance.Eventually(t, time.Minute, func() (bool, string) { + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + ready, _, _ := unstructured.NestedBool(gs.Object, "status", "capture", "ready") + return ready, "capture not ready" + }) + start := func(duration int) *unstructured.Unstructured { + raw := post("/servers/"+name+":capture-start", map[string]any{"filter": "tcp port 12345", "maxDurationSeconds": duration, "maxSizeBytes": int64(4 << 20), "ttlSecondsAfterFinished": 3600}, http.StatusAccepted) + var response struct { + CaptureID string `json:"captureId"` + } + if err := json.Unmarshal(raw, &response); err != nil || response.CaptureID == "" { + t.Fatalf("capture response: %s %v", raw, err) + } + nc, err := envInstance.Dyn.Resource(networkCaptureGVR).Namespace(ns).Get(ctx, response.CaptureID, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _ = envInstance.Dyn.Resource(networkCaptureGVR).Namespace(ns).Delete(cleanupCtx, nc.GetName(), metav1.DeleteOptions{}) + }) + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + pods, err := envInstance.K8s.CoreV1().Pods(diagnosticNS).List(ctx, metav1.ListOptions{LabelSelector: "gameplane.local/capture-uid=" + string(nc.GetUID()) + ",gameplane.local/capture-component=worker"}) + if err != nil { + return false, err.Error() + } + for _, p := range pods.Items { + if p.Status.Phase == corev1.PodRunning { + if p.Spec.NodeName != pod.Spec.NodeName || !p.Spec.HostPID || p.Spec.AutomountServiceAccountToken == nil || *p.Spec.AutomountServiceAccountToken { + t.Fatal("worker node/privilege/token mismatch") + } + return true, "" + } + } + return false, "waiting for bound worker" + }) + return nc + } + nc := start(60) + traffic := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "restricted-traffic-" + suffix}, Spec: corev1.PodSpec{RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{{Name: "traffic", Image: "busybox:1.36", Command: []string{"sh", "-c", "for i in $(seq 1 20); do nc -w 1 " + pod.Status.PodIP + " 12345 11*time.Minute, "waiting for rotation and original token expiry" + }) + rotatedAt := time.Now() + envInstance.Eventually(t, time.Minute, func() (bool, string) { + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + s, _, _ := unstructured.NestedString(gs.Object, "status", "agent", "lastHeartbeat") + heartbeat, err := time.Parse(time.RFC3339, s) + return err == nil && heartbeat.After(rotatedAt), "no post-rotation heartbeat" + }) + // Deleting the owning server garbage-collects both retained artifacts and + // the canceled capture across namespaces through each capture's finalizer. + if err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + for _, capture := range []*unstructured.Unstructured{nc, second} { + selector := "gameplane.local/capture-uid=" + string(capture.GetUID()) + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + _, err := envInstance.Dyn.Resource(networkCaptureGVR).Namespace(ns).Get(ctx, capture.GetName(), metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + return false, "capture finalizer still present" + } + pods, err := envInstance.K8s.CoreV1().Pods(diagnosticNS).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return false, err.Error() + } + pvcs, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(diagnosticNS).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return false, err.Error() + } + return len(pods.Items) == 0 && len(pvcs.Items) == 0, "diagnostic storage/pods remain" + }) + } +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index abf7cda91..426b5c37e 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -6,6 +6,27 @@ ## Purpose +### Restricted workload drill (#714) + +The dedicated `restricted-workloads` bucket runs on amd64 and arm64. It +is registered in the CI selector for operator, API, agent and capture-sidecar +changes, including the node capture worker. Full-scope and affected-component +runs retain the drill on both architectures. It +upgrades a compatible existing game to the real enforced PSA profile, asserts +nonroot config-init/writable data preservation, rejects root admission and an +incompatible template, and checks agent-only credential mounts. It observes a +kubelet-projected token hash change (never logs tokens) and a heartbeat after +the original 600s token expires. Node capture checks exercise audited per-server +opt-in, traffic-bearing PCAP download, unchanged game Pod UID/restart counts, +worker cleanup on completion/revocation, and cross-namespace resource cleanup +after GameServer deletion. Local work compiles test sources only; CI executes +the drill and all unit/chart regressions. Idempotent capture-setting requests +retry only HTTP 409 conflicts with controller updates; capture creation and +transport or authorization failures are never retried. Completion still +requires all capture workers to have been drained before `Completed` is reported. +The drill explicitly verifies that shared PID remains enabled for agent metrics +after the Restricted upgrade and after capture completion and revocation. + The `test/e2e` module is Gameplane's end-to-end test suite: it stands up a real (kind-based) Kubernetes cluster, installs the Helm chart, and drives the operator/api/agent/sentinel stack as real components rather than mocks — CRDs applied through the typed/dynamic client, HTTP calls through a real port-forward, RCON/wire-protocol joins against real game server containers. Where the other modules' test tiers (unit, envtest) prove a component's own logic in isolation, this module proves the whole product works together on a cluster shaped like the ones users actually run. It was one of three `go.work` modules without CI lint coverage until this feature (`api`, `agent`, and `test/e2e` — see FR-001); all 13 `go.work` members (`netguard`, `gameaction`, `gameproto`, `operator`, `agent`, `api`, `sentinel`, `audit-syslog-bridge`, `telemetry-receiver`, `mcp-server`, `svcutil`, `tunnel`, and `test/e2e`) are golangci-lint gated in CI now — see the `lint` job's matrix in `.github/workflows/ci.yaml`. `test/e2e` lints with `--build-tags=e2e`, same tier as `operator`/`api`'s `--build-tags=envtest`, run as their own step so the catch-all lint step excludes all three and nothing is linted twice. @@ -71,10 +92,11 @@ This is a name-matching contract, not a structural one: - **`Kubectl(ctx, args...)`** and **`KubectlWithStdin(ctx, stdin, args...)`** both run `exec.CommandContext(ctx, "kubectl", all...)`. Both still validate every argument up to a literal `--` separator against shell metacharacters (`|;&$\`<>()\\`) before building `all` — this is the caller-supplied-args gosec finding scoped by the `.golangci.yml` G204 exclusion on `env.go`: gosec can't see that the args come from the suite's own test code and are pre-validated, not from untrusted external input. - **`tryPortForward(ctx, ns, target, remotePort)`** (the single-attempt implementation behind the public, retrying `PortForward`) also runs its `kubectl port-forward` child via `exec.CommandContext(ctx, ...)`. **The context passed here must outlive the call that starts the forward** — it only bounds the *maximum* process lifetime, not the normal teardown path. The returned `stop` func is the real owner of the forward's lifecycle: every caller `defer`/`t.Cleanup`s it well before the test (and therefore its context) ends. Binding the forward to anything shorter-lived than the test itself — e.g. a context scoped to just the readiness-poll loop — would risk the process being killed by context expiry out from under an in-flight caller still using the tunnel. `PortForward` retries `tryPortForward` up to 4 times on a fresh local port before giving up, tolerating a loaded self-hosted runner where a single attempt's readiness window is too short. +- **`APIClient` initial login** may reopen its forward at most once when a typed dial error reports `ECONNREFUSED`, proving the local listener did not receive the request. The reconnect consumes an attempt from the existing seven-attempt login budget and remains bound to the test context. Read/write errors, resets, timeouts, ambiguous failures and HTTP errors retain their existing failure behavior (apart from the existing bounded HTTP 429 backoff). Established clients never reconnect or replay requests. `TestAPI_LoginTransportRefusalClassification` covers accepted and rejected transport errors in `api-auth` without consuming a cluster login. ## Conventions (already normative via `CLAUDE.md`, restated here for the module's own record) -- **`t.Parallel()`**: tests call it unless shared infrastructure requires serialization. The `ratelimit` bucket deliberately drains the shared login limiter; gateway parity temporarily changes the shared remote Helm release and trust. These are documented exceptions. +- **`t.Parallel()`**: tests call it unless shared infrastructure requires serialization. The `ratelimit` bucket deliberately drains the shared login limiter; gateway parity temporarily changes the shared remote Helm release and trust; `restricted-workloads` upgrades the shared release and namespace admission policy. These are documented exceptions. - **Per-test unique resource names**: CRs, namespaces, and users a test creates get a name unique to that test run (commonly time-based, e.g. `CreateUser`'s `fmt.Sprintf("%s-%d", prefix, time.Now().UnixNano())`) so parallel tests never collide on a shared cluster. - **`ociPushMu`**: a package-level mutex guarding tests that push through the shared, fixed-name `oras-push` Job (`module_e2e_test.go`, `module_verify_e2e_test.go`). Because the Job's name and namespace are fixed fixtures, two tests racing to apply/wait on it would stomp each other; `ociPushMu.Lock()`/`Unlock()` serializes them. - **`ensureResticRepo(t)`**: guards tests that run a backup against the shared restic repository fixture (`backup_e2e_test.go`, `backupschedule_e2e_test.go`, `failure_paths_e2e_test.go`) — restic repo initialization is not safe to race. diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index 017d1f88f..9de70cc5f 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -16,6 +16,7 @@ "test/e2e/suite-components.json" ], "suites": [ + { "tests": "^TestRestrictedWorkloads_", "components": ["operator", "api", "agent", "capture-sidecar"] }, { "tests": "^TestHelmInstall_", "components": ["*"] }, { "tests": "^TestCRD_Validation_", "components": ["operator"] }, { "tests": "^TestGameServer_", "components": ["operator", "api", "agent"] },