feat: add bounded game performance and relay monitoring

.devcontainer/devcontainer.json

@@ -8,7 +8,7 @@
 			"username": "vscode"
 		},
 		"ghcr.io/devcontainers/features/go:1": {
-			"version": "1.26"
+			"version": "1.27.3"
 		},
 		"ghcr.io/devcontainers/features/node:1": {
 			"version": "24"

.github/actions/go-cache/action.yml

@@ -11,7 +11,7 @@ inputs:
   go-version:
     description: Go version to setup
     required: false
-    default: '1.26'
+    default: '1.27'
 
 runs:
   using: composite
@@ -20,7 +20,7 @@ runs:
       id: go
       with:
         go-version: ${{ inputs.go-version }}
-        # Resolve '1.26' to the newest 1.26.x patch instead of the runner's
+        # Resolve '1.27' to the newest 1.27.x patch (at least 1.27.3) instead of the runner's
         # preinstalled one, so stdlib fixes (spec 013 G1, govulncheck) land
         # without a manual pin bump.
         check-latest: true

.github/workflows/ci.yaml

@@ -439,7 +439,8 @@ jobs:
     needs: [changes]
     if: (needs.changes.outputs.images == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'build-images')
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    # Include all image builds, cache exports, artifact upload and cleanup.
+    timeout-minutes: 30
     permissions:
       contents: read
     steps:
@@ -1169,6 +1170,8 @@ jobs:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1
       - run: helm lint charts/gameplane
+      - name: evaluate monitoring rules
+        run: bash charts/gameplane/tests/monitoring-rules.sh
 
   standalone-compose:
     name: standalone Compose smoke
@@ -1232,6 +1235,10 @@ jobs:
         run: python3 charts/tests/check_gateway.py
       - name: standalone panel isolation and persistence chart renders
         run: python3 charts/tests/check_standalone.py
+      - name: auxiliary resources and management placement chart renders
+        run: python3 charts/tests/check_workloads.py
+      - name: strict network profile and sentinel policy selectors
+        run: python3 charts/tests/check_network_isolation.py
       - name: sanity-check rendered manifests
         run: |
           grep -q 'kind: Deployment' /tmp/rendered.yaml
@@ -1316,10 +1323,16 @@ jobs:
             | grep -q 'app.kubernetes.io/name: gameplane-telemetry-receiver'
           awk '/kind: Service/,/^---$/' /tmp/telemetry-receiver-scrape.yaml \
             | grep -q 'name: dashboard'
-          # scrapeNamespaceSelector unset (the chart default): neither
-          # exception renders, matching the pre-fix behavior.
+          # Missing scrape ingress must fail with actionable guidance.
+          if helm template gameplane charts/gameplane --set serviceMonitors.enabled=true > /tmp/monitoring-invalid.txt 2>&1; then
+            echo "::error::monitoring without a scrape selector unexpectedly rendered"
+            exit 1
+          fi
+          grep -q 'serviceMonitors.scrapeNamespaceSelector' /tmp/monitoring-invalid.txt
+          # Explicit external policy ownership renders no chart exception.
           helm template gameplane charts/gameplane \
             --set serviceMonitors.enabled=true \
+            --set serviceMonitors.externallyManagedPolicy=true \
             --show-only templates/networkpolicies.yaml > /tmp/netpol-default.yaml
           if grep -q 'allow-prometheus-to-agent' /tmp/netpol-default.yaml; then
             echo "::error::allow-prometheus-to-agent rendered without serviceMonitors.scrapeNamespaceSelector set"
@@ -1496,7 +1509,8 @@ jobs:
           # keeps working without api.telemetry.enabled.
           helm template gameplane charts/gameplane \
             --set api.telemetry.receiver.enabled=true \
-            --set serviceMonitors.enabled=true > /tmp/rendered-reuse-values-receiver.yaml
+            --set serviceMonitors.enabled=true \
+            --set serviceMonitors.externallyManagedPolicy=true > /tmp/rendered-reuse-values-receiver.yaml
           python3 - <<'PY'
           import yaml
           docs = [d for d in yaml.safe_load_all(open('/tmp/rendered-reuse-values-receiver.yaml')) if d]
@@ -1793,13 +1807,25 @@ jobs:
         # (operator does zero logins and is wait-dominated, so it runs
         # wide; the api buckets stay within the login limiter's budget).
         # The arm64 job below uses the same steps and per-bucket settings.
-        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components]
+        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement, network-isolation, monitoring]
         arch: [amd64]
         include:
+          - bucket: monitoring
+            parallel: 1
+            test_timeout: 20m
+            job_timeout: 75
+          - bucket: network-isolation
+            parallel: 1
+            test_timeout: 15m
+            job_timeout: 60
           - bucket: operator
             parallel: 8
             test_timeout: 20m
             job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite.
+          - bucket: management-placement
+            parallel: 1
+            test_timeout: 15m
+            job_timeout: 60
           - bucket: api-auth
             parallel: 4
             test_timeout: 15m
@@ -1827,8 +1853,8 @@ jobs:
             job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite.
           - bucket: optional-components
             parallel: 1
-            test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order
-            job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite.
+            test_timeout: 35m # lifecycle plus two-GameServer TCP/UDP relay identity, outage and recovery checks
+            job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite.
     env:
       CLUSTER: gameplane-e2e
       CTX: kind-gameplane-e2e
@@ -1856,7 +1882,13 @@ jobs:
           arch: ${{ matrix.arch }}
       - name: bring up cluster + helm install
         if: env.RUN_REGEX != '' || env.TAIL_REGEX != ''
-        run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e
+        env:
+          GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }}
+          GAMEPLANE_E2E_CNI: ${{ (matrix.bucket == 'network-isolation' || matrix.bucket == 'monitoring') && 'cilium' || 'kindnet' }}
+        run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE"
+      - name: install real monitoring fixture
+        if: matrix.bucket == 'monitoring' && env.RUN_REGEX != ''
+        run: bash deploy/kind/monitoring.sh
       - name: run go e2e (${{ matrix.bucket }})
         if: env.RUN_REGEX != ''
         run: |
@@ -1880,7 +1912,7 @@ jobs:
         uses: ./.github/actions/dump-cluster-state
         with:
           context: ${{ env.CTX }}
-          namespaces: "gameplane-system,gameplane-games"
+          namespaces: "${{ matrix.bucket == 'monitoring' && 'gameplane-system,gameplane-games,monitoring' || 'gameplane-system,gameplane-games' }}"
           include-previous-logs: "true"
 
   e2e-go-arm64:
@@ -1934,8 +1966,8 @@ jobs:
             job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite.
           - bucket: optional-components
             parallel: 1
-            test_timeout: 20m # one lifecycle test: helm upgrade, then mcp, syslog, frp tunnel and core checks in order
-            job_timeout: 60 # Boots a kind cluster, installs the chart, runs the bucket's suite.
+            test_timeout: 35m # lifecycle plus two-GameServer TCP/UDP relay identity, outage and recovery checks
+            job_timeout: 75 # Boots a kind cluster, installs the chart, runs the bucket's suite.
     env:
       CLUSTER: gameplane-e2e
       CTX: kind-gameplane-e2e
@@ -2660,7 +2692,7 @@ jobs:
               'go-e2e-unit': (n) => n === 'go (test/e2e)',
               'api-postgres': (n) => n === 'api (postgres)',
               'e2e-buckets': (n) => n === 'e2e bucket coverage',
-              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '),
+              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|network-isolation|monitoring) \//.test(n) && n.includes('/ amd64 '),
               'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '),
               'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '),
               'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '),
@@ -2813,7 +2845,7 @@ jobs:
             const bucketSet = new Set();
             for (const j of apiJobs) {
               const n = j.name || '';
-              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//);
+              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|network-isolation|monitoring) \//);
               if (m) bucketSet.add(m[1]);
               if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster');
               if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade');

agent/cmd/main.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/go-chi/chi/v5"
 	"github.com/go-chi/chi/v5/middleware"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/GameplanePanel/gameplane/agent/internal/actions"
 	"github.com/GameplanePanel/gameplane/agent/internal/auth"
@@ -178,6 +179,7 @@ func main() {
 	// and isn't meant to be trusted with the console/files/RCON routes that
 	// cert would also unlock.
 
+	statusSource := status.New(rconClient, statusSpec)
 	mountProtected := func(protected chi.Router) {
 
 		// Console/Logs open long-lived WebSockets and Mods installs can run
@@ -198,7 +200,7 @@ func main() {
 			quiesce.Mount(bounded, rconClient, gameName, quiesceSpec)
 			lifecycle.Mount(bounded, rconClient, gameName, lifecycleSpec)
 			actions.Mount(bounded, rconClient, gameName, actionSpecs)
-			status.Mount(bounded, rconClient, statusSpec)
+			statusSource.Mount(bounded)
 		})
 	}
 	r.Group(func(protected chi.Router) {
@@ -230,6 +232,8 @@ func main() {
 	// Prometheus metrics for per-server resource usage and player counts.
 	// Registered with the default registry so promhttp.Handler() picks it up.
 	m := metrics.New()
+	prometheus.MustRegister(statusSource)
+	go statusSource.Run(ctx)
 
 	// Resource usage: in proc mode (the operator shares the pod's PID
 	// namespace and sets GAMEPLANE_USAGE_PROC) the agent reports the game

agent/internal/caps/caps.go

@@ -83,11 +83,19 @@ type ModInstall struct {
 // StatusMetric reads one live readout from an RCON command's output via a
 // named-group regex (group "value").
 type StatusMetric struct {
-	ID          string `json:"id"`
-	DisplayName string `json:"displayName,omitempty"`
-	Command     string `json:"command"`
-	Regex       string `json:"regex"`
-	Unit        string `json:"unit,omitempty"`
+	ID          string             `json:"id"`
+	DisplayName string             `json:"displayName,omitempty"`
+	Command     string             `json:"command"`
+	Regex       string             `json:"regex"`
+	Unit        string             `json:"unit,omitempty"`
+	Performance *PerformanceMetric `json:"performance,omitempty"`
+}
+
+// PerformanceMetric mirrors the optional, explicitly numeric status contract.
+type PerformanceMetric struct {
+	Signal            string  `json:"signal"`
+	Unit              string  `json:"unit"`
+	DegradedThreshold *string `json:"degradedThreshold,omitempty"`
 }
 
 // PlayerActions maps moderation actions to console command templates

agent/internal/rcon/nuclearoption.go

@@ -112,14 +112,21 @@ func (c *NuclearOption) Close() error {
 // If the response has no body (body length = 0), an empty string is returned.
 // Non-2000 status codes produce an error that includes the status code and,
 // if present, the error detail from the response body.
+// One deadline bounds the complete exchange, including dialing and request writes.
 func (c *NuclearOption) Exec(cmd string) (string, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	d := c.execDeadline
+	if d <= 0 {
+		d = defaultNuclearOptionExecDeadline
+	}
+	exchangeDeadline := time.Now().Add(d)
+
 	// Ensure connection is open.
 	if c.conn == nil {
 		d := net.Dialer{Timeout: c.dialTimeout}
-		ctx, cancel := context.WithTimeout(context.Background(), c.dialTimeout)
+		ctx, cancel := context.WithDeadline(context.Background(), exchangeDeadline)
 		defer cancel()
 		conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(c.host, fmt.Sprint(c.port)))
 		if err != nil {
@@ -144,6 +151,14 @@ func (c *NuclearOption) Exec(cmd string) (string, error) {
 		return "", fmt.Errorf("nuclearoption rcon: marshal request: %w", err)
 	}
 
+	// Bound writes as well as reads. Arm this before sending anything: a
+	// server that stops reading must not retain the shared client forever.
+	if err := c.conn.SetDeadline(exchangeDeadline); err != nil {
+		_ = c.conn.Close()
+		c.conn = nil
+		return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err)
+	}
+
 	// Write request: 4-byte LE length + body.
 	reqLen := len(reqBody)
 	if reqLen < 0 || reqLen > math.MaxUint32 {
@@ -162,17 +177,6 @@ func (c *NuclearOption) Exec(cmd string) (string, error) {
 		return "", fmt.Errorf("nuclearoption rcon: write body: %w", err)
 	}
 
-	// Set deadline for the entire response read.
-	d := c.execDeadline
-	if d <= 0 {
-		d = defaultNuclearOptionExecDeadline
-	}
-	if err := c.conn.SetDeadline(time.Now().Add(d)); err != nil {
-		_ = c.conn.Close()
-		c.conn = nil
-		return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err)
-	}
-
 	// Read response: 4-byte status + 4-byte body length + body.
 	statusBuf := make([]byte, 4)
 	if _, err := io.ReadFull(c.conn, statusBuf); err != nil {

agent/internal/rcon/nuclearoption_deadline_test.go

@@ -0,0 +1,46 @@
+package rcon
+
+import (
+	"io"
+	"net"
+	"strings"
+	"testing"
+	"time"
+)
+
+func TestNuclearOptionDeadlineIncludesRequestWrites(t *testing.T) {
+	t.Parallel()
+	for _, readHeader := range []bool{false, true} {
+		stage := "write length"
+		if readHeader {
+			stage = "write body"
+		}
+		t.Run(stage, func(t *testing.T) {
+			t.Parallel()
+			local, remote := net.Pipe()
+			defer local.Close()
+			defer remote.Close()
+			client := NewNuclearOption("ignored", 1, nil)
+			client.conn = local
+			client.execDeadline = 100 * time.Millisecond
+			if readHeader {
+				go func() { _, _ = io.CopyN(io.Discard, remote, 4) }()
+			}
+			done := make(chan error, 1)
+			go func() { _, err := client.Exec("get-server-id"); done <- err }()
+			select {
+			case err := <-done:
+				if err == nil || !isTimeout(err) || !strings.Contains(err.Error(), stage) {
+					t.Fatalf("stalled %s must time out: %v", stage, err)
+				}
+			case <-time.After(2 * time.Second):
+				_ = remote.Close()
+				<-done
+				t.Fatalf("stalled %s exceeded the command budget", stage)
+			}
+			if client.conn != nil {
+				t.Fatal("timed-out partial request must discard the connection")
+			}
+		})
+	}
+}

agent/internal/rcon/nuclearoption_test.go

@@ -740,7 +740,7 @@ func TestNuclearOptionWriteBodyError(t *testing.T) {
 }
 
 // TestNuclearOptionSetDeadlineError verifies that a failure setting the
-// response read deadline (after the request was fully written) is reported
+// exchange deadline (before sending the request) is reported
 // and the connection is reset.
 func TestNuclearOptionSetDeadlineError(t *testing.T) {
 	local, remote := net.Pipe()