From da63dc3e3e39dc7a24b08810a3bf3ebf731abffc Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Mon, 16 Nov 2026 16:59:17 +0100 Subject: [PATCH 1/9] fix(controllers): persist Cruise Control results through status conflicts Retry fresh API reads and status writes without repeating external submission. Preserve stop-operation identity and retry history, reject stale object/task identities, add seven regression cases, and document the remaining submit-before-persist crash window. Refs #369. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cruisecontroloperation_controller.go | 37 ++-- .../cruisecontroloperation_controller_test.go | 169 ++++++++++++++++++ docs/monitoring.md | 19 +- 3 files changed, 212 insertions(+), 13 deletions(-) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index d7da19d39..68a343bb2 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -243,24 +243,37 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques } } - conflictRetryFunction := func() error { - if err = updateResult(log, cruseControlTaskResult, ccOperationExecution, true); err != nil { - return err - } - err = r.Status().Update(ctx, ccOperationExecution) - if apiErrors.IsConflict(err) { - err = r.Get(ctx, client.ObjectKey{Name: ccOperationExecution.GetName(), Namespace: ccOperationExecution.GetNamespace()}, ccOperationExecution) - } - return err - } - // Protection to avoid re-execution task when status update is failed - if err := util.RetryOnConflict(util.DefaultBackOffForConflict, conflictRetryFunction); err != nil { + // Retry persistence without resubmitting the external task. + if err := r.persistExecutionResult(ctx, log, ccOperationExecution, cruseControlTaskResult); err != nil { return requeueWithError(log, "could not update the result of the Cruise Control user task execution to the CruiseControlOperation status", err) } return reconciled() } +func (r *CruiseControlOperationReconciler) persistExecutionResult( + ctx context.Context, + log logr.Logger, + operation *banzaiv1alpha1.CruiseControlOperation, + result *scale.Result, +) error { + return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { + latest := &banzaiv1alpha1.CruiseControlOperation{} + if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + return err + } + if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { + return errors.New("CruiseControlOperation identity or current task changed after external task submission") + } + // Preserve the stop operation selected in memory during finalization. + latest.Status.CurrentTask.Operation = operation.CurrentTaskOperation() + if err := updateResult(log, result, latest, true); err != nil { + return err + } + return r.Status().Update(ctx, latest) + }) +} + func (r *CruiseControlOperationReconciler) addFinalizer(ctx context.Context, currentCCOperation *banzaiv1alpha1.CruiseControlOperation) error { // examine DeletionTimestamp to determine if object is under deletion if currentCCOperation.DeletionTimestamp.IsZero() { diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index d9a68fd87..0c25b5c50 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -17,16 +17,22 @@ package controllers import ( "context" + "errors" "testing" "time" "github.com/go-logr/logr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" @@ -34,6 +40,169 @@ import ( "github.com/banzaicloud/koperator/pkg/scale" ) +func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { + for _, tt := range []struct { + name string + retry bool + deleting bool + replaced bool + taskRemoved bool + taskChanged bool + exhausted bool + conflicts int + wantRetries int + wantFailed int + wantOp v1alpha1.CruiseControlTaskOperation + }{ + {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, + {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, + {name: "replacement object rejects stale result", replaced: true, conflicts: 1}, + {name: "removed task rejects stale result", taskRemoved: true, conflicts: 1}, + {name: "newer task rejects stale result", taskChanged: true, conflicts: 1}, + {name: "exhausted conflicts return error", exhausted: true, conflicts: 1000}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := v1beta1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "cluster", Namespace: "ns"}} + operation := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: "operation", Namespace: "ns", UID: "operation-uid", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: "cluster"}, + }, + Spec: v1alpha1.CruiseControlOperationSpec{ErrorPolicy: v1alpha1.ErrorPolicyRetry}, + } + operation.Status.ErrorPolicy = v1alpha1.ErrorPolicyRetry + operation.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationAddBroker} + taskResult := &scale.Result{ + TaskID: "submitted-task", State: v1beta1.CruiseControlTaskInExecution, + StartedAt: "Mon, 02 Jan 2006 15:04:05 GMT", ResponseStatusCode: 202, + RequestURL: "http://cruise-control/add_broker", + } + var previous []*scale.Result + if tt.retry { + operation.Status.CurrentTask.ID = "failed-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskCompletedWithError + operation.Status.CurrentTask.Finished = &v1.Time{Time: time.Date(2006, 1, 2, 15, 5, 0, 0, time.UTC)} + operation.Status.RetryCount = 2 + operation.Status.FailedTasks = []v1alpha1.CruiseControlTask{{ID: "earlier-failed-task"}} + previous = []*scale.Result{{TaskID: "failed-task", State: v1beta1.CruiseControlTaskCompletedWithError}} + } + if tt.deleting { + operation.DeletionTimestamp = &v1.Time{Time: time.Now()} + operation.Finalizers = []string{ccOperationFinalizerGroup} + operation.Status.CurrentTask.ID = "running-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskInExecution + previous = []*scale.Result{{TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}} + } + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(t)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: true, + }}, nil) + var taskIDs []any + if len(previous) != 0 { + taskIDs = []any{previous[0].TaskID} + } + scaler.EXPECT().UserTasks(gomock.Any(), taskIDs...).Return(previous, nil) + if tt.deleting { + scaler.EXPECT().StopExecution(gomock.Any()).Return(taskResult, nil).Times(1) + } else { + scaler.EXPECT().AddBrokersWithParams(gomock.Any(), operation.CurrentTaskParameters()).Return(taskResult, nil).Times(1) + } + var resultWrites int + kubeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(cluster, operation). + WithStatusSubresource(operation). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if op, ok := obj.(*v1alpha1.CruiseControlOperation); ok && subresource == "status" && op.CurrentTaskID() == "submitted-task" { + resultWrites++ + if resultWrites <= tt.conflicts { + latest := &v1alpha1.CruiseControlOperation{} + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), latest); err != nil { + return err + } + latest.Annotations = map[string]string{"other-writer": "preserved"} + if err := c.Update(ctx, latest); err != nil { + return err + } + if tt.replaced { + latest.Finalizers = nil + if err := c.Update(ctx, latest); err != nil { + return err + } + if err := c.Delete(ctx, latest); err != nil { + return err + } + latest.ResourceVersion = "" + latest.UID = "replacement-uid" + if err := c.Create(ctx, latest); err != nil { + return err + } + } + if tt.taskRemoved || tt.taskChanged { + if tt.taskRemoved { + latest.Status.CurrentTask = nil + } else { + latest.Status.CurrentTask.ID = "newer-task" + } + if err := c.Status().Update(ctx, latest); err != nil { + return err + } + } + return apierrors.NewConflict(schema.GroupResource{Group: "kafka.banzaicloud.io", Resource: "cruisecontroloperations"}, obj.GetName(), errors.New("injected concurrent update")) + } + } + return c.SubResource(subresource).Update(ctx, obj, opts...) + }, + }).Build() + reconciler := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: kubeClient, Scheme: scheme, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { + return scaler, nil + }, + } + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(operation)}) + if err != nil && !tt.replaced && !tt.taskRemoved && !tt.taskChanged && !tt.exhausted { + t.Fatal(err) + } + persisted := &v1alpha1.CruiseControlOperation{} + if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(operation), persisted); err != nil { + t.Fatal(err) + } + if tt.replaced || tt.taskRemoved || tt.taskChanged || tt.exhausted { + if err == nil { + t.Error("reported success without persisting the result to the original task") + } + if persisted.CurrentTaskID() == "submitted-task" { + t.Error("attached stale external result to a replaced operation or removed task") + } + return + } + assert.Equal(t, "submitted-task", persisted.CurrentTaskID()) + assert.Equal(t, v1beta1.CruiseControlTaskInExecution, persisted.CurrentTaskState()) + assert.Equal(t, tt.wantOp, persisted.CurrentTaskOperation()) + task := persisted.CurrentTask() + require.NotNil(t, task) + assert.Equal(t, taskResult.RequestURL, task.HTTPRequest) + assert.Equal(t, &taskResult.ResponseStatusCode, task.HTTPResponseCode) + require.NotNil(t, task.Started) + assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) + assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) + assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + assert.Equal(t, tt.conflicts+1, resultWrites) + }) + } +} + func createCCRetryExecutionOperation(createTime time.Time, id string, operation v1alpha1.CruiseControlTaskOperation) *v1alpha1.CruiseControlOperation { return &v1alpha1.CruiseControlOperation{ ObjectMeta: v1.ObjectMeta{ diff --git a/docs/monitoring.md b/docs/monitoring.md index b6f802f1c..8d917bade 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -37,4 +37,21 @@ sum by (verb) (rate(rest_client_rate_limiter_duration_seconds_sum[5m])) Record cluster size, resource counts, workload, test duration, leader changes, and error rates with each measurement. These metrics establish instrumentation, not evidence that an optimization improves performance. This change does not -alter metrics authentication, scraper routing, worker counts, or cache scope. \ No newline at end of file +alter metrics authentication, scraper routing, worker counts, or cache scope. + +## Cruise Control execution-result persistence + +After an external task is submitted, the operator retries conflicting status +writes using fresh API-server reads. These persistence retries do not submit +the external task again. They retain the task identity, response metadata, +retry history, and selected stop-operation type. A replaced operation, removed +current task, or different task identity produces an error rather than accepting +a stale result. + +This is not exactly-once execution. If all writes fail, or the process stops +after Cruise Control accepts a request but before its task ID is recorded, the +external task can outlive the recorded operation state. Recorded task IDs allow +subsequent reconciliation to query task progress. Automatic adoption of an +unrecorded submission is not implemented; inspect Cruise Control's task history +when investigating this window, and do not infer that a missing status means +the external operation never ran. \ No newline at end of file From f022f61913658b17b705a737c545c368e36fc831 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 08:37:01 +0100 Subject: [PATCH 2/9] perf(controllers): refresh Cruise Control results only on conflict Reuse the selected operation for the first status write and read directly only when retrying a conflict. Cover conflict-free submission, retry, and stop paths. Refs #369. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cruisecontroloperation_controller.go | 6 +++++- .../cruisecontroloperation_controller_test.go | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index 68a343bb2..0bf53b9e8 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -257,11 +257,15 @@ func (r *CruiseControlOperationReconciler) persistExecutionResult( operation *banzaiv1alpha1.CruiseControlOperation, result *scale.Result, ) error { + firstAttempt := true return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { latest := &banzaiv1alpha1.CruiseControlOperation{} - if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + if firstAttempt { + latest = operation.DeepCopy() + } else if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { return err } + firstAttempt = false if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { return errors.New("CruiseControlOperation identity or current task changed after external task submission") } diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index 0c25b5c50..770ffa962 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -54,6 +54,9 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { wantFailed int wantOp v1alpha1.CruiseControlTaskOperation }{ + {name: "first submission without conflict", wantOp: v1alpha1.OperationAddBroker}, + {name: "retry without conflict preserves history once", retry: true, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission without conflict retains operation", deleting: true, wantOp: v1alpha1.OperationStopExecution}, {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, @@ -163,8 +166,16 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { return c.SubResource(subresource).Update(ctx, obj, opts...) }, }).Build() + var directReads int + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + directReads++ + assert.Positive(t, resultWrites, "direct GET must follow a conflicting result write") + return c.Get(ctx, key, obj, opts...) + }, + }) reconciler := &CruiseControlOperationReconciler{ - Client: kubeClient, DirectClient: kubeClient, Scheme: scheme, + Client: kubeClient, DirectClient: directClient, Scheme: scheme, ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { return scaler, nil }, @@ -197,8 +208,11 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) - assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + if tt.conflicts > 0 { + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + } assert.Equal(t, tt.conflicts+1, resultWrites) + assert.Equal(t, tt.conflicts, directReads) }) } } From eef4101275357774f25e5ee6cf5c1f346b78ea7d Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 10:16:06 +0100 Subject: [PATCH 3/9] fix(alerts): bound listener shutdown and report serving errors Return alert listener bind failures, drain requests on manager cancellation, and close connections after shutdown timeout or serve failure. Add focused lifecycle regressions. Refs #392 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- controllers/alertmanager_controller.go | 33 ++++++- controllers/controller_common_test.go | 122 +++++++++++++++++++++++++ 2 files changed, 152 insertions(+), 3 deletions(-) diff --git a/controllers/alertmanager_controller.go b/controllers/alertmanager_controller.go index 037b48b9b..2cf226ee3 100644 --- a/controllers/alertmanager_controller.go +++ b/controllers/alertmanager_controller.go @@ -16,8 +16,11 @@ package controllers import ( "context" + "errors" + "fmt" "net" "net/http" + "time" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -47,7 +50,31 @@ func (c AController) Start(ctx context.Context) error { logf.SetLogger(util.CreateLogger(false, false)) log := logf.Log.WithName("alertmanager") - ln, _ := net.Listen("tcp", receiverAddr) - httpServer := &http.Server{Handler: alertmanager.NewApp(log, c.Client)} - return httpServer.Serve(ln) + ln, err := net.Listen("tcp", receiverAddr) + if err != nil { + return fmt.Errorf("listen for alerts: %w", err) + } + return serveAlerts(ctx, ln, alertmanager.NewApp(log, c.Client), 5*time.Second) +} + +func serveAlerts(ctx context.Context, ln net.Listener, handler http.Handler, shutdownTimeout time.Duration) error { + httpServer := &http.Server{Handler: handler} + defer httpServer.Close() + defer ln.Close() + + serveDone := make(chan error, 1) + go func() { serveDone <- httpServer.Serve(ln) }() + select { + case err := <-serveDone: + return err + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + shutdownErr := httpServer.Shutdown(shutdownCtx) + serveErr := <-serveDone + if errors.Is(serveErr, http.ErrServerClosed) { + serveErr = nil + } + return errors.Join(shutdownErr, serveErr) + } } diff --git a/controllers/controller_common_test.go b/controllers/controller_common_test.go index eb049668c..e38f7f3c4 100644 --- a/controllers/controller_common_test.go +++ b/controllers/controller_common_test.go @@ -16,8 +16,14 @@ package controllers import ( + "context" "errors" + "fmt" + "io" + "net" + "net/http" "reflect" + "sync" "testing" "time" @@ -38,6 +44,122 @@ const ( testNamespace = "test-namespace" ) +func TestAlertManagerReportsBindError(t *testing.T) { + listener, err := net.Listen("tcp", receiverAddr) + if err != nil { + t.Skipf("cannot reserve alert listener for bind-error regression: %v", err) + } + defer listener.Close() + + defer func() { + if recovered := recover(); recovered != nil { + t.Errorf("Start panicked instead of returning the bind error: %v", recovered) + } + }() + + if err := (AController{}).Start(context.Background()); err == nil { + t.Error("Start returned nil for an occupied alert listener") + } +} + +func TestAlertServerLifecycle(t *testing.T) { + for _, tc := range []struct { + name string + timeout time.Duration + finish bool + serveError bool + expectedErr error + }{ + {name: "drains in-flight request", timeout: 5 * time.Second, finish: true}, + {name: "bounds shutdown and closes connection", timeout: 50 * time.Millisecond, expectedErr: context.DeadlineExceeded}, + {name: "returns serve error and closes connection", serveError: true, expectedErr: net.ErrClosed}, + } { + t.Run(tc.name, func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + finish := sync.OnceFunc(func() { close(release) }) + defer finish() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer close(stopped) + close(started) + select { + case <-release: + _, _ = w.Write([]byte("drained")) + case <-r.Context().Done(): + } + }) + serverDone := make(chan error, 1) + go func() { serverDone <- serveAlerts(ctx, listener, handler, tc.timeout) }() + requestDone := make(chan error, 1) + go func() { + httpClient := &http.Client{Timeout: 5 * time.Second} + response, requestErr := httpClient.Get("http://" + listener.Addr().String()) + if requestErr == nil { + defer response.Body.Close() + body, readErr := io.ReadAll(response.Body) + requestErr = readErr + if requestErr == nil && (response.StatusCode != http.StatusOK || string(body) != "drained") { + requestErr = fmt.Errorf("unexpected response: status=%d body=%q", response.StatusCode, body) + } + } + requestDone <- requestErr + }() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("alert handler did not start") + } + if tc.serveError { + if err := listener.Close(); err != nil { + t.Fatal(err) + } + } else { + cancel() + } + if tc.finish { + select { + case err := <-serverDone: + t.Fatalf("server returned before draining the request: %v", err) + case <-time.After(25 * time.Millisecond): + } + finish() + } + select { + case err := <-serverDone: + if !errors.Is(err, tc.expectedErr) { + t.Fatalf("server error = %v, want %v", err, tc.expectedErr) + } + case <-time.After(2 * time.Second): + t.Fatal("alert server did not stop") + } + select { + case <-stopped: + case <-time.After(2 * time.Second): + t.Fatal("alert handler connection was not closed") + } + select { + case err := <-requestDone: + if (err == nil) != tc.finish { + t.Fatalf("request error = %v, want successful drain = %t", err, tc.finish) + } + case <-time.After(2 * time.Second): + t.Fatal("alert request did not finish") + } + rebound, err := net.Listen("tcp", listener.Addr().String()) + if err != nil { + t.Fatalf("alert listener was not released: %v", err) + } + _ = rebound.Close() + }) + } +} + func TestRequeueAfter(t *testing.T) { sec := 5 res, err := requeueAfter(sec) From c176831dc9ec1ba7b83684a28f0124e9e78d696d Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 10:16:34 +0100 Subject: [PATCH 4/9] fix(chart): route protected metrics through the existing proxy Align PodMonitor protocol and port with proxy mode, keep raw metrics on loopback behind the proxy, and honor the configured upstream port without embedded bind-address quotes. Cover default/custom ports and proxy/direct modes in the existing chart validation block. Refs #392 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 35 +++++++++++++++++++ .../operator-deployment-with-webhook.yaml | 12 +++---- .../kafka-operator/templates/podmonitor.yaml | 6 ++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03c6d451f..3411a4ffc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,41 @@ jobs: for version in "$KUBERNETES_MIN_VERSION" "$KUBERNETES_MAX_VERSION"; do helm lint charts/kafka-operator --strict --kube-version "$version" helm template kafka-operator charts/kafka-operator --kube-version "$version" > /dev/null + for metrics in true false; do + for proxy in true false; do + for port in 8080 9090; do + echo "Checking metrics wiring: Kubernetes=$version metrics=$metrics proxy=$proxy port=$port" + args=( + --namespace serving-check --kube-version "$version" + --set webhook.enabled=false + --set prometheusMetrics.enabled="$metrics" + --set prometheusMetrics.authProxy.enabled="$proxy" + --set prometheusMetrics.podMonitor.enabled=true + ) + if [[ "$port" != 8080 ]]; then + args+=(--set metricEndpoint.port="$port") + fi + helm lint charts/kafka-operator --strict "${args[@]}" + rendered=$(helm template kafka-operator charts/kafka-operator "${args[@]}") + if [[ "$metrics" == true && "$proxy" == true ]]; then + bind="127.0.0.1:$port" + grep -Fq -- "--upstream=http://127.0.0.1:$port/" <<< "$rendered" + grep -Fxq ' port: https' <<< "$rendered" + grep -Fxq ' scheme: https' <<< "$rendered" + else + bind=":$port" + if grep -Fq -- '--upstream=' <<< "$rendered"; then + echo "::error::Inactive metrics proxy must not be rendered" + exit 1 + fi + grep -Fxq ' port: metrics' <<< "$rendered" + grep -Fxq ' scheme: http' <<< "$rendered" + fi + grep -Fxq " - --metrics-addr=$bind" <<< "$rendered" + grep -Fxq " - containerPort: $port" <<< "$rendered" + done + done + done done IFS=. read -r major minor _ <<< "$KUBERNETES_MIN_VERSION" unsupported="${major}.$((minor - 1)).99" diff --git a/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml b/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml index 15944d993..e244b31cf 100644 --- a/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml +++ b/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml @@ -1,3 +1,5 @@ +{{- $metricsPort := (.Values.metricEndpoint).port | default 8080 }} +{{- $metricsProxyEnabled := and .Values.prometheusMetrics.enabled .Values.prometheusMetrics.authProxy.enabled }} {{- $tlsCrt := "" }} {{- $tlsKey := "" }} {{- $caCrt := "" }} @@ -151,13 +153,13 @@ spec: {{ toYaml .Values.podSecurityContext | nindent 8 }} {{- end }} containers: - {{- if and .Values.prometheusMetrics.enabled .Values.prometheusMetrics.authProxy.enabled }} + {{- if $metricsProxyEnabled }} - name: kube-rbac-proxy image: "{{ .Values.prometheusMetrics.authProxy.image.repository }}:{{ .Values.prometheusMetrics.authProxy.image.tag }}" imagePullPolicy: {{ .Values.prometheusMetrics.authProxy.image.pullPolicy }} args: - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" + - "--upstream=http://127.0.0.1:{{ $metricsPort }}/" - "--v=10" ports: - containerPort: 8443 @@ -192,9 +194,7 @@ spec: {{- if .Values.operator.developmentLogging }} - --development {{- end }} - {{- if (.Values.metricEndpoint).port }} - - --metrics-addr=":{{ .Values.metricEndpoint.port }}" - {{- end }} + - --metrics-addr={{ if $metricsProxyEnabled }}127.0.0.1{{ end }}:{{ $metricsPort }} {{- if .Values.healthProbes.port }} - --health-probes-addr=:{{ .Values.healthProbes.port }} {{- end }} @@ -230,7 +230,7 @@ spec: name: webhook-server protocol: TCP {{- end}} - - containerPort: {{ (.Values.metricEndpoint).port | default 8080 }} + - containerPort: {{ $metricsPort }} name: metrics protocol: TCP - containerPort: {{ .Values.alertManager.port }} diff --git a/charts/kafka-operator/templates/podmonitor.yaml b/charts/kafka-operator/templates/podmonitor.yaml index da83c31ae..7d8f19fe2 100644 --- a/charts/kafka-operator/templates/podmonitor.yaml +++ b/charts/kafka-operator/templates/podmonitor.yaml @@ -26,6 +26,12 @@ spec: app.kubernetes.io/component: operator podMetricsEndpoints: - interval: {{ .Values.prometheusMetrics.podMonitor.interval }} + {{- if and .Values.prometheusMetrics.enabled .Values.prometheusMetrics.authProxy.enabled }} + port: https + scheme: https + {{- else }} port: metrics + scheme: http + {{- end }} path: /metrics {{- end }} From a79f6c0cce386c66e7ba9c020b75a28481f0b6b6 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 14:41:52 +0100 Subject: [PATCH 5/9] fix(alerts): publish leader-owned routes and serving readiness Scope routing label updates to uncached, UID-checked installation Pods, fail admission closed on observed cancellation, and bound self-label cleanup. Preserve legacy identity-absent listeners, usable webhook standbys, Kafka-independent liveness, and election identity bytes. Refs #392 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 15 + charts/kafka-operator/README.md | 57 +++ .../templates/alertmanager-service.yaml | 1 + .../operator-deployment-with-webhook.yaml | 10 + controllers/alertmanager_controller.go | 181 ++++++++- controllers/controller_common_test.go | 374 ++++++++++++++++++ main.go | 11 + main_test.go | 63 +++ 8 files changed, 708 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3411a4ffc..d446aa6db 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,6 +49,21 @@ jobs: for version in "$KUBERNETES_MIN_VERSION" "$KUBERNETES_MAX_VERSION"; do helm lint charts/kafka-operator --strict --kube-version "$version" helm template kafka-operator charts/kafka-operator --kube-version "$version" > /dev/null + deployment=$(helm template kafka-operator charts/kafka-operator \ + --kube-version "$version" --set webhook.enabled=false \ + --show-only templates/operator-deployment-with-webhook.yaml) + for identity in POD_NAMESPACE:metadata.namespace POD_NAME:metadata.name POD_UID:metadata.uid; do + echo "Checking downward Pod identity: $identity" + field=$(grep -A5 -Fx -- " - name: ${identity%%:*}" <<< "$deployment") + grep -Fxq -- " fieldPath: ${identity#*:}" <<< "$field" + done + if grep -Fq 'kafka.banzaicloud.io/alert-leader:' <<< "$deployment"; then + echo "::error::Only the elected process may publish the alert routing label" + exit 1 + fi + alerts=$(helm template kafka-operator charts/kafka-operator \ + --kube-version "$version" --show-only templates/alertmanager-service.yaml) + grep -Fxq ' kafka.banzaicloud.io/alert-leader: "true"' <<< "$alerts" for metrics in true false; do for proxy in true false; do for port in 8080 9090; do diff --git a/charts/kafka-operator/README.md b/charts/kafka-operator/README.md index 47c5fe046..cecf3353b 100644 --- a/charts/kafka-operator/README.md +++ b/charts/kafka-operator/README.md @@ -9,6 +9,63 @@ The [Koperator](https://github.com/adobe/koperator) is a Kubernetes operator to CI covers Kubernetes 1.35.0 and 1.37.0. The chart's `kubeVersion` constraint also accepts distribution-specific version suffixes. +## Alert routing, readiness, and election constraints + +The chart enables leader election. Its alert Service selects only the Pod +whose elected listener publishes `kafka.banzaicloud.io/alert-leader: "true"`. +Do not set this label statically on Pod templates. Webhook and metrics Services +retain their own selectors; webhook-serving standby replicas remain ready. +An elected replica is ready only after its alert listener and routing +publication are usable. Enabled webhooks also require local serving readiness. +Liveness does not depend on Kafka or leader status. + +The chart supplies `POD_NAMESPACE`, `POD_NAME`, and `POD_UID` through the downward +API. The operator reads its own Pod and routing peers directly from the API, +not the namespace-filtered Kafka cache. Managed routing requires: + +- The exact namespace/name/UID of the operator Pod, not a watched Kafka namespace. +- The chart's installation selector labels: `control-plane=controller-manager`, + `controller-tools.k8s.io=1.0`, `app.kubernetes.io/component=operator`, and + matching `app.kubernetes.io/name` and `app.kubernetes.io/instance`. +- Pod get/list/update permissions in the operator namespace, even when watched + Kafka resources are restricted to other namespaces. Existing chart RBAC + provides these permissions; custom/restricted RBAC must provide them. + +Before publication, the elected process removes stale active labels only from +that installation and namespace. Identity or label-write failures fail closed. +Observed manager cancellation closes admission, drains in-flight HTTP requests +for up to five seconds, and attempts bounded removal of its own label. Cleanup +failure cannot keep its listener open. Endpoint propagation is asynchronous; +stale connections may fail or receive HTTP 503 during failover. + +For existing local/non-Helm use, absence of **both** `POD_NAME` and `POD_UID` +preserves the legacy listener. `POD_NAMESPACE` alone does not enable managed +routing. Partial new identity, or complete new identity without a namespace, +is rejected. Non-Helm installations opting in must supply the complete identity, +installation labels and matching alert Service selector above. The existing +`alertManager.enable` value controls Service exposure, not listener startup. + +Leader-election identity is unchanged. Replicas sharing a mutation domain must +enable election, use byte-identical raw namespace input, and share the election +namespace. A nonempty `--namespaces` argument wins over `WATCH_NAMESPACE`. +Order, whitespace and duplicates in that raw string can change the lock ID +even when the cache watches equivalent sets. Do not normalize these inputs +during a rolling upgrade. The election namespace defaults to the in-cluster +service-account namespace, not the watched Kafka namespace set; `POD_NAMESPACE` +does not override it. + +Partially overlapping watch sets, differently spelled equivalent sets, +all-namespaces overlaps, and independent election namespaces are not supported +concurrent mutation arrangements. Use replicas of one installation behind its +alert Service, not separate releases with different Service selectors that +happen to share a Lease. Distinct election domains do not fence each other. + +Neither Lease election nor routing labels provide external-action fencing or +exactly-once execution. A process pause/partition before observed cancellation, +already admitted work, or an ambiguous submit-then-crash window can outlive +routing changes. HTTP draining and conflict retries do not resolve those +execution semantics or automatically adopt external operations. + ## Installing the chart Before installing the chart, you must first install the Koperator CustomResourceDefinition resources. diff --git a/charts/kafka-operator/templates/alertmanager-service.yaml b/charts/kafka-operator/templates/alertmanager-service.yaml index 7be7f3913..82369e157 100644 --- a/charts/kafka-operator/templates/alertmanager-service.yaml +++ b/charts/kafka-operator/templates/alertmanager-service.yaml @@ -18,6 +18,7 @@ metadata: app.kubernetes.io/component: alertmanager spec: selector: + kafka.banzaicloud.io/alert-leader: "true" control-plane: controller-manager controller-tools.k8s.io: "1.0" app.kubernetes.io/name: {{ include "kafka-operator.name" . }} diff --git a/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml b/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml index e244b31cf..e44fea975 100644 --- a/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml +++ b/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml @@ -216,6 +216,16 @@ spec: port: health-probes path: /readyz env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid - name: POD_NAMESPACE valueFrom: fieldRef: diff --git a/controllers/alertmanager_controller.go b/controllers/alertmanager_controller.go index 2cf226ee3..a36eaa3e4 100644 --- a/controllers/alertmanager_controller.go +++ b/controllers/alertmanager_controller.go @@ -20,8 +20,14 @@ import ( "fmt" "net" "net/http" + "os" + "sync" "time" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -31,18 +37,42 @@ import ( ) const ( - receiverAddr = ":9001" + receiverAddr = ":9001" + alertLeaderLabel = "kafka.banzaicloud.io/alert-leader" ) // AController implements Runnable type AController struct { - Client client.Client + Client client.Client + reader client.Reader + elected <-chan struct{} + state *alertServingState + pod types.NamespacedName + uid types.UID +} + +type alertServingState struct { + sync.RWMutex + started bool + ready bool + ctx context.Context } // SetAlertManagerWithManager creates a new Alertmanager Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func SetAlertManagerWithManager(mgr manager.Manager) error { - return mgr.Add(AController{Client: mgr.GetClient()}) + name, uid, namespace := os.Getenv("POD_NAME"), os.Getenv("POD_UID"), os.Getenv("POD_NAMESPACE") + if (name == "") != (uid == "") || (name != "" && namespace == "") { + return errors.New("managed alert routing requires POD_NAMESPACE, POD_NAME and POD_UID together") + } + c := &AController{ + Client: mgr.GetClient(), reader: mgr.GetAPIReader(), elected: mgr.Elected(), + state: &alertServingState{}, pod: types.NamespacedName{Namespace: namespace, Name: name}, uid: types.UID(uid), + } + if err := mgr.Add(c); err != nil { + return err + } + return mgr.AddReadyzCheck("alerts", c.readyz) } // Start initiates the alertmanager controller @@ -54,7 +84,150 @@ func (c AController) Start(ctx context.Context) error { if err != nil { return fmt.Errorf("listen for alerts: %w", err) } - return serveAlerts(ctx, ln, alertmanager.NewApp(log, c.Client), 5*time.Second) + return c.run(ctx, ln, alertmanager.NewApp(log, c.Client)) +} + +func (c AController) readyz(_ *http.Request) error { + if c.state == nil { + return errors.New("alert serving state is unavailable") + } + c.state.RLock() + defer c.state.RUnlock() + if !c.state.started { + select { + case <-c.elected: + return errors.New("elected alert listener has not started") + default: + return nil + } + } + if !c.state.ready || c.state.ctx.Err() != nil { + return errors.New("elected alert listener is not ready") + } + return nil +} + +func (c AController) run(ctx context.Context, ln net.Listener, handler http.Handler) (err error) { + if c.state == nil { + c.state = &alertServingState{} + } + c.state.Lock() + c.state.started, c.state.ready, c.state.ctx = true, false, ctx + c.state.Unlock() + var scope client.MatchingLabels + defer func() { + c.state.Lock() + c.state.ready = false + c.state.Unlock() + _ = ln.Close() + if len(scope) != 0 { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if cleanupErr := c.updateRouteLabel(cleanupCtx, c.pod, c.uid, scope, false); cleanupErr != nil { + err = errors.Join(err, fmt.Errorf("remove alert routing label: %w", cleanupErr)) + } + } + }() + if c.pod.Name != "" { + scope, err = c.prepareRouting(ctx) + if err != nil { + return err + } + } + c.state.Lock() + c.state.ready = true + c.state.Unlock() + return serveAlerts(ctx, ln, c.gatedHandler(ctx, handler), 5*time.Second) +} + +func (c AController) gatedHandler(ctx context.Context, handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.state.RLock() + ready := c.state.ready + c.state.RUnlock() + if ctx.Err() != nil || !ready { + http.Error(w, "alert listener is not accepting requests", http.StatusServiceUnavailable) + return + } + handler.ServeHTTP(w, r) + }) +} + +func (c AController) prepareRouting(ctx context.Context) (client.MatchingLabels, error) { + self := &corev1.Pod{} + if err := c.reader.Get(ctx, c.pod, self); err != nil { + return nil, fmt.Errorf("read alert listener Pod: %w", err) + } + if self.UID != c.uid { + return nil, errors.New("alert listener Pod UID does not match POD_UID") + } + scope := client.MatchingLabels{} + for _, key := range []string{ + "control-plane", "controller-tools.k8s.io", "app.kubernetes.io/name", + "app.kubernetes.io/instance", "app.kubernetes.io/component", + } { + if self.Labels[key] == "" { + return nil, fmt.Errorf("alert listener Pod requires installation label %s", key) + } + scope[key] = self.Labels[key] + } + if scope["control-plane"] != "controller-manager" || scope["controller-tools.k8s.io"] != "1.0" || + scope["app.kubernetes.io/component"] != "operator" { + return nil, errors.New("alert listener Pod labels do not match the alert Service selector") + } + pods := &corev1.PodList{} + if err := c.reader.List(ctx, pods, client.InNamespace(c.pod.Namespace), scope); err != nil { + return scope, fmt.Errorf("list alert routing Pods: %w", err) + } + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Labels[alertLeaderLabel] == True { + if err := c.updateRouteLabel(ctx, client.ObjectKeyFromObject(pod), pod.UID, scope, false); err != nil { + return scope, fmt.Errorf("clear stale alert routing label: %w", err) + } + } + } + if err := c.updateRouteLabel(ctx, c.pod, c.uid, scope, true); err != nil { + return scope, fmt.Errorf("publish alert routing label: %w", err) + } + return scope, nil +} + +func (c AController) updateRouteLabel(ctx context.Context, key client.ObjectKey, uid types.UID, scope client.MatchingLabels, active bool) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + pod := &corev1.Pod{} + if err := c.reader.Get(ctx, key, pod); err != nil { + if !active && apierrors.IsNotFound(err) { + return nil + } + return err + } + matches := pod.UID == uid + for label, value := range scope { + matches = matches && pod.Labels[label] == value + } + if !matches { + if !active { + return nil + } + return errors.New("alert routing Pod identity or installation labels changed") + } + if err := ctx.Err(); err != nil { + return err + } + if active { + if pod.DeletionTimestamp != nil { + return errors.New("cannot route alerts to a terminating Pod") + } + pod.Labels[alertLeaderLabel] = True + } else { + if pod.Labels[alertLeaderLabel] != True { + return nil + } + delete(pod.Labels, alertLeaderLabel) + } + return c.Client.Update(ctx, pod) + }) } func serveAlerts(ctx context.Context, ln net.Listener, handler http.Handler, shutdownTimeout time.Duration) error { diff --git a/controllers/controller_common_test.go b/controllers/controller_common_test.go index e38f7f3c4..f112610f9 100644 --- a/controllers/controller_common_test.go +++ b/controllers/controller_common_test.go @@ -22,15 +22,27 @@ import ( "io" "net" "net/http" + "net/http/httptest" "reflect" "sync" + "sync/atomic" "testing" "time" emperrors "emperror.dev/errors" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + //nolint:staticcheck "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/healthz" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" @@ -44,6 +56,361 @@ const ( testNamespace = "test-namespace" ) +type alertTestManager struct { + manager.Manager + client client.Client + reader client.Reader + elected chan struct{} + runnable manager.Runnable + check healthz.Checker +} + +func (m *alertTestManager) GetClient() client.Client { return m.client } +func (m *alertTestManager) GetAPIReader() client.Reader { return m.reader } +func (m *alertTestManager) Elected() <-chan struct{} { return m.elected } +func (m *alertTestManager) Add(r manager.Runnable) error { m.runnable = r; return nil } +func (m *alertTestManager) AddReadyzCheck(_ string, check healthz.Checker) error { + m.check = check + return nil +} + +func TestAlertManagerIdentityRegistration(t *testing.T) { + for _, tc := range []struct { + name, namespace, pod, uid string + wantErr bool + }{ + {name: "legacy"}, + {name: "namespace alone remains legacy", namespace: "operators"}, + {name: "name alone rejected", namespace: "operators", pod: "operator", wantErr: true}, + {name: "UID alone rejected", namespace: "operators", uid: "uid", wantErr: true}, + {name: "missing namespace rejected", pod: "operator", uid: "uid", wantErr: true}, + {name: "complete", namespace: "operators", pod: "operator", uid: "uid"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("POD_NAMESPACE", tc.namespace) + t.Setenv("POD_NAME", tc.pod) + t.Setenv("POD_UID", tc.uid) + mgr := &alertTestManager{elected: make(chan struct{})} + err := SetAlertManagerWithManager(mgr) + if (err != nil) != tc.wantErr { + t.Fatalf("registration error = %v, want error = %t", err, tc.wantErr) + } + if !tc.wantErr && mgr.check == nil { + t.Fatal("alert readiness was not registered") + } + }) + } +} + +type alertTestClient struct { + client.Client + update func(context.Context, client.Object, ...client.UpdateOption) error +} + +func (c *alertTestClient) Get(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("operator Pod is outside cached namespaces") +} + +func (c *alertTestClient) List(context.Context, client.ObjectList, ...client.ListOption) error { + return errors.New("operator Pods are outside cached namespaces") +} + +func (c *alertTestClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if c.update != nil { + return c.update(ctx, obj, opts...) + } + return c.Client.Update(ctx, obj, opts...) +} + +type alertTestReader struct { + client.Reader + getErr, listErr error +} + +func (r alertTestReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if r.getErr != nil { + return r.getErr + } + return r.Reader.Get(ctx, key, obj, opts...) +} + +func (r alertTestReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if r.listErr != nil { + return r.listErr + } + return r.Reader.List(ctx, list, opts...) +} + +func alertTestPod(name, release string) *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "operators", UID: types.UID(name), + Labels: map[string]string{ + "control-plane": "controller-manager", "controller-tools.k8s.io": "1.0", + "app.kubernetes.io/name": "kafka-operator", "app.kubernetes.io/instance": release, + "app.kubernetes.io/component": "operator", "kafka.banzaicloud.io/alert-leader": "true", + }, + }} +} + +func newAlertTestController(pods ...client.Object) (AController, *alertTestClient) { + storage := fake.NewClientBuilder().WithObjects(pods...).Build() + writer := &alertTestClient{Client: storage} + elected := make(chan struct{}) + close(elected) + return AController{ + Client: writer, reader: storage, elected: elected, state: &alertServingState{}, + pod: types.NamespacedName{Namespace: "operators", Name: "self"}, uid: "self", + }, writer +} + +func runAlertForTest(t *testing.T, c AController, handler http.Handler) (context.CancelFunc, <-chan error, string) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + done, stopped := make(chan error, 1), make(chan struct{}) + go func() { + done <- c.run(ctx, listener, handler) + close(stopped) + }() + t.Cleanup(func() { + cancel() + _ = listener.Close() + select { + case <-stopped: + case <-time.After(4 * time.Second): + t.Error("alert controller did not stop within its cleanup bound") + } + }) + return cancel, done, "http://" + listener.Addr().String() +} + +func TestAlertRoutingUsesScopedUncachedIdentity(t *testing.T) { + self, stale, other := alertTestPod("self", "release"), alertTestPod("stale", "release"), alertTestPod("other", "other-release") + foreign := alertTestPod("foreign", "release") + foreign.Namespace = "other-namespace" + c, writer := newAlertTestController(self, stale, other, foreign) + var handled atomic.Int32 + cancel, done, address := runAlertForTest(t, c, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + handled.Add(1) + _, _ = w.Write([]byte("ok")) + })) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + for _, expected := range []struct { + pod *corev1.Pod + active bool + }{{self, true}, {stale, false}, {other, true}, {foreign, true}} { + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), client.ObjectKeyFromObject(expected.pod), current)) + require.Equal(t, expected.active, current.Labels["kafka.banzaicloud.io/alert-leader"] == "true", expected.pod.Name) + } + transport := &http.Transport{} + defer transport.CloseIdleConnections() + httpClient := &http.Client{Transport: transport, Timeout: time.Second} + response, err := httpClient.Get(address) + require.NoError(t, err) + _, err = io.ReadAll(response.Body) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + cancel() + require.Error(t, c.readyz(nil), "observed cancellation must immediately fail elected readiness") + response, err = httpClient.Get(address) + if err == nil { + require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) + require.NoError(t, response.Body.Close()) + } + require.NoError(t, <-done) + require.EqualValues(t, 1, handled.Load(), "stale keepalive traffic reached the receiver") + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Empty(t, current.Labels["kafka.banzaicloud.io/alert-leader"]) +} + +func TestAlertRoutingStartupFailures(t *testing.T) { + for _, failure := range []string{"UID", "labels", "get", "deleted", "list", "write", "conflicts", "ambiguous"} { + t.Run(failure, func(t *testing.T) { + self := alertTestPod("self", "release") + delete(self.Labels, "kafka.banzaicloud.io/alert-leader") + if failure == "labels" { + delete(self.Labels, "app.kubernetes.io/instance") + } + c, writer := newAlertTestController(self) + switch failure { + case "UID": + c.uid = "replacement" + case "get": + c.reader = alertTestReader{Reader: c.reader, getErr: errors.New("read denied")} + case "deleted": + c.reader = alertTestReader{Reader: c.reader, getErr: apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, "self")} + case "list": + c.reader = alertTestReader{Reader: c.reader, listErr: errors.New("list denied")} + case "write", "conflicts", "ambiguous": + writer.update = func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if obj.GetLabels()["kafka.banzaicloud.io/alert-leader"] == "true" { + if failure == "ambiguous" { + if err := writer.Client.Update(ctx, obj, opts...); err != nil { + return err + } + } + if failure == "conflicts" { + return apierrors.NewConflict(schema.GroupResource{Resource: "pods"}, "self", errors.New("conflict")) + } + return errors.New("publication failed") + } + return writer.Client.Update(ctx, obj, opts...) + } + } + _, done, _ := runAlertForTest(t, c, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("failed startup admitted an alert") + })) + select { + case err := <-done: + require.Error(t, err) + case <-time.After(time.Second): + t.Fatal("startup failure was not returned") + } + require.Error(t, c.readyz(nil)) + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Empty(t, current.Labels["kafka.banzaicloud.io/alert-leader"]) + }) + } +} + +func TestAlertRoutingConflictPreservesMetadata(t *testing.T) { + self := alertTestPod("self", "release") + delete(self.Labels, "kafka.banzaicloud.io/alert-leader") + c, writer := newAlertTestController(self) + var updates atomic.Int32 + writer.update = func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if updates.Add(1) == 1 { + current := &corev1.Pod{} + if err := writer.Client.Get(ctx, c.pod, current); err != nil { + return err + } + current.Labels["concurrent-label"] = "preserved" + if err := writer.Client.Update(ctx, current); err != nil { + return err + } + return apierrors.NewConflict(schema.GroupResource{Resource: "pods"}, "self", errors.New("concurrent update")) + } + return writer.Client.Update(ctx, obj, opts...) + } + cancel, done, _ := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + cancel() + require.NoError(t, <-done) + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Equal(t, "preserved", current.Labels["concurrent-label"]) + require.GreaterOrEqual(t, updates.Load(), int32(3)) +} + +func TestAlertReadinessPreservesStandbysAndLegacy(t *testing.T) { + c, _ := newAlertTestController() + elected := make(chan struct{}) + c.elected = elected + require.NoError(t, c.readyz(nil), "standby must not require an alert listener") + close(elected) + require.Error(t, c.readyz(nil), "elected startup must not claim serving readiness") + c.pod, c.uid = types.NamespacedName{}, "" + cancel, done, _ := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + cancel() + require.Error(t, c.readyz(nil)) + require.NoError(t, <-done) +} + +func TestAlertRoutingGateAndBoundedCleanup(t *testing.T) { + c, writer := newAlertTestController(alertTestPod("self", "release")) + cleanupStarted := make(chan struct{}) + var cleanupOnce sync.Once + writer.update = func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if obj.GetLabels()["kafka.banzaicloud.io/alert-leader"] != "true" { + // Only block final cleanup, not startup's stale-label removal. + select { + case <-cleanupStarted: + default: + c.state.RLock() + stopping := c.state.ctx != nil && c.state.ctx.Err() != nil + c.state.RUnlock() + if stopping { + cleanupOnce.Do(func() { close(cleanupStarted) }) + <-ctx.Done() + return ctx.Err() + } + } + } + return writer.Client.Update(ctx, obj, opts...) + } + cancel, done, address := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + cancel() + select { + case <-cleanupStarted: + case <-time.After(time.Second): + t.Fatal("cleanup did not start") + } + connection, err := net.DialTimeout("tcp", address[len("http://"):], time.Second) + if connection != nil { + _ = connection.Close() + } + require.Error(t, err, "listener remained open while cleanup was blocked") + c.state.RLock() + ctx := c.state.ctx + c.state.RUnlock() + recorder := httptest.NewRecorder() + c.gatedHandler(ctx, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("canceled admission reached the receiver") + })).ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/", nil)) + require.Equal(t, http.StatusServiceUnavailable, recorder.Code) + select { + case err := <-done: + require.ErrorIs(t, err, context.DeadlineExceeded) + case <-time.After(3 * time.Second): + t.Fatal("blocked label cleanup exceeded its bound") + } +} + +func TestAlertRoutingCleanupPreservesReplacementUID(t *testing.T) { + self := alertTestPod("self", "release") + c, writer := newAlertTestController(self) + cancel, done, _ := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + require.NoError(t, writer.Delete(context.Background(), self)) + replacement := alertTestPod("self", "release") + replacement.UID = "replacement" + require.NoError(t, writer.Create(context.Background(), replacement)) + cancel() + require.NoError(t, <-done) + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Equal(t, "true", current.Labels["kafka.banzaicloud.io/alert-leader"]) +} + +func TestAlertRoutingServeFailureRemovesLabel(t *testing.T) { + c, writer := newAlertTestController(alertTestPod("self", "release")) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- c.run(ctx, listener, http.NotFoundHandler()) }() + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + require.NoError(t, listener.Close()) + select { + case err := <-done: + require.ErrorIs(t, err, net.ErrClosed) + case <-time.After(3 * time.Second): + t.Fatal("Serve failure did not stop routing") + } + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Empty(t, current.Labels["kafka.banzaicloud.io/alert-leader"]) + require.Error(t, c.readyz(nil)) +} + func TestAlertManagerReportsBindError(t *testing.T) { listener, err := net.Listen("tcp", receiverAddr) if err != nil { @@ -60,6 +427,13 @@ func TestAlertManagerReportsBindError(t *testing.T) { if err := (AController{}).Start(context.Background()); err == nil { t.Error("Start returned nil for an occupied alert listener") } + c, writer := newAlertTestController(alertTestPod("self", "release")) + writer.update = func(context.Context, client.Object, ...client.UpdateOption) error { + t.Error("bind failure attempted to change a routing label") + return errors.New("unexpected update") + } + require.Error(t, c.Start(context.Background())) + require.Error(t, c.readyz(nil)) } func TestAlertServerLifecycle(t *testing.T) { diff --git a/main.go b/main.go index a854dd863..88b43f844 100644 --- a/main.go +++ b/main.go @@ -174,6 +174,10 @@ func main() { setupLog.Error(err, "unable to start /readyz endpoint") os.Exit(1) } + if err = addWebhookReadyzCheck(mgr, webhookDisabled); err != nil { + setupLog.Error(err, "unable to add webhook readiness check") + os.Exit(1) + } if err := certv1.AddToScheme(mgr.GetScheme()); err != nil { setupLog.Error(err, "") @@ -288,3 +292,10 @@ func main() { os.Exit(1) } } + +func addWebhookReadyzCheck(mgr ctrl.Manager, disabled bool) error { + if disabled { + return nil + } + return mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()) +} diff --git a/main_test.go b/main_test.go index b3841021d..98b521b88 100644 --- a/main_test.go +++ b/main_test.go @@ -16,6 +16,7 @@ package main import ( "context" + "errors" "net/http" "net/http/httptest" "testing" @@ -23,9 +24,71 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics" + "sigs.k8s.io/controller-runtime/pkg/webhook" ) +type webhookReadyTestServer struct { + webhook.Server + err error +} + +func (s *webhookReadyTestServer) StartedChecker() healthz.Checker { + return func(*http.Request) error { return s.err } +} + +type webhookReadyTestManager struct { + ctrl.Manager + server *webhookReadyTestServer + check healthz.Checker + serverRequests, healthRegistrations int +} + +func (m *webhookReadyTestManager) GetWebhookServer() webhook.Server { + m.serverRequests++ + return m.server +} + +func (m *webhookReadyTestManager) AddReadyzCheck(_ string, check healthz.Checker) error { + m.check = check + return nil +} + +func (m *webhookReadyTestManager) AddHealthzCheck(string, healthz.Checker) error { + m.healthRegistrations++ + return nil +} + +func TestWebhookReadinessConditionalRegistration(t *testing.T) { + for _, disabled := range []bool{false, true} { + mgr := &webhookReadyTestManager{server: &webhookReadyTestServer{err: errors.New("not serving TLS")}} + if err := addWebhookReadyzCheck(mgr, disabled); err != nil { + t.Fatal(err) + } + if mgr.healthRegistrations != 0 { + t.Fatal("serving readiness changed liveness registration") + } + if disabled { + if mgr.serverRequests != 0 || mgr.check != nil { + t.Fatal("disabled webhooks started or registered a TLS server") + } + continue + } + if mgr.check == nil || mgr.serverRequests != 1 { + t.Fatal("enabled webhook serving readiness was not registered") + } + if err := mgr.check(httptest.NewRequest(http.MethodGet, "/readyz", nil)); err == nil { + t.Fatal("unusable webhook standby reported ready") + } + mgr.server.err = nil + if err := mgr.check(httptest.NewRequest(http.MethodGet, "/readyz", nil)); err != nil { + t.Fatalf("usable webhook standby is not ready: %v", err) + } + } +} + func TestRESTRequestMetricsAreExposed(t *testing.T) { apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") From 7c2c3da2a7cbfa42d27c6c08dce5fcdfc12dc013 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 14:48:33 +0100 Subject: [PATCH 6/9] fix(chart): grant proxy reviews to the actual Pod account Preserve the configured proxy subject, avoid duplicate shared subjects, and test default, custom, shared and externally managed RBAC cases. Document caller-owned scrape authentication/trust, deferred managed integration and existing certificate renewal limits. Refs #392 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 31 ++++++++++++ charts/kafka-operator/README.md | 47 +++++++++++++++++++ .../templates/authproxy-rbac.yaml | 5 ++ 3 files changed, 83 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d446aa6db..4d9314c9e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,6 +64,37 @@ jobs: alerts=$(helm template kafka-operator charts/kafka-operator \ --kube-version "$version" --show-only templates/alertmanager-service.yaml) grep -Fxq ' kafka.banzaicloud.io/alert-leader: "true"' <<< "$alerts" + for accounts in kafka-operator:kafka-operator-authproxy:2 custom-operator:custom-proxy:2 shared:shared:1; do + IFS=: read -r operator_account proxy_account subject_count <<< "$accounts" + echo "Checking proxy review subjects: Kubernetes=$version operator=$operator_account proxy=$proxy_account" + args=( + --namespace serving-check --kube-version "$version" + --set webhook.enabled=false + --set operator.serviceAccount.name="$operator_account" + --set prometheusMetrics.authProxy.serviceAccount.name="$proxy_account" + ) + if [[ "$operator_account" != kafka-operator ]]; then + args+=(--set operator.serviceAccount.create=false --set prometheusMetrics.authProxy.serviceAccount.create=false) + fi + deployment=$(helm template kafka-operator charts/kafka-operator "${args[@]}" \ + --show-only templates/operator-deployment-with-webhook.yaml) + grep -Fxq " serviceAccountName: $operator_account" <<< "$deployment" + binding=$(helm template kafka-operator charts/kafka-operator "${args[@]}" \ + --show-only templates/authproxy-rbac.yaml) + subjects=$(sed -n '/^subjects:/,$p' <<< "$binding") + grep -Fxq " name: $operator_account" <<< "$subjects" + grep -Fxq " name: $proxy_account" <<< "$subjects" + [[ $(grep -c '^- kind: ServiceAccount$' <<< "$subjects") == "$subject_count" ]] + [[ $(grep -c '^ namespace: serving-check$' <<< "$subjects") == "$subject_count" ]] + done + echo "Checking externally supplied proxy RBAC: Kubernetes=$version" + external_rbac=$(helm template kafka-operator charts/kafka-operator \ + --kube-version "$version" --set webhook.enabled=false --set rbac.enabled=false \ + --show-only templates/authproxy-rbac.yaml) + if grep -Eq '^kind: ClusterRole(Binding)?$' <<< "$external_rbac"; then + echo "::error::rbac.enabled=false must leave proxy review permissions to the installer" + exit 1 + fi for metrics in true false; do for proxy in true false; do for port in 8080 9090; do diff --git a/charts/kafka-operator/README.md b/charts/kafka-operator/README.md index cecf3353b..f89e5f772 100644 --- a/charts/kafka-operator/README.md +++ b/charts/kafka-operator/README.md @@ -18,6 +18,8 @@ retain their own selectors; webhook-serving standby replicas remain ready. An elected replica is ready only after its alert listener and routing publication are usable. Enabled webhooks also require local serving readiness. Liveness does not depend on Kafka or leader status. +Upgrades from versions without the routing label have no alert endpoint until a +label-aware replica becomes leader; plan alert retries during that handoff. The chart supplies `POD_NAMESPACE`, `POD_NAME`, and `POD_UID` through the downward API. The operator reads its own Pod and routing peers directly from the API, @@ -66,6 +68,51 @@ already admitted work, or an ambiguous submit-then-crash window can outlive routing changes. HTTP draining and conflict retries do not resolve those execution semantics or automatically adopt external operations. +## Protected metrics prerequisites + +When metrics and the auth proxy are enabled, the manager's raw metrics listener +binds to loopback. The proxy exposes HTTPS on port 8443, and the managed +PodMonitor targets its `https` port rather than bypassing it. Port/protocol +alignment alone does not establish a working authenticated scrape. + +All containers in the operator Pod share `operator.serviceAccount.name` +(`default` if the name is empty). +`prometheusMetrics.authProxy.serviceAccount.name` does not give the sidecar a +separate identity. With `rbac.enabled=true`, the proxy review-role binding +includes the actual Pod account and preserves the configured proxy account, +without duplicating a shared account. This also applies when account creation +is disabled. With `rbac.enabled=false`, the installer must grant the actual Pod +account `create` on `authentication.k8s.io/tokenreviews` and +`authorization.k8s.io/subjectaccessreviews`. + +A working protected scrape separately requires a scraping identity authorized +to `get` the non-resource URL `/metrics`, usable proxy HTTPS serving credentials, +and scraper trust in that certificate's CA and expected DNS/server name. Review +permissions for the proxy do not authorize the scraper. The webhook certificate +is not automatically wired into the metrics proxy. + +Managed PodMonitor authentication/CA/server-name integration is **deferred**: +the chart has no managed monitor fields for bearer-token/client-certificate +credentials, trusted CA, or TLS server name, and does not configure proxy +certificate arguments or mounts. Supply caller-owned external workload and +scrape configuration for these requirements; enabling chart values or adding +`additionalVolumes` alone does not solve them. `additionalSidecars` does not +configure the existing proxy. Do not disable TLS verification to work around +missing trust configuration. Validate authenticated scraping and rejection of +unauthorized requests against the actual endpoint before relying on it. + +### Existing webhook certificate renewal limits + +Helm-generated webhook serving certificates last 365 days (the generated CA +lasts 3650 days). The chart has no continuous renewal controller. A Helm upgrade +with generation enabled regenerates CA and serving material; it is not a +guaranteed zero-downtime rotation. Plan renewal before expiry and coordinate the +Secret, webhook CA bundle, and serving Pod rollout. With +`webhook.certs.generate=false`, the caller must refresh the supplied CA, +certificate, and key through the existing certificate values and deployment +process. No metrics certificate reuse, issuance, or automatic renewal is added +by the proxy wiring/RBAC fixes. + ## Installing the chart Before installing the chart, you must first install the Koperator CustomResourceDefinition resources. diff --git a/charts/kafka-operator/templates/authproxy-rbac.yaml b/charts/kafka-operator/templates/authproxy-rbac.yaml index 4a9dbc414..079da3e4c 100644 --- a/charts/kafka-operator/templates/authproxy-rbac.yaml +++ b/charts/kafka-operator/templates/authproxy-rbac.yaml @@ -58,4 +58,9 @@ subjects: - kind: ServiceAccount name: {{ include "operator.metricsAuthProxy.serviceAccountName" .}} namespace: {{ .Release.Namespace }} +{{- if ne (include "operator.serviceAccountName" .) (include "operator.metricsAuthProxy.serviceAccountName" .) }} +- kind: ServiceAccount + name: {{ include "operator.serviceAccountName" .}} + namespace: {{ .Release.Namespace }} +{{- end }} {{- end }} From cabcf8ae7a35b968037b297599e13d3bd9c5c2aa Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 14:50:55 +0100 Subject: [PATCH 7/9] test(alerts): reject canceled requests on reused connections Exercise the live admission gate on an explicitly reused HTTP connection while the serving flag remains set, so stale routing cannot bypass observed manager cancellation. Refs #392 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- controllers/controller_common_test.go | 33 +++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/controllers/controller_common_test.go b/controllers/controller_common_test.go index f112610f9..6024d0b58 100644 --- a/controllers/controller_common_test.go +++ b/controllers/controller_common_test.go @@ -23,6 +23,7 @@ import ( "net" "net/http" "net/http/httptest" + "net/http/httptrace" "reflect" "sync" "sync/atomic" @@ -372,6 +373,38 @@ func TestAlertRoutingGateAndBoundedCleanup(t *testing.T) { } } +func TestAlertRoutingRejectsCanceledKeepalive(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := AController{state: &alertServingState{started: true, ready: true, ctx: ctx}} + var admitted atomic.Int32 + // Keep the connection open to model stale routing during listener shutdown. + server := httptest.NewServer(c.gatedHandler(ctx, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + admitted.Add(1) + w.WriteHeader(http.StatusNoContent) + }))) + defer server.Close() + transport := &http.Transport{} + defer transport.CloseIdleConnections() + httpClient := &http.Client{Transport: transport, Timeout: time.Second} + response, err := httpClient.Post(server.URL, "application/json", nil) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + require.Equal(t, http.StatusNoContent, response.StatusCode) + + cancel() + var reused atomic.Bool + trace := &httptrace.ClientTrace{GotConn: func(info httptrace.GotConnInfo) { reused.Store(info.Reused) }} + request, err := http.NewRequestWithContext(httptrace.WithClientTrace(context.Background(), trace), http.MethodPost, server.URL, nil) + require.NoError(t, err) + response, err = httpClient.Do(request) + require.NoError(t, err) + defer response.Body.Close() + require.True(t, reused.Load(), "request did not reuse the stale connection") + require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) + require.Equal(t, int32(1), admitted.Load(), "canceled keepalive request reached the receiver") +} + func TestAlertRoutingCleanupPreservesReplacementUID(t *testing.T) { self := alertTestPod("self", "release") c, writer := newAlertTestController(self) From ea5bdb3721b2f0d0cc5d345b96883e4beea5b79c Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 19:35:32 +0100 Subject: [PATCH 8/9] chore: simplify serving validation and guidance Restore the existing supported-version Helm lint/render checks instead of maintaining the expanded configuration matrix and formatting-sensitive assertions. Consolidate listener setup and duplicated cancellation checks while retaining the focused serving failure regressions and explicit reused-connection test. Condense installation guidance without changing runtime code or chart templates. Refs #392, #401 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 81 ------------- charts/kafka-operator/README.md | 159 +++++++++++--------------- controllers/controller_common_test.go | 96 ++++------------ 3 files changed, 86 insertions(+), 250 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4d9314c9e..03c6d451f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,87 +49,6 @@ jobs: for version in "$KUBERNETES_MIN_VERSION" "$KUBERNETES_MAX_VERSION"; do helm lint charts/kafka-operator --strict --kube-version "$version" helm template kafka-operator charts/kafka-operator --kube-version "$version" > /dev/null - deployment=$(helm template kafka-operator charts/kafka-operator \ - --kube-version "$version" --set webhook.enabled=false \ - --show-only templates/operator-deployment-with-webhook.yaml) - for identity in POD_NAMESPACE:metadata.namespace POD_NAME:metadata.name POD_UID:metadata.uid; do - echo "Checking downward Pod identity: $identity" - field=$(grep -A5 -Fx -- " - name: ${identity%%:*}" <<< "$deployment") - grep -Fxq -- " fieldPath: ${identity#*:}" <<< "$field" - done - if grep -Fq 'kafka.banzaicloud.io/alert-leader:' <<< "$deployment"; then - echo "::error::Only the elected process may publish the alert routing label" - exit 1 - fi - alerts=$(helm template kafka-operator charts/kafka-operator \ - --kube-version "$version" --show-only templates/alertmanager-service.yaml) - grep -Fxq ' kafka.banzaicloud.io/alert-leader: "true"' <<< "$alerts" - for accounts in kafka-operator:kafka-operator-authproxy:2 custom-operator:custom-proxy:2 shared:shared:1; do - IFS=: read -r operator_account proxy_account subject_count <<< "$accounts" - echo "Checking proxy review subjects: Kubernetes=$version operator=$operator_account proxy=$proxy_account" - args=( - --namespace serving-check --kube-version "$version" - --set webhook.enabled=false - --set operator.serviceAccount.name="$operator_account" - --set prometheusMetrics.authProxy.serviceAccount.name="$proxy_account" - ) - if [[ "$operator_account" != kafka-operator ]]; then - args+=(--set operator.serviceAccount.create=false --set prometheusMetrics.authProxy.serviceAccount.create=false) - fi - deployment=$(helm template kafka-operator charts/kafka-operator "${args[@]}" \ - --show-only templates/operator-deployment-with-webhook.yaml) - grep -Fxq " serviceAccountName: $operator_account" <<< "$deployment" - binding=$(helm template kafka-operator charts/kafka-operator "${args[@]}" \ - --show-only templates/authproxy-rbac.yaml) - subjects=$(sed -n '/^subjects:/,$p' <<< "$binding") - grep -Fxq " name: $operator_account" <<< "$subjects" - grep -Fxq " name: $proxy_account" <<< "$subjects" - [[ $(grep -c '^- kind: ServiceAccount$' <<< "$subjects") == "$subject_count" ]] - [[ $(grep -c '^ namespace: serving-check$' <<< "$subjects") == "$subject_count" ]] - done - echo "Checking externally supplied proxy RBAC: Kubernetes=$version" - external_rbac=$(helm template kafka-operator charts/kafka-operator \ - --kube-version "$version" --set webhook.enabled=false --set rbac.enabled=false \ - --show-only templates/authproxy-rbac.yaml) - if grep -Eq '^kind: ClusterRole(Binding)?$' <<< "$external_rbac"; then - echo "::error::rbac.enabled=false must leave proxy review permissions to the installer" - exit 1 - fi - for metrics in true false; do - for proxy in true false; do - for port in 8080 9090; do - echo "Checking metrics wiring: Kubernetes=$version metrics=$metrics proxy=$proxy port=$port" - args=( - --namespace serving-check --kube-version "$version" - --set webhook.enabled=false - --set prometheusMetrics.enabled="$metrics" - --set prometheusMetrics.authProxy.enabled="$proxy" - --set prometheusMetrics.podMonitor.enabled=true - ) - if [[ "$port" != 8080 ]]; then - args+=(--set metricEndpoint.port="$port") - fi - helm lint charts/kafka-operator --strict "${args[@]}" - rendered=$(helm template kafka-operator charts/kafka-operator "${args[@]}") - if [[ "$metrics" == true && "$proxy" == true ]]; then - bind="127.0.0.1:$port" - grep -Fq -- "--upstream=http://127.0.0.1:$port/" <<< "$rendered" - grep -Fxq ' port: https' <<< "$rendered" - grep -Fxq ' scheme: https' <<< "$rendered" - else - bind=":$port" - if grep -Fq -- '--upstream=' <<< "$rendered"; then - echo "::error::Inactive metrics proxy must not be rendered" - exit 1 - fi - grep -Fxq ' port: metrics' <<< "$rendered" - grep -Fxq ' scheme: http' <<< "$rendered" - fi - grep -Fxq " - --metrics-addr=$bind" <<< "$rendered" - grep -Fxq " - containerPort: $port" <<< "$rendered" - done - done - done done IFS=. read -r major minor _ <<< "$KUBERNETES_MIN_VERSION" unsupported="${major}.$((minor - 1)).99" diff --git a/charts/kafka-operator/README.md b/charts/kafka-operator/README.md index f89e5f772..c93d50e7a 100644 --- a/charts/kafka-operator/README.md +++ b/charts/kafka-operator/README.md @@ -11,107 +11,80 @@ CI covers Kubernetes 1.35.0 and 1.37.0. The chart's `kubeVersion` constraint als ## Alert routing, readiness, and election constraints -The chart enables leader election. Its alert Service selects only the Pod -whose elected listener publishes `kafka.banzaicloud.io/alert-leader: "true"`. -Do not set this label statically on Pod templates. Webhook and metrics Services -retain their own selectors; webhook-serving standby replicas remain ready. -An elected replica is ready only after its alert listener and routing -publication are usable. Enabled webhooks also require local serving readiness. -Liveness does not depend on Kafka or leader status. -Upgrades from versions without the routing label have no alert endpoint until a -label-aware replica becomes leader; plan alert retries during that handoff. - -The chart supplies `POD_NAMESPACE`, `POD_NAME`, and `POD_UID` through the downward -API. The operator reads its own Pod and routing peers directly from the API, -not the namespace-filtered Kafka cache. Managed routing requires: - -- The exact namespace/name/UID of the operator Pod, not a watched Kafka namespace. -- The chart's installation selector labels: `control-plane=controller-manager`, - `controller-tools.k8s.io=1.0`, `app.kubernetes.io/component=operator`, and - matching `app.kubernetes.io/name` and `app.kubernetes.io/instance`. -- Pod get/list/update permissions in the operator namespace, even when watched - Kafka resources are restricted to other namespaces. Existing chart RBAC - provides these permissions; custom/restricted RBAC must provide them. - -Before publication, the elected process removes stale active labels only from -that installation and namespace. Identity or label-write failures fail closed. -Observed manager cancellation closes admission, drains in-flight HTTP requests -for up to five seconds, and attempts bounded removal of its own label. Cleanup -failure cannot keep its listener open. Endpoint propagation is asynchronous; -stale connections may fail or receive HTTP 503 during failover. - -For existing local/non-Helm use, absence of **both** `POD_NAME` and `POD_UID` -preserves the legacy listener. `POD_NAMESPACE` alone does not enable managed -routing. Partial new identity, or complete new identity without a namespace, -is rejected. Non-Helm installations opting in must supply the complete identity, -installation labels and matching alert Service selector above. The existing -`alertManager.enable` value controls Service exposure, not listener startup. - -Leader-election identity is unchanged. Replicas sharing a mutation domain must -enable election, use byte-identical raw namespace input, and share the election -namespace. A nonempty `--namespaces` argument wins over `WATCH_NAMESPACE`. -Order, whitespace and duplicates in that raw string can change the lock ID -even when the cache watches equivalent sets. Do not normalize these inputs -during a rolling upgrade. The election namespace defaults to the in-cluster -service-account namespace, not the watched Kafka namespace set; `POD_NAMESPACE` -does not override it. - -Partially overlapping watch sets, differently spelled equivalent sets, -all-namespaces overlaps, and independent election namespaces are not supported -concurrent mutation arrangements. Use replicas of one installation behind its -alert Service, not separate releases with different Service selectors that -happen to share a Lease. Distinct election domains do not fence each other. - -Neither Lease election nor routing labels provide external-action fencing or -exactly-once execution. A process pause/partition before observed cancellation, -already admitted work, or an ambiguous submit-then-crash window can outlive -routing changes. HTTP draining and conflict retries do not resolve those -execution semantics or automatically adopt external operations. +The chart enables leader election. Only its elected listener publishes +`kafka.banzaicloud.io/alert-leader: "true"`, which the alert Service requires. +Never set this label on Pod templates. Webhook-serving standbys remain ready; +elected replicas require usable alert serving/routing, and enabled webhooks +require local serving readiness. Liveness is independent of Kafka and election. +Upgrades from label-unaware versions have an alert endpoint gap until a new +replica becomes leader; plan alert retries during that handoff. + +The chart supplies the operator Pod's `POD_NAMESPACE`, `POD_NAME` and `POD_UID` +through the downward API. Routing uses uncached API reads and UID-checked +updates within that namespace and the exact installation selector: +`control-plane=controller-manager`, `controller-tools.k8s.io=1.0`, +`app.kubernetes.io/component=operator`, and matching +`app.kubernetes.io/name`/`app.kubernetes.io/instance`. Existing chart RBAC grants +Pod get/list/update; custom RBAC must include the operator namespace even when +Kafka is watched elsewhere. Identity/read/write failures fail startup closed. + +For local/non-Helm use, absence of **both** `POD_NAME` and `POD_UID` preserves the +legacy listener, including with `POD_NAMESPACE` alone. Partial identity is +rejected. Opting in requires all three inputs, installation labels and matching +Service selector above. `alertManager.enable` controls exposure, not startup. +Observed cancellation closes admission, drains HTTP requests for up to five +seconds and bounds self-label cleanup to two seconds; cleanup failure cannot +keep the listener open. Endpoint updates are asynchronous; stale requests may +fail or receive HTTP 503. + +Election identity is unchanged. Replicas must enable election, use byte-identical +raw namespace input and share the election namespace. Nonempty `--namespaces` +wins over `WATCH_NAMESPACE`; order, whitespace and duplicates affect the lock +ID even for equivalent watch sets. Do not normalize inputs during rolling +upgrades. Election namespace defaults to the in-cluster service-account +namespace, not watched namespaces or `POD_NAMESPACE`. +Overlapping watch sets (including all-namespaces overlap), differently spelled +equivalent sets and independent election namespaces are unsupported concurrency +topologies. Use replicas of one installation, not separate releases with +different Service selectors sharing a Lease. + +Election and routing labels are not external-action fencing or exactly-once +execution. Pauses/partitions before observed cancellation, admitted work and +ambiguous submissions can outlive routing changes; HTTP draining and conflict +retries do not automatically adopt external operations. ## Protected metrics prerequisites -When metrics and the auth proxy are enabled, the manager's raw metrics listener -binds to loopback. The proxy exposes HTTPS on port 8443, and the managed -PodMonitor targets its `https` port rather than bypassing it. Port/protocol -alignment alone does not establish a working authenticated scrape. - -All containers in the operator Pod share `operator.serviceAccount.name` -(`default` if the name is empty). -`prometheusMetrics.authProxy.serviceAccount.name` does not give the sidecar a -separate identity. With `rbac.enabled=true`, the proxy review-role binding -includes the actual Pod account and preserves the configured proxy account, -without duplicating a shared account. This also applies when account creation -is disabled. With `rbac.enabled=false`, the installer must grant the actual Pod -account `create` on `authentication.k8s.io/tokenreviews` and +With metrics and the auth proxy enabled, raw metrics bind to loopback and the +managed PodMonitor targets the proxy's `https` port (8443). This wiring alone +does not establish authenticated scraping. + +All containers share `operator.serviceAccount.name` (`default` if empty), not a +separate sidecar identity. The review-role binding includes that actual Pod +account and the configured `prometheusMetrics.authProxy.serviceAccount.name`, +de-duplicated when equal, including externally created accounts. With +`rbac.enabled=false`, the installer must grant the actual Pod account `create` +on `authentication.k8s.io/tokenreviews` and `authorization.k8s.io/subjectaccessreviews`. -A working protected scrape separately requires a scraping identity authorized -to `get` the non-resource URL `/metrics`, usable proxy HTTPS serving credentials, -and scraper trust in that certificate's CA and expected DNS/server name. Review -permissions for the proxy do not authorize the scraper. The webhook certificate -is not automatically wired into the metrics proxy. - -Managed PodMonitor authentication/CA/server-name integration is **deferred**: -the chart has no managed monitor fields for bearer-token/client-certificate -credentials, trusted CA, or TLS server name, and does not configure proxy -certificate arguments or mounts. Supply caller-owned external workload and -scrape configuration for these requirements; enabling chart values or adding -`additionalVolumes` alone does not solve them. `additionalSidecars` does not -configure the existing proxy. Do not disable TLS verification to work around -missing trust configuration. Validate authenticated scraping and rejection of -unauthorized requests against the actual endpoint before relying on it. +Managed scrape auth/TLS integration is **deferred**: there are no PodMonitor +credential/CA/server-name fields or proxy certificate arguments/mounts. +Caller-owned external workload/scrape configuration must supply proxy HTTPS +credentials and a scraper identity allowed to `get` non-resource URL `/metrics`, +with trusted CA and expected server name. Proxy review permissions do not +authorize the scraper. Webhook certificates are not wired into the proxy; +`additionalVolumes` alone and `additionalSidecars` do not configure it. Do not +bypass TLS verification; validate authenticated scraping and unauthorized +request rejection against the actual endpoint. ### Existing webhook certificate renewal limits -Helm-generated webhook serving certificates last 365 days (the generated CA -lasts 3650 days). The chart has no continuous renewal controller. A Helm upgrade -with generation enabled regenerates CA and serving material; it is not a -guaranteed zero-downtime rotation. Plan renewal before expiry and coordinate the -Secret, webhook CA bundle, and serving Pod rollout. With -`webhook.certs.generate=false`, the caller must refresh the supplied CA, -certificate, and key through the existing certificate values and deployment -process. No metrics certificate reuse, issuance, or automatic renewal is added -by the proxy wiring/RBAC fixes. +Generated webhook serving certificates last 365 days (CA: 3650 days), without +continuous renewal. A generation-enabled Helm upgrade replaces CA and serving +material, not necessarily without downtime. Renew before expiry and coordinate +the Secret, webhook CA bundle and Pod rollout. With `webhook.certs.generate=false`, +refresh supplied CA/certificate/key through existing values and deployment. +No metrics certificate reuse, issuance or automatic renewal is added. ## Installing the chart diff --git a/controllers/controller_common_test.go b/controllers/controller_common_test.go index 6024d0b58..e20ccb2d2 100644 --- a/controllers/controller_common_test.go +++ b/controllers/controller_common_test.go @@ -164,7 +164,7 @@ func newAlertTestController(pods ...client.Object) (AController, *alertTestClien }, writer } -func runAlertForTest(t *testing.T, c AController, handler http.Handler) (context.CancelFunc, <-chan error, string) { +func runAlertForTest(t *testing.T, c AController, handler http.Handler) (context.CancelFunc, <-chan error, net.Listener) { t.Helper() listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) @@ -183,7 +183,7 @@ func runAlertForTest(t *testing.T, c AController, handler http.Handler) (context t.Error("alert controller did not stop within its cleanup bound") } }) - return cancel, done, "http://" + listener.Addr().String() + return cancel, done, listener } func TestAlertRoutingUsesScopedUncachedIdentity(t *testing.T) { @@ -191,11 +191,7 @@ func TestAlertRoutingUsesScopedUncachedIdentity(t *testing.T) { foreign := alertTestPod("foreign", "release") foreign.Namespace = "other-namespace" c, writer := newAlertTestController(self, stale, other, foreign) - var handled atomic.Int32 - cancel, done, address := runAlertForTest(t, c, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - handled.Add(1) - _, _ = w.Write([]byte("ok")) - })) + cancel, done, _ := runAlertForTest(t, c, http.NotFoundHandler()) require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) for _, expected := range []struct { pod *corev1.Pod @@ -205,23 +201,8 @@ func TestAlertRoutingUsesScopedUncachedIdentity(t *testing.T) { require.NoError(t, writer.Client.Get(context.Background(), client.ObjectKeyFromObject(expected.pod), current)) require.Equal(t, expected.active, current.Labels["kafka.banzaicloud.io/alert-leader"] == "true", expected.pod.Name) } - transport := &http.Transport{} - defer transport.CloseIdleConnections() - httpClient := &http.Client{Transport: transport, Timeout: time.Second} - response, err := httpClient.Get(address) - require.NoError(t, err) - _, err = io.ReadAll(response.Body) - require.NoError(t, err) - require.NoError(t, response.Body.Close()) cancel() - require.Error(t, c.readyz(nil), "observed cancellation must immediately fail elected readiness") - response, err = httpClient.Get(address) - if err == nil { - require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) - require.NoError(t, response.Body.Close()) - } require.NoError(t, <-done) - require.EqualValues(t, 1, handled.Load(), "stale keepalive traffic reached the receiver") current := &corev1.Pod{} require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) require.Empty(t, current.Labels["kafka.banzaicloud.io/alert-leader"]) @@ -322,29 +303,25 @@ func TestAlertReadinessPreservesStandbysAndLegacy(t *testing.T) { require.NoError(t, <-done) } -func TestAlertRoutingGateAndBoundedCleanup(t *testing.T) { +func TestAlertRoutingBoundedCleanup(t *testing.T) { c, writer := newAlertTestController(alertTestPod("self", "release")) cleanupStarted := make(chan struct{}) - var cleanupOnce sync.Once + cleanupOnce := sync.OnceFunc(func() { close(cleanupStarted) }) writer.update = func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { if obj.GetLabels()["kafka.banzaicloud.io/alert-leader"] != "true" { // Only block final cleanup, not startup's stale-label removal. - select { - case <-cleanupStarted: - default: - c.state.RLock() - stopping := c.state.ctx != nil && c.state.ctx.Err() != nil - c.state.RUnlock() - if stopping { - cleanupOnce.Do(func() { close(cleanupStarted) }) - <-ctx.Done() - return ctx.Err() - } + c.state.RLock() + stopping := c.state.ctx != nil && c.state.ctx.Err() != nil + c.state.RUnlock() + if stopping { + cleanupOnce() + <-ctx.Done() + return ctx.Err() } } return writer.Client.Update(ctx, obj, opts...) } - cancel, done, address := runAlertForTest(t, c, http.NotFoundHandler()) + cancel, done, listener := runAlertForTest(t, c, http.NotFoundHandler()) require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) cancel() select { @@ -352,19 +329,11 @@ func TestAlertRoutingGateAndBoundedCleanup(t *testing.T) { case <-time.After(time.Second): t.Fatal("cleanup did not start") } - connection, err := net.DialTimeout("tcp", address[len("http://"):], time.Second) + connection, err := net.DialTimeout("tcp", listener.Addr().String(), time.Second) if connection != nil { _ = connection.Close() } require.Error(t, err, "listener remained open while cleanup was blocked") - c.state.RLock() - ctx := c.state.ctx - c.state.RUnlock() - recorder := httptest.NewRecorder() - c.gatedHandler(ctx, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { - t.Error("canceled admission reached the receiver") - })).ServeHTTP(recorder, httptest.NewRequest(http.MethodPost, "/", nil)) - require.Equal(t, http.StatusServiceUnavailable, recorder.Code) select { case err := <-done: require.ErrorIs(t, err, context.DeadlineExceeded) @@ -423,13 +392,7 @@ func TestAlertRoutingCleanupPreservesReplacementUID(t *testing.T) { func TestAlertRoutingServeFailureRemovesLabel(t *testing.T) { c, writer := newAlertTestController(alertTestPod("self", "release")) - listener, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - defer listener.Close() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - done := make(chan error, 1) - go func() { done <- c.run(ctx, listener, http.NotFoundHandler()) }() + _, done, listener := runAlertForTest(t, c, http.NotFoundHandler()) require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) require.NoError(t, listener.Close()) select { @@ -451,15 +414,6 @@ func TestAlertManagerReportsBindError(t *testing.T) { } defer listener.Close() - defer func() { - if recovered := recover(); recovered != nil { - t.Errorf("Start panicked instead of returning the bind error: %v", recovered) - } - }() - - if err := (AController{}).Start(context.Background()); err == nil { - t.Error("Start returned nil for an occupied alert listener") - } c, writer := newAlertTestController(alertTestPod("self", "release")) writer.update = func(context.Context, client.Object, ...client.UpdateOption) error { t.Error("bind failure attempted to change a routing label") @@ -483,9 +437,7 @@ func TestAlertServerLifecycle(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer listener.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -523,9 +475,7 @@ func TestAlertServerLifecycle(t *testing.T) { t.Fatal("alert handler did not start") } if tc.serveError { - if err := listener.Close(); err != nil { - t.Fatal(err) - } + require.NoError(t, listener.Close()) } else { cancel() } @@ -539,9 +489,7 @@ func TestAlertServerLifecycle(t *testing.T) { } select { case err := <-serverDone: - if !errors.Is(err, tc.expectedErr) { - t.Fatalf("server error = %v, want %v", err, tc.expectedErr) - } + require.ErrorIs(t, err, tc.expectedErr) case <-time.After(2 * time.Second): t.Fatal("alert server did not stop") } @@ -552,16 +500,12 @@ func TestAlertServerLifecycle(t *testing.T) { } select { case err := <-requestDone: - if (err == nil) != tc.finish { - t.Fatalf("request error = %v, want successful drain = %t", err, tc.finish) - } + require.Equal(t, tc.finish, err == nil, "request error: %v", err) case <-time.After(2 * time.Second): t.Fatal("alert request did not finish") } rebound, err := net.Listen("tcp", listener.Addr().String()) - if err != nil { - t.Fatalf("alert listener was not released: %v", err) - } + require.NoError(t, err, "alert listener was not released") _ = rebound.Close() }) } From afad275964dbbc1ee46527be5a58e656917fab0e Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Wed, 18 Nov 2026 09:01:50 +0100 Subject: [PATCH 9/9] fix(chart): restore baseline HTTP metrics scraping (#392) Defer incomplete protected scraping, retain custom-port alignment, and add focused Helm render regressions without changing alert serving. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yml | 21 ++++++++++ charts/kafka-operator/README.md | 38 +++++++------------ .../templates/authproxy-rbac.yaml | 5 --- .../operator-deployment-with-webhook.yaml | 5 +-- .../kafka-operator/templates/podmonitor.yaml | 6 --- 5 files changed, 36 insertions(+), 39 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03c6d451f..6468a15c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,27 @@ jobs: helm lint charts/kafka-operator --strict --kube-version "$version" helm template kafka-operator charts/kafka-operator --kube-version "$version" > /dev/null done + rendered=$(helm template kafka-operator charts/kafka-operator) + if grep -q '^kind: PodMonitor$' <<< "$rendered"; then exit 1; fi + grep -F 'kafka.banzaicloud.io/alert-leader: "true"' <<< "$rendered" + grep -E 'fieldPath: metadata\.name$' <<< "$rendered" + grep -E 'fieldPath: metadata\.uid$' <<< "$rendered" + for proxy in true false; do + for port in 8080 9090; do + args=() + if [ "$port" != 8080 ]; then args+=(--set metricEndpoint.port="$port"); fi + rendered=$(helm template kafka-operator charts/kafka-operator \ + --set prometheusMetrics.podMonitor.enabled=true \ + --set prometheusMetrics.authProxy.enabled="$proxy" "${args[@]}") + grep -F ' port: metrics' <<< "$rendered" + if grep -q ' scheme: https' <<< "$rendered"; then exit 1; fi + grep -F -- "--metrics-addr=:$port" <<< "$rendered" + grep -F "containerPort: $port" <<< "$rendered" + if [ "$proxy" = true ]; then + grep -F -- "--upstream=http://127.0.0.1:$port/" <<< "$rendered" + fi + done + done IFS=. read -r major minor _ <<< "$KUBERNETES_MIN_VERSION" unsupported="${major}.$((minor - 1)).99" if output=$(helm template kafka-operator charts/kafka-operator --kube-version "$unsupported" 2>&1); then diff --git a/charts/kafka-operator/README.md b/charts/kafka-operator/README.md index c93d50e7a..a7ec04551 100644 --- a/charts/kafka-operator/README.md +++ b/charts/kafka-operator/README.md @@ -53,31 +53,19 @@ execution. Pauses/partitions before observed cancellation, admitted work and ambiguous submissions can outlive routing changes; HTTP draining and conflict retries do not automatically adopt external operations. -## Protected metrics prerequisites - -With metrics and the auth proxy enabled, raw metrics bind to loopback and the -managed PodMonitor targets the proxy's `https` port (8443). This wiring alone -does not establish authenticated scraping. - -All containers share `operator.serviceAccount.name` (`default` if empty), not a -separate sidecar identity. The review-role binding includes that actual Pod -account and the configured `prometheusMetrics.authProxy.serviceAccount.name`, -de-duplicated when equal, including externally created accounts. With -`rbac.enabled=false`, the installer must grant the actual Pod account `create` -on `authentication.k8s.io/tokenreviews` and -`authorization.k8s.io/subjectaccessreviews`. - -Managed scrape auth/TLS integration is **deferred**: there are no PodMonitor -credential/CA/server-name fields or proxy certificate arguments/mounts. -Caller-owned external workload/scrape configuration must supply proxy HTTPS -credentials and a scraper identity allowed to `get` non-resource URL `/metrics`, -with trusted CA and expected server name. Proxy review permissions do not -authorize the scraper. Webhook certificates are not wired into the proxy; -`additionalVolumes` alone and `additionalSidecars` do not configure it. Do not -bypass TLS verification; validate authenticated scraping and unauthorized -request rejection against the actual endpoint. - -### Existing webhook certificate renewal limits +## Metrics compatibility + +Raw metrics remain bound to all interfaces over unauthenticated HTTP. The +managed PodMonitor, when enabled, scrapes the `metrics` port directly, including +when the existing auth proxy is enabled. The proxy does not protect this raw +endpoint. Listener, container and proxy-upstream ports follow `metricEndpoint.port` +(default 8080). + +Managed TLS/authenticated scraping and its certificate/credential lifecycle are +deferred. This preserves baseline HTTP scraping; it does not secure metrics. +Do not treat the proxy's presence as authentication for the raw endpoint. + +## Existing webhook certificate renewal limits Generated webhook serving certificates last 365 days (CA: 3650 days), without continuous renewal. A generation-enabled Helm upgrade replaces CA and serving diff --git a/charts/kafka-operator/templates/authproxy-rbac.yaml b/charts/kafka-operator/templates/authproxy-rbac.yaml index 079da3e4c..4a9dbc414 100644 --- a/charts/kafka-operator/templates/authproxy-rbac.yaml +++ b/charts/kafka-operator/templates/authproxy-rbac.yaml @@ -58,9 +58,4 @@ subjects: - kind: ServiceAccount name: {{ include "operator.metricsAuthProxy.serviceAccountName" .}} namespace: {{ .Release.Namespace }} -{{- if ne (include "operator.serviceAccountName" .) (include "operator.metricsAuthProxy.serviceAccountName" .) }} -- kind: ServiceAccount - name: {{ include "operator.serviceAccountName" .}} - namespace: {{ .Release.Namespace }} -{{- end }} {{- end }} diff --git a/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml b/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml index e44fea975..8f67d3bc7 100644 --- a/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml +++ b/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml @@ -1,5 +1,4 @@ {{- $metricsPort := (.Values.metricEndpoint).port | default 8080 }} -{{- $metricsProxyEnabled := and .Values.prometheusMetrics.enabled .Values.prometheusMetrics.authProxy.enabled }} {{- $tlsCrt := "" }} {{- $tlsKey := "" }} {{- $caCrt := "" }} @@ -153,7 +152,7 @@ spec: {{ toYaml .Values.podSecurityContext | nindent 8 }} {{- end }} containers: - {{- if $metricsProxyEnabled }} + {{- if and .Values.prometheusMetrics.enabled .Values.prometheusMetrics.authProxy.enabled }} - name: kube-rbac-proxy image: "{{ .Values.prometheusMetrics.authProxy.image.repository }}:{{ .Values.prometheusMetrics.authProxy.image.tag }}" imagePullPolicy: {{ .Values.prometheusMetrics.authProxy.image.pullPolicy }} @@ -194,7 +193,7 @@ spec: {{- if .Values.operator.developmentLogging }} - --development {{- end }} - - --metrics-addr={{ if $metricsProxyEnabled }}127.0.0.1{{ end }}:{{ $metricsPort }} + - --metrics-addr=:{{ $metricsPort }} {{- if .Values.healthProbes.port }} - --health-probes-addr=:{{ .Values.healthProbes.port }} {{- end }} diff --git a/charts/kafka-operator/templates/podmonitor.yaml b/charts/kafka-operator/templates/podmonitor.yaml index 7d8f19fe2..da83c31ae 100644 --- a/charts/kafka-operator/templates/podmonitor.yaml +++ b/charts/kafka-operator/templates/podmonitor.yaml @@ -26,12 +26,6 @@ spec: app.kubernetes.io/component: operator podMetricsEndpoints: - interval: {{ .Values.prometheusMetrics.podMonitor.interval }} - {{- if and .Values.prometheusMetrics.enabled .Values.prometheusMetrics.authProxy.enabled }} - port: https - scheme: https - {{- else }} port: metrics - scheme: http - {{- end }} path: /metrics {{- end }}