From da63dc3e3e39dc7a24b08810a3bf3ebf731abffc Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Mon, 16 Nov 2026 16:59:17 +0100 Subject: [PATCH 1/5] fix(controllers): persist Cruise Control results through status conflicts Retry fresh API reads and status writes without repeating external submission. Preserve stop-operation identity and retry history, reject stale object/task identities, add seven regression cases, and document the remaining submit-before-persist crash window. Refs #369. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cruisecontroloperation_controller.go | 37 ++-- .../cruisecontroloperation_controller_test.go | 169 ++++++++++++++++++ docs/monitoring.md | 19 +- 3 files changed, 212 insertions(+), 13 deletions(-) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index d7da19d39..68a343bb2 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -243,24 +243,37 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques } } - conflictRetryFunction := func() error { - if err = updateResult(log, cruseControlTaskResult, ccOperationExecution, true); err != nil { - return err - } - err = r.Status().Update(ctx, ccOperationExecution) - if apiErrors.IsConflict(err) { - err = r.Get(ctx, client.ObjectKey{Name: ccOperationExecution.GetName(), Namespace: ccOperationExecution.GetNamespace()}, ccOperationExecution) - } - return err - } - // Protection to avoid re-execution task when status update is failed - if err := util.RetryOnConflict(util.DefaultBackOffForConflict, conflictRetryFunction); err != nil { + // Retry persistence without resubmitting the external task. + if err := r.persistExecutionResult(ctx, log, ccOperationExecution, cruseControlTaskResult); err != nil { return requeueWithError(log, "could not update the result of the Cruise Control user task execution to the CruiseControlOperation status", err) } return reconciled() } +func (r *CruiseControlOperationReconciler) persistExecutionResult( + ctx context.Context, + log logr.Logger, + operation *banzaiv1alpha1.CruiseControlOperation, + result *scale.Result, +) error { + return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { + latest := &banzaiv1alpha1.CruiseControlOperation{} + if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + return err + } + if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { + return errors.New("CruiseControlOperation identity or current task changed after external task submission") + } + // Preserve the stop operation selected in memory during finalization. + latest.Status.CurrentTask.Operation = operation.CurrentTaskOperation() + if err := updateResult(log, result, latest, true); err != nil { + return err + } + return r.Status().Update(ctx, latest) + }) +} + func (r *CruiseControlOperationReconciler) addFinalizer(ctx context.Context, currentCCOperation *banzaiv1alpha1.CruiseControlOperation) error { // examine DeletionTimestamp to determine if object is under deletion if currentCCOperation.DeletionTimestamp.IsZero() { diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index d9a68fd87..0c25b5c50 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -17,16 +17,22 @@ package controllers import ( "context" + "errors" "testing" "time" "github.com/go-logr/logr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" @@ -34,6 +40,169 @@ import ( "github.com/banzaicloud/koperator/pkg/scale" ) +func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { + for _, tt := range []struct { + name string + retry bool + deleting bool + replaced bool + taskRemoved bool + taskChanged bool + exhausted bool + conflicts int + wantRetries int + wantFailed int + wantOp v1alpha1.CruiseControlTaskOperation + }{ + {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, + {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, + {name: "replacement object rejects stale result", replaced: true, conflicts: 1}, + {name: "removed task rejects stale result", taskRemoved: true, conflicts: 1}, + {name: "newer task rejects stale result", taskChanged: true, conflicts: 1}, + {name: "exhausted conflicts return error", exhausted: true, conflicts: 1000}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := v1beta1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "cluster", Namespace: "ns"}} + operation := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: "operation", Namespace: "ns", UID: "operation-uid", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: "cluster"}, + }, + Spec: v1alpha1.CruiseControlOperationSpec{ErrorPolicy: v1alpha1.ErrorPolicyRetry}, + } + operation.Status.ErrorPolicy = v1alpha1.ErrorPolicyRetry + operation.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationAddBroker} + taskResult := &scale.Result{ + TaskID: "submitted-task", State: v1beta1.CruiseControlTaskInExecution, + StartedAt: "Mon, 02 Jan 2006 15:04:05 GMT", ResponseStatusCode: 202, + RequestURL: "http://cruise-control/add_broker", + } + var previous []*scale.Result + if tt.retry { + operation.Status.CurrentTask.ID = "failed-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskCompletedWithError + operation.Status.CurrentTask.Finished = &v1.Time{Time: time.Date(2006, 1, 2, 15, 5, 0, 0, time.UTC)} + operation.Status.RetryCount = 2 + operation.Status.FailedTasks = []v1alpha1.CruiseControlTask{{ID: "earlier-failed-task"}} + previous = []*scale.Result{{TaskID: "failed-task", State: v1beta1.CruiseControlTaskCompletedWithError}} + } + if tt.deleting { + operation.DeletionTimestamp = &v1.Time{Time: time.Now()} + operation.Finalizers = []string{ccOperationFinalizerGroup} + operation.Status.CurrentTask.ID = "running-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskInExecution + previous = []*scale.Result{{TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}} + } + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(t)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: true, + }}, nil) + var taskIDs []any + if len(previous) != 0 { + taskIDs = []any{previous[0].TaskID} + } + scaler.EXPECT().UserTasks(gomock.Any(), taskIDs...).Return(previous, nil) + if tt.deleting { + scaler.EXPECT().StopExecution(gomock.Any()).Return(taskResult, nil).Times(1) + } else { + scaler.EXPECT().AddBrokersWithParams(gomock.Any(), operation.CurrentTaskParameters()).Return(taskResult, nil).Times(1) + } + var resultWrites int + kubeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(cluster, operation). + WithStatusSubresource(operation). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if op, ok := obj.(*v1alpha1.CruiseControlOperation); ok && subresource == "status" && op.CurrentTaskID() == "submitted-task" { + resultWrites++ + if resultWrites <= tt.conflicts { + latest := &v1alpha1.CruiseControlOperation{} + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), latest); err != nil { + return err + } + latest.Annotations = map[string]string{"other-writer": "preserved"} + if err := c.Update(ctx, latest); err != nil { + return err + } + if tt.replaced { + latest.Finalizers = nil + if err := c.Update(ctx, latest); err != nil { + return err + } + if err := c.Delete(ctx, latest); err != nil { + return err + } + latest.ResourceVersion = "" + latest.UID = "replacement-uid" + if err := c.Create(ctx, latest); err != nil { + return err + } + } + if tt.taskRemoved || tt.taskChanged { + if tt.taskRemoved { + latest.Status.CurrentTask = nil + } else { + latest.Status.CurrentTask.ID = "newer-task" + } + if err := c.Status().Update(ctx, latest); err != nil { + return err + } + } + return apierrors.NewConflict(schema.GroupResource{Group: "kafka.banzaicloud.io", Resource: "cruisecontroloperations"}, obj.GetName(), errors.New("injected concurrent update")) + } + } + return c.SubResource(subresource).Update(ctx, obj, opts...) + }, + }).Build() + reconciler := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: kubeClient, Scheme: scheme, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { + return scaler, nil + }, + } + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(operation)}) + if err != nil && !tt.replaced && !tt.taskRemoved && !tt.taskChanged && !tt.exhausted { + t.Fatal(err) + } + persisted := &v1alpha1.CruiseControlOperation{} + if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(operation), persisted); err != nil { + t.Fatal(err) + } + if tt.replaced || tt.taskRemoved || tt.taskChanged || tt.exhausted { + if err == nil { + t.Error("reported success without persisting the result to the original task") + } + if persisted.CurrentTaskID() == "submitted-task" { + t.Error("attached stale external result to a replaced operation or removed task") + } + return + } + assert.Equal(t, "submitted-task", persisted.CurrentTaskID()) + assert.Equal(t, v1beta1.CruiseControlTaskInExecution, persisted.CurrentTaskState()) + assert.Equal(t, tt.wantOp, persisted.CurrentTaskOperation()) + task := persisted.CurrentTask() + require.NotNil(t, task) + assert.Equal(t, taskResult.RequestURL, task.HTTPRequest) + assert.Equal(t, &taskResult.ResponseStatusCode, task.HTTPResponseCode) + require.NotNil(t, task.Started) + assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) + assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) + assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + assert.Equal(t, tt.conflicts+1, resultWrites) + }) + } +} + func createCCRetryExecutionOperation(createTime time.Time, id string, operation v1alpha1.CruiseControlTaskOperation) *v1alpha1.CruiseControlOperation { return &v1alpha1.CruiseControlOperation{ ObjectMeta: v1.ObjectMeta{ diff --git a/docs/monitoring.md b/docs/monitoring.md index b6f802f1c..8d917bade 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -37,4 +37,21 @@ sum by (verb) (rate(rest_client_rate_limiter_duration_seconds_sum[5m])) Record cluster size, resource counts, workload, test duration, leader changes, and error rates with each measurement. These metrics establish instrumentation, not evidence that an optimization improves performance. This change does not -alter metrics authentication, scraper routing, worker counts, or cache scope. \ No newline at end of file +alter metrics authentication, scraper routing, worker counts, or cache scope. + +## Cruise Control execution-result persistence + +After an external task is submitted, the operator retries conflicting status +writes using fresh API-server reads. These persistence retries do not submit +the external task again. They retain the task identity, response metadata, +retry history, and selected stop-operation type. A replaced operation, removed +current task, or different task identity produces an error rather than accepting +a stale result. + +This is not exactly-once execution. If all writes fail, or the process stops +after Cruise Control accepts a request but before its task ID is recorded, the +external task can outlive the recorded operation state. Recorded task IDs allow +subsequent reconciliation to query task progress. Automatic adoption of an +unrecorded submission is not implemented; inspect Cruise Control's task history +when investigating this window, and do not infer that a missing status means +the external operation never ran. \ No newline at end of file From f022f61913658b17b705a737c545c368e36fc831 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 08:37:01 +0100 Subject: [PATCH 2/5] perf(controllers): refresh Cruise Control results only on conflict Reuse the selected operation for the first status write and read directly only when retrying a conflict. Cover conflict-free submission, retry, and stop paths. Refs #369. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cruisecontroloperation_controller.go | 6 +++++- .../cruisecontroloperation_controller_test.go | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index 68a343bb2..0bf53b9e8 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -257,11 +257,15 @@ func (r *CruiseControlOperationReconciler) persistExecutionResult( operation *banzaiv1alpha1.CruiseControlOperation, result *scale.Result, ) error { + firstAttempt := true return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { latest := &banzaiv1alpha1.CruiseControlOperation{} - if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + if firstAttempt { + latest = operation.DeepCopy() + } else if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { return err } + firstAttempt = false if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { return errors.New("CruiseControlOperation identity or current task changed after external task submission") } diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index 0c25b5c50..770ffa962 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -54,6 +54,9 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { wantFailed int wantOp v1alpha1.CruiseControlTaskOperation }{ + {name: "first submission without conflict", wantOp: v1alpha1.OperationAddBroker}, + {name: "retry without conflict preserves history once", retry: true, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission without conflict retains operation", deleting: true, wantOp: v1alpha1.OperationStopExecution}, {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, @@ -163,8 +166,16 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { return c.SubResource(subresource).Update(ctx, obj, opts...) }, }).Build() + var directReads int + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + directReads++ + assert.Positive(t, resultWrites, "direct GET must follow a conflicting result write") + return c.Get(ctx, key, obj, opts...) + }, + }) reconciler := &CruiseControlOperationReconciler{ - Client: kubeClient, DirectClient: kubeClient, Scheme: scheme, + Client: kubeClient, DirectClient: directClient, Scheme: scheme, ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { return scaler, nil }, @@ -197,8 +208,11 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) - assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + if tt.conflicts > 0 { + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + } assert.Equal(t, tt.conflicts+1, resultWrites) + assert.Equal(t, tt.conflicts, directReads) }) } } From a1f3488f738122064a5bd361827fa52f04f3435b Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 09:57:27 +0100 Subject: [PATCH 3/5] fix(webhooks): skip Kafka checks for irrelevant topic updates (#394) Retain Kubernetes validation and fail-closed Kafka checks for spec and managedBy changes, including adoption. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- pkg/webhooks/kafkatopic_validator.go | 25 ++- pkg/webhooks/kafkatopic_validator_test.go | 213 ++++++++++++++++++++++ 2 files changed, 235 insertions(+), 3 deletions(-) diff --git a/pkg/webhooks/kafkatopic_validator.go b/pkg/webhooks/kafkatopic_validator.go index d1e09e41a..02d6e6f78 100644 --- a/pkg/webhooks/kafkatopic_validator.go +++ b/pkg/webhooks/kafkatopic_validator.go @@ -55,8 +55,15 @@ func (s KafkaTopicValidator) ValidateCreate(ctx context.Context, kafkaTopic *ban return s.validate(ctx, kafkaTopic) } -func (s KafkaTopicValidator) ValidateUpdate(ctx context.Context, _, kafkaTopic *banzaicloudv1alpha1.KafkaTopic) (warnings admission.Warnings, err error) { - return s.validate(ctx, kafkaTopic) +func (s KafkaTopicValidator) ValidateUpdate(ctx context.Context, oldTopic, kafkaTopic *banzaicloudv1alpha1.KafkaTopic) (warnings admission.Warnings, err error) { + checkKafka := true + if oldTopic != nil { + oldManager, oldHasManager := oldTopic.Annotations[TopicManagedByAnnotationKey] + manager, hasManager := kafkaTopic.Annotations[TopicManagedByAnnotationKey] + checkKafka = !cmp.Equal(oldTopic.Spec, kafkaTopic.Spec) || + oldHasManager != hasManager || oldManager != manager + } + return s.validateWithKafkaCheck(ctx, kafkaTopic, checkKafka) } func (s KafkaTopicValidator) ValidateDelete(_ context.Context, _ *banzaicloudv1alpha1.KafkaTopic) (warnings admission.Warnings, err error) { @@ -64,9 +71,13 @@ func (s KafkaTopicValidator) ValidateDelete(_ context.Context, _ *banzaicloudv1a } func (s *KafkaTopicValidator) validate(ctx context.Context, kafkaTopic *banzaicloudv1alpha1.KafkaTopic) (warnings admission.Warnings, err error) { + return s.validateWithKafkaCheck(ctx, kafkaTopic, true) +} + +func (s *KafkaTopicValidator) validateWithKafkaCheck(ctx context.Context, kafkaTopic *banzaicloudv1alpha1.KafkaTopic, checkKafka bool) (warnings admission.Warnings, err error) { log := s.Log.WithValues("name", kafkaTopic.GetName(), "namespace", kafkaTopic.GetNamespace()) - fieldErrs, err := s.validateKafkaTopic(ctx, log, kafkaTopic) + fieldErrs, err := s.validateKafkaTopicWithKafkaCheck(ctx, log, kafkaTopic, checkKafka) if err != nil { log.Error(err, errorDuringValidationMsg) return nil, apierrors.NewInternalError(errors.WithMessage(err, errorDuringValidationMsg)) @@ -81,6 +92,10 @@ func (s *KafkaTopicValidator) validate(ctx context.Context, kafkaTopic *banzaicl } func (s *KafkaTopicValidator) validateKafkaTopic(ctx context.Context, log logr.Logger, topic *banzaicloudv1alpha1.KafkaTopic) (field.ErrorList, error) { + return s.validateKafkaTopicWithKafkaCheck(ctx, log, topic, true) +} + +func (s *KafkaTopicValidator) validateKafkaTopicWithKafkaCheck(ctx context.Context, log logr.Logger, topic *banzaicloudv1alpha1.KafkaTopic, checkKafka bool) (field.ErrorList, error) { var allErrs field.ErrorList var logMsg string // First check if the kafkatopic is valid @@ -136,6 +151,10 @@ func (s *KafkaTopicValidator) validateKafkaTopic(ctx context.Context, log logr.L allErrs = append(allErrs, fieldErr) } + if !checkKafka { + return allErrs, nil + } + fieldErrList, err := s.checkKafka(ctx, topic, cluster) if err != nil { return nil, err diff --git a/pkg/webhooks/kafkatopic_validator_test.go b/pkg/webhooks/kafkatopic_validator_test.go index 328cbd4c4..8d43cda14 100644 --- a/pkg/webhooks/kafkatopic_validator_test.go +++ b/pkg/webhooks/kafkatopic_validator_test.go @@ -17,9 +17,13 @@ package webhooks import ( "context" + "errors" + "fmt" "strings" "testing" + "github.com/IBM/sarama" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -91,6 +95,215 @@ func newMockClients(cluster *v1beta1.KafkaCluster) (runtimeClient.WithWatch, kaf return client, kafkaClient, returnMockedKafkaClient } +type admissionCountingKafkaClient struct { + kafkaclient.KafkaClient + topicReads int +} + +func (c *admissionCountingKafkaClient) GetTopic(name string) (*sarama.TopicDetail, error) { + c.topicReads++ + return c.KafkaClient.GetTopic(name) +} + +func TestKafkaTopicValidateUpdateDependencies(t *testing.T) { + testCases := []struct { + name string + change func(*v1alpha1.KafkaTopic) + validationRequired bool + withoutManager bool + }{ + {name: "finalizer addition", change: func(topic *v1alpha1.KafkaTopic) { + topic.Finalizers = append(topic.Finalizers, "example.com/cleanup") + }}, + {name: "finalizer removal", change: func(topic *v1alpha1.KafkaTopic) { + topic.Finalizers = nil + }}, + {name: "irrelevant label", change: func(topic *v1alpha1.KafkaTopic) { + topic.Labels = map[string]string{"example.com/team": "streaming"} + }}, + {name: "irrelevant annotation", change: func(topic *v1alpha1.KafkaTopic) { + topic.Annotations["example.com/note"] = "updated" + }}, + {name: "partition change", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Spec.Partitions++ + }}, + {name: "config change", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Spec.Config = map[string]string{"retention.ms": "1000"} + }}, + {name: "cluster namespace default", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Spec.ClusterRef.Namespace = "" + }}, + {name: "managedBy change", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Annotations[TopicManagedByAnnotationKey] = "external" + }}, + {name: "managedBy removal", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + delete(topic.Annotations, TopicManagedByAnnotationKey) + }}, + {name: "managedBy addition", validationRequired: true, withoutManager: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Annotations[TopicManagedByAnnotationKey] = TopicManagedByKoperatorAnnotationValue + }}, + {name: "empty managedBy addition", validationRequired: true, withoutManager: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Annotations[TopicManagedByAnnotationKey] = "" + }}, + } + for _, unavailable := range []bool{false, true} { + mode := "available" + if unavailable { + mode = "unavailable" + } + t.Run(mode, func(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + cluster := newMockCluster() + client, broker, _ := newMockClients(cluster) + if err := client.Create(ctx, cluster); err != nil { + t.Fatal(err) + } + oldTopic := newMockTopic() + oldTopic.Spec.Partitions = 2 + oldTopic.Spec.ReplicationFactor = 1 + oldTopic.Finalizers = []string{"finalizer.kafkatopics.kafka.banzaicloud.io"} + oldTopic.Annotations = map[string]string{TopicManagedByAnnotationKey: TopicManagedByKoperatorAnnotationValue} + if tc.withoutManager { + delete(oldTopic.Annotations, TopicManagedByAnnotationKey) + } + if err := client.Create(ctx, oldTopic); err != nil { + t.Fatal(err) + } + newTopic := oldTopic.DeepCopy() + tc.change(newTopic) + countingBroker := &admissionCountingKafkaClient{KafkaClient: broker} + connections, closes := 0, 0 + validator := KafkaTopicValidator{ + Client: client, + Log: logr.Discard(), + NewKafkaFromCluster: func(runtimeClient.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + connections++ + if unavailable { + return nil, nil, errors.New("Kafka unavailable") + } + return countingBroker, func() { closes++ }, nil + }, + } + _, err := validator.ValidateUpdate(ctx, oldTopic, newTopic) + wantError := unavailable && tc.validationRequired + if (err != nil) != wantError { + t.Errorf("admission error = %v, want error = %v", err, wantError) + } + wantConnections, wantReads := 0, 0 + if tc.validationRequired { + wantConnections = 1 + if !unavailable { + wantReads = 1 + } + } + if connections != wantConnections || countingBroker.topicReads != wantReads || closes != wantReads { + t.Errorf("Kafka connections/GetTopic/closes = %d/%d/%d, want %d/%d/%d", + connections, countingBroker.topicReads, closes, wantConnections, wantReads, wantReads) + } + }) + } + }) + } +} + +func TestKafkaTopicValidateUpdateRetainsKubernetesValidation(t *testing.T) { + for _, invalidInput := range []string{"partitions", "missing cluster", "duplicate topic"} { + t.Run(invalidInput, func(t *testing.T) { + ctx := context.Background() + cluster := newMockCluster() + client, _, _ := newMockClients(cluster) + oldTopic := newMockTopic() + oldTopic.Spec.Partitions = 2 + oldTopic.Spec.ReplicationFactor = 1 + if invalidInput == "partitions" { + oldTopic.Spec.Partitions = 0 + } + if invalidInput != "missing cluster" { + if err := client.Create(ctx, cluster); err != nil { + t.Fatal(err) + } + } + if invalidInput == "duplicate topic" { + duplicate := oldTopic.DeepCopy() + duplicate.Name = "duplicate" + if err := client.Create(ctx, duplicate); err != nil { + t.Fatal(err) + } + } + if err := client.Create(ctx, oldTopic); err != nil { + t.Fatal(err) + } + newTopic := oldTopic.DeepCopy() + newTopic.Labels = map[string]string{"example.com/team": "streaming"} + connections := 0 + validator := KafkaTopicValidator{ + Client: client, + Log: logr.Discard(), + NewKafkaFromCluster: func(runtimeClient.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + connections++ + return nil, nil, errors.New("Kafka unavailable") + }, + } + _, err := validator.ValidateUpdate(ctx, oldTopic, newTopic) + if !apierrors.IsInvalid(err) { + t.Errorf("admission error = %v, want Kubernetes validation error", err) + } + if connections != 0 { + t.Errorf("Kafka connections = %d, want 0", connections) + } + }) + } +} + +func TestKafkaTopicValidateCreateAdoptionDependencies(t *testing.T) { + for _, unavailable := range []bool{false, true} { + for _, adopt := range []bool{false, true} { + name := fmt.Sprintf("unavailable=%t/adopt=%t", unavailable, adopt) + t.Run(name, func(t *testing.T) { + ctx := context.Background() + cluster := newMockCluster() + client, broker, _ := newMockClients(cluster) + if err := client.Create(ctx, cluster); err != nil { + t.Fatal(err) + } + topic := newMockTopic() + topic.Spec.Partitions = 2 + topic.Spec.ReplicationFactor = 1 + if adopt { + topic.Annotations = map[string]string{TopicManagedByAnnotationKey: TopicManagedByKoperatorAnnotationValue} + } + if err := broker.CreateTopic(&kafkaclient.CreateTopicOptions{ + Name: topic.Spec.Name, Partitions: 2, ReplicationFactor: 1, + }); err != nil { + t.Fatal(err) + } + connections := 0 + validator := KafkaTopicValidator{ + Client: client, + Log: logr.Discard(), + NewKafkaFromCluster: func(runtimeClient.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + connections++ + if unavailable { + return nil, nil, errors.New("Kafka unavailable") + } + return broker, func() {}, nil + }, + } + _, err := validator.ValidateCreate(ctx, topic) + wantError := unavailable || !adopt + if (err != nil) != wantError { + t.Errorf("adoption error = %v, want error = %v", err, wantError) + } + if connections != 1 { + t.Errorf("Kafka connections = %d, want 1", connections) + } + }) + } + } +} + func TestCheckKafkaTopicExist(t *testing.T) { cluster := newMockCluster() client, kafkaClient, returnMockedKafkaClient := newMockClients(cluster) From 523e4b91e1bc924f88f4a950aae6e7df84777ce7 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 09:57:33 +0100 Subject: [PATCH 4/5] fix(controllers): recover CSR user Secrets on dependency events (#394) Watch KafkaUser-owned Secrets only when certificate signing is enabled, preserving existing ownership and optional cert-manager behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- controllers/kafkauser_controller.go | 2 ++ .../tests/kafkauser_controller_test.go | 31 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/controllers/kafkauser_controller.go b/controllers/kafkauser_controller.go index 54c67774d..cea11a710 100644 --- a/controllers/kafkauser_controller.go +++ b/controllers/kafkauser_controller.go @@ -31,6 +31,7 @@ import ( certv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "github.com/go-logr/logr" certsigningreqv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -63,6 +64,7 @@ func SetupKafkaUserWithManager(mgr ctrl.Manager, certSigningEnabled bool, certMa WithEventFilter(SkipClusterRegistryOwnedResourcePredicate{}). Named("KafkaUser") if certSigningEnabled { + builder.Owns(&corev1.Secret{}) csrMapper := csrMapper{ client: mgr.GetClient(), log: log, diff --git a/controllers/tests/kafkauser_controller_test.go b/controllers/tests/kafkauser_controller_test.go index be3293c9a..15c850ef1 100644 --- a/controllers/tests/kafkauser_controller_test.go +++ b/controllers/tests/kafkauser_controller_test.go @@ -361,5 +361,36 @@ x0eepFeUNacaeg7O1ftIrzNlYsSLi2Qm+tnu7odyxafZ65GJ9lcSLUqXuNDCrNOl } return user.Status.State, nil }, 5*time.Second, 100*time.Millisecond).Should(Equal(v1alpha1.UserStateCreated)) + + By("waiting for outstanding certificate retries to drain after readiness") + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: user.Namespace, + Name: user.Name, + }, &user)).To(Succeed()) + readyVersion := user.ResourceVersion + Consistently(ctx, func() (string, error) { + current := &v1alpha1.KafkaUser{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: user.Namespace, + Name: user.Name, + }, current) + return current.ResourceVersion, err + }, 16*time.Second, 100*time.Millisecond).Should(Equal(readyVersion)) + + By("recovering an owned certificate Secret from its deletion event alone") + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: user.Namespace, + Name: user.Spec.SecretName, + }, secret)).To(Succeed()) + originalUID := secret.UID + Expect(k8sClient.Delete(ctx, secret)).To(Succeed()) + Eventually(ctx, func() (types.UID, error) { + recreated := &corev1.Secret{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: user.Namespace, + Name: user.Spec.SecretName, + }, recreated) + return recreated.UID, err + }, 5*time.Second, 100*time.Millisecond).Should(And(Not(BeEmpty()), Not(Equal(originalUID)))) }) }) From 61a674cd0e03d8abc7d9e88b26cd1ca921c7385f Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Wed, 18 Nov 2026 11:57:47 +0100 Subject: [PATCH 5/5] fix(controllers): ignore certificate Secret self-write events (#394) Watch owned Secret deletions without reacting to self-generated create or update events that can race the CSR annotation cache. Reproduce duplicate CSR creation under held Secret cache visibility and verify deletion-driven Secret and CSR recovery without extras. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- controllers/kafkauser_controller.go | 8 +- .../tests/kafkauser_controller_test.go | 78 +++++++++++++++++++ controllers/tests/suite_test.go | 4 +- 3 files changed, 88 insertions(+), 2 deletions(-) diff --git a/controllers/kafkauser_controller.go b/controllers/kafkauser_controller.go index cea11a710..6372e6f05 100644 --- a/controllers/kafkauser_controller.go +++ b/controllers/kafkauser_controller.go @@ -64,7 +64,13 @@ func SetupKafkaUserWithManager(mgr ctrl.Manager, certSigningEnabled bool, certMa WithEventFilter(SkipClusterRegistryOwnedResourcePredicate{}). Named("KafkaUser") if certSigningEnabled { - builder.Owns(&corev1.Secret{}) + // Self-generated Secret events can race the CSR annotation cache; only deletion needs recovery. + builder.Owns(&corev1.Secret{}, ctrlBuilder.WithPredicates(predicate.Funcs{ + CreateFunc: func(event.CreateEvent) bool { return false }, + UpdateFunc: func(event.UpdateEvent) bool { return false }, + DeleteFunc: func(event.DeleteEvent) bool { return true }, + GenericFunc: func(event.GenericEvent) bool { return false }, + })) csrMapper := csrMapper{ client: mgr.GetClient(), log: log, diff --git a/controllers/tests/kafkauser_controller_test.go b/controllers/tests/kafkauser_controller_test.go index 15c850ef1..3ea11b0ba 100644 --- a/controllers/tests/kafkauser_controller_test.go +++ b/controllers/tests/kafkauser_controller_test.go @@ -16,7 +16,9 @@ package tests import ( + "context" "fmt" + "sync" "sync/atomic" "time" @@ -32,12 +34,56 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" "github.com/banzaicloud/koperator/pkg/util" + pkicommon "github.com/banzaicloud/koperator/pkg/util/pki" ) +type laggingUserSecretClient struct { + client.Client + mu sync.Mutex + key types.NamespacedName + snapshot *corev1.Secret +} + +func (c *laggingUserSecretClient) holdSecretUpdates(key types.NamespacedName) { + c.mu.Lock() + defer c.mu.Unlock() + c.key = key + c.snapshot = nil +} + +func (c *laggingUserSecretClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if secret, ok := obj.(*corev1.Secret); ok { + c.mu.Lock() + if key == c.key && c.snapshot != nil { + *secret = *c.snapshot.DeepCopy() + c.mu.Unlock() + return nil + } + c.mu.Unlock() + } + return c.Client.Get(ctx, key, obj, opts...) +} + +func (c *laggingUserSecretClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if err := c.Client.Create(ctx, obj, opts...); err != nil { + return err + } + if secret, ok := obj.(*corev1.Secret); ok { + c.mu.Lock() + defer c.mu.Unlock() + if client.ObjectKeyFromObject(secret) == c.key { + // Model an informer that has observed creation but not the subsequent CSR annotation update. + c.snapshot = secret.DeepCopy() + } + } + return nil +} + var _ = Describe("KafkaTopic", func() { var ( count uint64 = 0 @@ -241,6 +287,24 @@ var _ = Describe("KafkaTopic", func() { }, }, } + secretKey := types.NamespacedName{Name: user.Spec.SecretName, Namespace: user.Namespace} + kafkaUserClient.holdSecretUpdates(secretKey) + DeferCleanup(func() { + kafkaUserClient.holdSecretUpdates(types.NamespacedName{}) + }) + userCSRNames := func() ([]string, error) { + requests := &certsigningreqv1.CertificateSigningRequestList{} + if err := k8sClient.List(ctx, requests); err != nil { + return nil, err + } + var names []string + for _, request := range requests.Items { + if request.Annotations[pkicommon.KafkaUserAnnotationName] == client.ObjectKeyFromObject(&user).String() { + names = append(names, request.Name) + } + } + return names, nil + } err := k8sClient.Create(ctx, &user) Expect(err).NotTo(HaveOccurred()) @@ -256,6 +320,9 @@ var _ = Describe("KafkaTopic", func() { }, 5*time.Second, 100*time.Millisecond).Should(HaveKey(k8scsrpki.DependingCsrAnnotation)) csrName := secret.Annotations[k8scsrpki.DependingCsrAnnotation] + By("not creating duplicate CSRs from Secret self-writes while its annotation cache lags") + Consistently(ctx, userCSRNames, 2*time.Second, 100*time.Millisecond).Should(ConsistOf(csrName)) + kafkaUserClient.holdSecretUpdates(types.NamespacedName{}) csr := &certsigningreqv1.CertificateSigningRequest{} Eventually(ctx, func() error { err := k8sClient.Get(ctx, types.NamespacedName{ @@ -383,6 +450,7 @@ x0eepFeUNacaeg7O1ftIrzNlYsSLi2Qm+tnu7odyxafZ65GJ9lcSLUqXuNDCrNOl Name: user.Spec.SecretName, }, secret)).To(Succeed()) originalUID := secret.UID + kafkaUserClient.holdSecretUpdates(secretKey) Expect(k8sClient.Delete(ctx, secret)).To(Succeed()) Eventually(ctx, func() (types.UID, error) { recreated := &corev1.Secret{} @@ -392,5 +460,15 @@ x0eepFeUNacaeg7O1ftIrzNlYsSLi2Qm+tnu7odyxafZ65GJ9lcSLUqXuNDCrNOl }, recreated) return recreated.UID, err }, 5*time.Second, 100*time.Millisecond).Should(And(Not(BeEmpty()), Not(Equal(originalUID)))) + Eventually(ctx, func() (string, error) { + err := k8sClient.Get(ctx, secretKey, secret) + return secret.Annotations[k8scsrpki.DependingCsrAnnotation], err + }, 5*time.Second, 100*time.Millisecond).Should(And(Not(BeEmpty()), Not(Equal(csrName)))) + recoveredCSRName := secret.Annotations[k8scsrpki.DependingCsrAnnotation] + recoveredCSR := &certsigningreqv1.CertificateSigningRequest{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: recoveredCSRName}, recoveredCSR)).To(Succeed()) + Expect(recoveredCSR.Spec.Request).NotTo(BeEmpty()) + Consistently(ctx, userCSRNames, 2*time.Second, 100*time.Millisecond).Should(ConsistOf(csrName, recoveredCSRName)) + kafkaUserClient.holdSecretUpdates(types.NamespacedName{}) }) }) diff --git a/controllers/tests/suite_test.go b/controllers/tests/suite_test.go index 10ecaa739..b717f9f6a 100644 --- a/controllers/tests/suite_test.go +++ b/controllers/tests/suite_test.go @@ -69,6 +69,7 @@ import ( // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var k8sClient client.Client +var kafkaUserClient *laggingUserSecretClient var csrClient *csrclient.CertificatesV1Client var testEnv *envtest.Environment var mockKafkaClients map[types.NamespacedName]kafkaclient.KafkaClient @@ -175,8 +176,9 @@ var _ = BeforeSuite(func(ctx SpecContext) { Expect(err).NotTo(HaveOccurred()) // Create a new kafka user reconciler + kafkaUserClient = &laggingUserSecretClient{Client: mgr.GetClient()} kafkaUserReconciler := controllers.KafkaUserReconciler{ - Client: mgr.GetClient(), + Client: kafkaUserClient, Scheme: mgr.GetScheme(), }