From e076e05986908d04a10c340a073cb773731855bb Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Mon, 16 Nov 2026 16:59:17 +0100 Subject: [PATCH 1/3] fix(controllers): persist Cruise Control results through status conflicts Retry fresh API reads and status writes without repeating external submission. Preserve stop-operation identity and retry history, reject stale object/task identities, add seven regression cases, and document the remaining submit-before-persist crash window. Refs #369. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cruisecontroloperation_controller.go | 37 ++-- .../cruisecontroloperation_controller_test.go | 169 ++++++++++++++++++ docs/monitoring.md | 19 +- 3 files changed, 212 insertions(+), 13 deletions(-) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index d7da19d39..68a343bb2 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -243,24 +243,37 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques } } - conflictRetryFunction := func() error { - if err = updateResult(log, cruseControlTaskResult, ccOperationExecution, true); err != nil { - return err - } - err = r.Status().Update(ctx, ccOperationExecution) - if apiErrors.IsConflict(err) { - err = r.Get(ctx, client.ObjectKey{Name: ccOperationExecution.GetName(), Namespace: ccOperationExecution.GetNamespace()}, ccOperationExecution) - } - return err - } - // Protection to avoid re-execution task when status update is failed - if err := util.RetryOnConflict(util.DefaultBackOffForConflict, conflictRetryFunction); err != nil { + // Retry persistence without resubmitting the external task. + if err := r.persistExecutionResult(ctx, log, ccOperationExecution, cruseControlTaskResult); err != nil { return requeueWithError(log, "could not update the result of the Cruise Control user task execution to the CruiseControlOperation status", err) } return reconciled() } +func (r *CruiseControlOperationReconciler) persistExecutionResult( + ctx context.Context, + log logr.Logger, + operation *banzaiv1alpha1.CruiseControlOperation, + result *scale.Result, +) error { + return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { + latest := &banzaiv1alpha1.CruiseControlOperation{} + if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + return err + } + if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { + return errors.New("CruiseControlOperation identity or current task changed after external task submission") + } + // Preserve the stop operation selected in memory during finalization. + latest.Status.CurrentTask.Operation = operation.CurrentTaskOperation() + if err := updateResult(log, result, latest, true); err != nil { + return err + } + return r.Status().Update(ctx, latest) + }) +} + func (r *CruiseControlOperationReconciler) addFinalizer(ctx context.Context, currentCCOperation *banzaiv1alpha1.CruiseControlOperation) error { // examine DeletionTimestamp to determine if object is under deletion if currentCCOperation.DeletionTimestamp.IsZero() { diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index d9a68fd87..0c25b5c50 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -17,16 +17,22 @@ package controllers import ( "context" + "errors" "testing" "time" "github.com/go-logr/logr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" @@ -34,6 +40,169 @@ import ( "github.com/banzaicloud/koperator/pkg/scale" ) +func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { + for _, tt := range []struct { + name string + retry bool + deleting bool + replaced bool + taskRemoved bool + taskChanged bool + exhausted bool + conflicts int + wantRetries int + wantFailed int + wantOp v1alpha1.CruiseControlTaskOperation + }{ + {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, + {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, + {name: "replacement object rejects stale result", replaced: true, conflicts: 1}, + {name: "removed task rejects stale result", taskRemoved: true, conflicts: 1}, + {name: "newer task rejects stale result", taskChanged: true, conflicts: 1}, + {name: "exhausted conflicts return error", exhausted: true, conflicts: 1000}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := v1beta1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "cluster", Namespace: "ns"}} + operation := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: "operation", Namespace: "ns", UID: "operation-uid", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: "cluster"}, + }, + Spec: v1alpha1.CruiseControlOperationSpec{ErrorPolicy: v1alpha1.ErrorPolicyRetry}, + } + operation.Status.ErrorPolicy = v1alpha1.ErrorPolicyRetry + operation.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationAddBroker} + taskResult := &scale.Result{ + TaskID: "submitted-task", State: v1beta1.CruiseControlTaskInExecution, + StartedAt: "Mon, 02 Jan 2006 15:04:05 GMT", ResponseStatusCode: 202, + RequestURL: "http://cruise-control/add_broker", + } + var previous []*scale.Result + if tt.retry { + operation.Status.CurrentTask.ID = "failed-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskCompletedWithError + operation.Status.CurrentTask.Finished = &v1.Time{Time: time.Date(2006, 1, 2, 15, 5, 0, 0, time.UTC)} + operation.Status.RetryCount = 2 + operation.Status.FailedTasks = []v1alpha1.CruiseControlTask{{ID: "earlier-failed-task"}} + previous = []*scale.Result{{TaskID: "failed-task", State: v1beta1.CruiseControlTaskCompletedWithError}} + } + if tt.deleting { + operation.DeletionTimestamp = &v1.Time{Time: time.Now()} + operation.Finalizers = []string{ccOperationFinalizerGroup} + operation.Status.CurrentTask.ID = "running-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskInExecution + previous = []*scale.Result{{TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}} + } + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(t)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: true, + }}, nil) + var taskIDs []any + if len(previous) != 0 { + taskIDs = []any{previous[0].TaskID} + } + scaler.EXPECT().UserTasks(gomock.Any(), taskIDs...).Return(previous, nil) + if tt.deleting { + scaler.EXPECT().StopExecution(gomock.Any()).Return(taskResult, nil).Times(1) + } else { + scaler.EXPECT().AddBrokersWithParams(gomock.Any(), operation.CurrentTaskParameters()).Return(taskResult, nil).Times(1) + } + var resultWrites int + kubeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(cluster, operation). + WithStatusSubresource(operation). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if op, ok := obj.(*v1alpha1.CruiseControlOperation); ok && subresource == "status" && op.CurrentTaskID() == "submitted-task" { + resultWrites++ + if resultWrites <= tt.conflicts { + latest := &v1alpha1.CruiseControlOperation{} + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), latest); err != nil { + return err + } + latest.Annotations = map[string]string{"other-writer": "preserved"} + if err := c.Update(ctx, latest); err != nil { + return err + } + if tt.replaced { + latest.Finalizers = nil + if err := c.Update(ctx, latest); err != nil { + return err + } + if err := c.Delete(ctx, latest); err != nil { + return err + } + latest.ResourceVersion = "" + latest.UID = "replacement-uid" + if err := c.Create(ctx, latest); err != nil { + return err + } + } + if tt.taskRemoved || tt.taskChanged { + if tt.taskRemoved { + latest.Status.CurrentTask = nil + } else { + latest.Status.CurrentTask.ID = "newer-task" + } + if err := c.Status().Update(ctx, latest); err != nil { + return err + } + } + return apierrors.NewConflict(schema.GroupResource{Group: "kafka.banzaicloud.io", Resource: "cruisecontroloperations"}, obj.GetName(), errors.New("injected concurrent update")) + } + } + return c.SubResource(subresource).Update(ctx, obj, opts...) + }, + }).Build() + reconciler := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: kubeClient, Scheme: scheme, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { + return scaler, nil + }, + } + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(operation)}) + if err != nil && !tt.replaced && !tt.taskRemoved && !tt.taskChanged && !tt.exhausted { + t.Fatal(err) + } + persisted := &v1alpha1.CruiseControlOperation{} + if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(operation), persisted); err != nil { + t.Fatal(err) + } + if tt.replaced || tt.taskRemoved || tt.taskChanged || tt.exhausted { + if err == nil { + t.Error("reported success without persisting the result to the original task") + } + if persisted.CurrentTaskID() == "submitted-task" { + t.Error("attached stale external result to a replaced operation or removed task") + } + return + } + assert.Equal(t, "submitted-task", persisted.CurrentTaskID()) + assert.Equal(t, v1beta1.CruiseControlTaskInExecution, persisted.CurrentTaskState()) + assert.Equal(t, tt.wantOp, persisted.CurrentTaskOperation()) + task := persisted.CurrentTask() + require.NotNil(t, task) + assert.Equal(t, taskResult.RequestURL, task.HTTPRequest) + assert.Equal(t, &taskResult.ResponseStatusCode, task.HTTPResponseCode) + require.NotNil(t, task.Started) + assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) + assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) + assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + assert.Equal(t, tt.conflicts+1, resultWrites) + }) + } +} + func createCCRetryExecutionOperation(createTime time.Time, id string, operation v1alpha1.CruiseControlTaskOperation) *v1alpha1.CruiseControlOperation { return &v1alpha1.CruiseControlOperation{ ObjectMeta: v1.ObjectMeta{ diff --git a/docs/monitoring.md b/docs/monitoring.md index b6f802f1c..8d917bade 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -37,4 +37,21 @@ sum by (verb) (rate(rest_client_rate_limiter_duration_seconds_sum[5m])) Record cluster size, resource counts, workload, test duration, leader changes, and error rates with each measurement. These metrics establish instrumentation, not evidence that an optimization improves performance. This change does not -alter metrics authentication, scraper routing, worker counts, or cache scope. \ No newline at end of file +alter metrics authentication, scraper routing, worker counts, or cache scope. + +## Cruise Control execution-result persistence + +After an external task is submitted, the operator retries conflicting status +writes using fresh API-server reads. These persistence retries do not submit +the external task again. They retain the task identity, response metadata, +retry history, and selected stop-operation type. A replaced operation, removed +current task, or different task identity produces an error rather than accepting +a stale result. + +This is not exactly-once execution. If all writes fail, or the process stops +after Cruise Control accepts a request but before its task ID is recorded, the +external task can outlive the recorded operation state. Recorded task IDs allow +subsequent reconciliation to query task progress. Automatic adoption of an +unrecorded submission is not implemented; inspect Cruise Control's task history +when investigating this window, and do not infer that a missing status means +the external operation never ran. \ No newline at end of file From 33c4e138f402807e579731d41bc2ba9bf222b263 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 08:37:01 +0100 Subject: [PATCH 2/3] perf(controllers): refresh Cruise Control results only on conflict Reuse the selected operation for the first status write and read directly only when retrying a conflict. Cover conflict-free submission, retry, and stop paths. Refs #369. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cruisecontroloperation_controller.go | 6 +++++- .../cruisecontroloperation_controller_test.go | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index 68a343bb2..0bf53b9e8 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -257,11 +257,15 @@ func (r *CruiseControlOperationReconciler) persistExecutionResult( operation *banzaiv1alpha1.CruiseControlOperation, result *scale.Result, ) error { + firstAttempt := true return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { latest := &banzaiv1alpha1.CruiseControlOperation{} - if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + if firstAttempt { + latest = operation.DeepCopy() + } else if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { return err } + firstAttempt = false if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { return errors.New("CruiseControlOperation identity or current task changed after external task submission") } diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index 0c25b5c50..770ffa962 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -54,6 +54,9 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { wantFailed int wantOp v1alpha1.CruiseControlTaskOperation }{ + {name: "first submission without conflict", wantOp: v1alpha1.OperationAddBroker}, + {name: "retry without conflict preserves history once", retry: true, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission without conflict retains operation", deleting: true, wantOp: v1alpha1.OperationStopExecution}, {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, @@ -163,8 +166,16 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { return c.SubResource(subresource).Update(ctx, obj, opts...) }, }).Build() + var directReads int + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + directReads++ + assert.Positive(t, resultWrites, "direct GET must follow a conflicting result write") + return c.Get(ctx, key, obj, opts...) + }, + }) reconciler := &CruiseControlOperationReconciler{ - Client: kubeClient, DirectClient: kubeClient, Scheme: scheme, + Client: kubeClient, DirectClient: directClient, Scheme: scheme, ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { return scaler, nil }, @@ -197,8 +208,11 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) - assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + if tt.conflicts > 0 { + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + } assert.Equal(t, tt.conflicts+1, resultWrites) + assert.Equal(t, tt.conflicts, directReads) }) } } From be634e12988a11f823a10306e58a45b2436d1936 Mon Sep 17 00:00:00 2001 From: Adrian Muraru Date: Sat, 21 Nov 2026 15:14:41 +0100 Subject: [PATCH 3/3] perf(controllers): scope Cruise Control operation retrieval (#399) perf(controllers): scope Cruise Control operation retrieval (#393) Use the cached cluster label only as a hint for an authoritative direct LIST. Fall back to the namespace LIST for missing or stale hints without adding direct GETs before conflict. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cruisecontroloperation_controller.go | 24 +- .../cruisecontroloperation_controller_test.go | 341 +++++++++++++++++- .../cruisecontroloperation_controller_test.go | 149 ++++++++ 3 files changed, 511 insertions(+), 3 deletions(-) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index 0bf53b9e8..cec8b1684 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -83,8 +83,7 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques log := logr.FromContextOrDiscard(ctx) log.V(1).Info("reconciling CruiseControlOperation custom resources") - ccOperationListClusterWide := banzaiv1alpha1.CruiseControlOperationList{} - err := r.DirectClient.List(ctx, &ccOperationListClusterWide, client.ListOption(client.InNamespace(request.Namespace))) + ccOperationListClusterWide, err := r.listOperations(ctx, request.NamespacedName) if err != nil { return requeueWithError(log, err.Error(), err) } @@ -251,6 +250,27 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques return reconciled() } +func (r *CruiseControlOperationReconciler) listOperations(ctx context.Context, key client.ObjectKey) (banzaiv1alpha1.CruiseControlOperationList, error) { + operations := banzaiv1alpha1.CruiseControlOperationList{} + hint := &banzaiv1alpha1.CruiseControlOperation{} + // The cache supplies only a selector hint; scheduling always uses the direct LIST. + if err := r.Get(ctx, key, hint); err == nil && hint.GetClusterRef() != "" { + if err := r.DirectClient.List(ctx, &operations, client.InNamespace(key.Namespace), + client.MatchingLabels{banzaiv1beta1.KafkaCRLabelKey: hint.GetClusterRef()}); err != nil { + return operations, err + } + for i := range operations.Items { + if client.ObjectKeyFromObject(&operations.Items[i]) == key { + return operations, nil + } + } + } + + operations = banzaiv1alpha1.CruiseControlOperationList{} + err := r.DirectClient.List(ctx, &operations, client.InNamespace(key.Namespace)) + return operations, err +} + func (r *CruiseControlOperationReconciler) persistExecutionResult( ctx context.Context, log logr.Logger, diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index 770ffa962..c9a3e2610 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -18,6 +18,7 @@ package controllers import ( "context" "errors" + "fmt" "testing" "time" @@ -29,6 +30,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -40,6 +42,324 @@ import ( "github.com/banzaicloud/koperator/pkg/scale" ) +func TestOperationRetrievalCompatibility(t *testing.T) { + for _, tt := range []struct { + name string + hint string + cacheMiss bool + cacheError bool + actualLabel string + deleted bool + legacy bool + listErrorAt int + wantSelectors []string + wantObjects []int + }{ + {name: "scoped list excludes unrelated history and namespaces", hint: "target", actualLabel: "target", wantSelectors: []string{"target"}, wantObjects: []int{3}}, + {name: "cache miss falls back", cacheMiss: true, actualLabel: "target", wantSelectors: []string{""}, wantObjects: []int{5}}, + {name: "cache error falls back", cacheError: true, actualLabel: "target", wantSelectors: []string{""}, wantObjects: []int{5}}, + {name: "unlabeled hint falls back", actualLabel: "target", wantSelectors: []string{""}, wantObjects: []int{5}}, + {name: "reassigned label falls back", hint: "old", actualLabel: "target", wantSelectors: []string{"old", ""}, wantObjects: []int{1, 5}}, + {name: "removed label retains legacy error", hint: "target", legacy: true, wantSelectors: []string{"target", ""}, wantObjects: []int{2, 5}}, + {name: "unlabeled legacy object retains error", legacy: true, wantSelectors: []string{""}, wantObjects: []int{5}}, + {name: "deleted object is not recovered from cache or another namespace", hint: "target", deleted: true, wantSelectors: []string{"target", ""}, wantObjects: []int{2, 4}}, + {name: "scoped list error propagates", hint: "target", actualLabel: "target", listErrorAt: 1, wantSelectors: []string{"target"}}, + {name: "fallback list error propagates", hint: "old", actualLabel: "target", listErrorAt: 2, wantSelectors: []string{"old", ""}, wantObjects: []int{1}}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(scheme)) + current := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{Name: "current", Namespace: "ns", UID: "replacement", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: tt.actualLabel}}, + Status: v1alpha1.CruiseControlOperationStatus{CurrentTask: &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationStatus}}, + } + if tt.legacy { + current.Labels = nil + current.OwnerReferences = []v1.OwnerReference{{Name: "target", Kind: "KafkaCluster"}} + current.Status.CurrentTask.Operation = v1alpha1.OperationAddBroker + } + hint := current.DeepCopy() + hint.UID = "old-uid" + hint.Labels = map[string]string{v1beta1.KafkaCRLabelKey: tt.hint} + hint.Status.CurrentTask.Operation = v1alpha1.OperationRemoveBroker + cacheBuilder := fake.NewClientBuilder().WithScheme(scheme) + if !tt.cacheMiss { + cacheBuilder.WithObjects(hint) + } + cacheFailure := errors.New("cache unavailable") + cachedReads := 0 + cachedClient := interceptor.NewClient(cacheBuilder.Build(), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + cachedReads++ + if tt.cacheError { + return cacheFailure + } + return c.Get(ctx, key, obj, opts...) + }, + }) + var objects []client.Object + if !tt.deleted { + objects = append(objects, current) + } + for _, peer := range []struct{ name, namespace, label string }{ + {"z-peer", "ns", "target"}, {"a-peer", "ns", "target"}, + {"unrelated", "ns", "old"}, {"legacy", "ns", ""}, + {"current", "other-ns", "target"}, + } { + objects = append(objects, &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{Name: peer.name, Namespace: peer.namespace, + Labels: map[string]string{v1beta1.KafkaCRLabelKey: peer.label}}, + }) + } + var selectors []string + var objectCounts []int + listFailure := errors.New("direct list failed") + directClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + t.Error("operation retrieval must not issue a direct GET") + return errors.New("unexpected direct GET") + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + options := (&client.ListOptions{}).ApplyOptions(opts) + require.Equal(t, "ns", options.Namespace) + selector := "" + if options.LabelSelector != nil { + value, ok := options.LabelSelector.RequiresExactMatch(v1beta1.KafkaCRLabelKey) + require.True(t, ok) + selector = value + } + selectors = append(selectors, selector) + if len(selectors) == tt.listErrorAt { + return listFailure + } + if err := c.List(ctx, list, opts...); err != nil { + return err + } + objectCounts = append(objectCounts, len(list.(*v1alpha1.CruiseControlOperationList).Items)) + return nil + }, + }).Build() + r := &CruiseControlOperationReconciler{Client: cachedClient, DirectClient: directClient} + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(current)}) + switch { + case tt.listErrorAt != 0: + require.ErrorIs(t, err, listFailure) + case tt.legacy: + require.ErrorContains(t, err, "could not find kafka cluster reference label") + default: + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + } + assert.Equal(t, 1, cachedReads) + assert.Equal(t, tt.wantSelectors, selectors) + assert.Equal(t, tt.wantObjects, objectCounts) + }) + } +} + +func TestOperationRetrievalPreservesExecutionOrder(t *testing.T) { + for _, stopping := range []bool{false, true} { + t.Run(fmt.Sprintf("stopping=%t", stopping), func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "target", Namespace: "ns"}} + objects := []client.Object{cluster} + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i, name := range []string{"z-first", "a-second", "request", "unrelated"} { + op := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: name, Namespace: "ns", UID: "uid-" + types.UID(name), + CreationTimestamp: v1.NewTime(start.Add(time.Duration(900-i*100) * time.Millisecond)), + Labels: map[string]string{v1beta1.KafkaCRLabelKey: "target"}, + Finalizers: []string{ccOperationFinalizerGroup}, + }, + Status: v1alpha1.CruiseControlOperationStatus{CurrentTask: &v1alpha1.CruiseControlTask{ + Operation: v1alpha1.OperationAddBroker, Parameters: map[string]string{"brokerid": name}, + }}, + } + if name == "request" { + op.CreationTimestamp = v1.NewTime(start.Add(-time.Hour)) + op.Status.CurrentTask.Operation = v1alpha1.OperationRebalance + } + if name == "unrelated" { + op.Labels[v1beta1.KafkaCRLabelKey] = "other" + } + if stopping && name == "z-first" { + op.DeletionTimestamp = &v1.Time{Time: start} + op.Status.CurrentTask.ID = "running-task" + op.Status.CurrentTask.State = v1beta1.CruiseControlTaskInExecution + } + objects = append(objects, op) + } + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). + WithStatusSubresource(&v1alpha1.CruiseControlOperation{}).Build() + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + t.Fatal("first execution must not issue a direct GET") + return nil + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := c.List(ctx, list, opts...); err != nil { + return err + } + operations := list.(*v1alpha1.CruiseControlOperationList) + items := operations.Items + operations.Items = nil + for _, name := range []string{"z-first", "unrelated", "a-second", "request"} { + for _, op := range items { + if op.Name == name { + operations.Items = append(operations.Items, op) + } + } + } + return nil + }, + }) + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(t)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: true, + }}, nil) + var taskIDs []any + var tasks []*scale.Result + if stopping { + taskIDs = []any{"running-task"} + tasks = []*scale.Result{{TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}} + } + scaler.EXPECT().UserTasks(gomock.Any(), taskIDs...).Return(tasks, nil) + result := &scale.Result{ + TaskID: "submitted", State: v1beta1.CruiseControlTaskInExecution, + StartedAt: "Mon, 02 Jan 2006 15:04:05 GMT", + } + if stopping { + scaler.EXPECT().StopExecution(gomock.Any()).Return(result, nil) + } else { + scaler.EXPECT().AddBrokersWithParams(gomock.Any(), map[string]string{"brokerid": "z-first"}).Return(result, nil) + } + r := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: directClient, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { return scaler, nil }, + } + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKey{Name: "request", Namespace: "ns"}}) + require.NoError(t, err) + persisted := &v1alpha1.CruiseControlOperation{} + require.NoError(t, kubeClient.Get(ctx, client.ObjectKey{Name: "z-first", Namespace: "ns"}, persisted)) + assert.Equal(t, "submitted", persisted.CurrentTaskID()) + }) + } +} + +func BenchmarkCruiseControlUnrelatedHistory(b *testing.B) { + for _, unrelatedClusters := range []int{0, 10, 100} { + b.Run(fmt.Sprintf("unrelated-clusters=%d_history-each=10", unrelatedClusters), func(b *testing.B) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(b, v1alpha1.AddToScheme(scheme)) + require.NoError(b, v1beta1.AddToScheme(scheme)) + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "target", Namespace: "ns"}} + operation := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: "active", Namespace: "ns", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: cluster.Name}, + Finalizers: []string{ccOperationFinalizerGroup}, + }, + Status: v1alpha1.CruiseControlOperationStatus{ + CurrentTask: &v1alpha1.CruiseControlTask{ + ID: "running-task", Operation: v1alpha1.OperationAddBroker, State: v1beta1.CruiseControlTaskInExecution, + }, + }, + } + objects := []client.Object{cluster, operation} + for clusterIndex := 0; clusterIndex <= unrelatedClusters; clusterIndex++ { + clusterName := cluster.Name + if clusterIndex != 0 { + clusterName = fmt.Sprintf("unrelated-%03d", clusterIndex) + } + for history := 0; history < 10; history++ { + completed := operation.DeepCopy() + completed.Name = fmt.Sprintf("%s-history-%02d", clusterName, history) + completed.Labels = map[string]string{v1beta1.KafkaCRLabelKey: clusterName} + completed.Finalizers = nil + completed.Status.CurrentTask.State = v1beta1.CruiseControlTaskCompleted + objects = append(objects, completed) + } + } + var lists, listedObjects, gets, writes int + kubeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(objects...).WithStatusSubresource(operation). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lists++ + if err := c.List(ctx, list, opts...); err != nil { + return err + } + listedObjects += len(list.(*v1alpha1.CruiseControlOperationList).Items) + return nil + }, + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + gets++ + return c.Get(ctx, key, obj, opts...) + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + writes++ + return c.Update(ctx, obj, opts...) + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + writes++ + return c.SubResource(subresource).Update(ctx, obj, opts...) + }, + }).Build() + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(b)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: false, + }}, nil).Times(b.N) + scaler.EXPECT().UserTasks(gomock.Any(), "running-task").Return([]*scale.Result{ + {TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}, + }, nil).Times(b.N) + var directGets int + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + directGets++ + return c.Get(ctx, key, obj, opts...) + }, + }) + reconciler := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: directClient, Scheme: scheme, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { + return scaler, nil + }, + } + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(operation)} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := reconciler.Reconcile(ctx, request) + if err != nil { + b.Fatal(err) + } + if result.RequeueAfter != time.Duration(defaultRequeueIntervalInSeconds)*time.Second { + b.Fatalf("unexpected scheduling result: %+v", result) + } + } + b.StopTimer() + require.Equal(b, b.N, lists) + require.Equal(b, b.N*11, listedObjects) + require.Equal(b, b.N*2, gets) + require.Zero(b, directGets) + require.Zero(b, writes) + b.ReportMetric(float64(lists)/float64(b.N), "LIST/reconcile") + b.ReportMetric(float64(listedObjects)/float64(b.N), "objects/reconcile") + b.ReportMetric(float64(gets)/float64(b.N), "cachedGET/reconcile") + b.ReportMetric(float64(directGets)/float64(b.N), "directGET/reconcile") + b.ReportMetric(float64(writes)/float64(b.N), "writes/reconcile") + }) + } +} + func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { for _, tt := range []struct { name string @@ -53,10 +373,13 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { wantRetries int wantFailed int wantOp v1alpha1.CruiseControlTaskOperation + staleHint bool }{ {name: "first submission without conflict", wantOp: v1alpha1.OperationAddBroker}, {name: "retry without conflict preserves history once", retry: true, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, {name: "stop submission without conflict retains operation", deleting: true, wantOp: v1alpha1.OperationStopExecution}, + {name: "recreated operation uses fresh UID task and parameters", staleHint: true, wantOp: v1alpha1.OperationAddBroker}, + {name: "fresh deletion overrides stale cached task", deleting: true, staleHint: true, wantOp: v1alpha1.OperationStopExecution}, {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, @@ -174,8 +497,24 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { return c.Get(ctx, key, obj, opts...) }, }) + cachedClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + if op, ok := obj.(*v1alpha1.CruiseControlOperation); ok && tt.staleHint { + op.UID = "deleted-operation-uid" + op.DeletionTimestamp = nil + op.Status.CurrentTask = &v1alpha1.CruiseControlTask{ + ID: "stale-task", Operation: v1alpha1.OperationRemoveBroker, + Parameters: map[string]string{"brokerid": "stale"}, + } + } + return nil + }, + }) reconciler := &CruiseControlOperationReconciler{ - Client: kubeClient, DirectClient: directClient, Scheme: scheme, + Client: cachedClient, DirectClient: directClient, Scheme: scheme, ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { return scaler, nil }, diff --git a/controllers/tests/cruisecontroloperation_controller_test.go b/controllers/tests/cruisecontroloperation_controller_test.go index 49354f0f3..38a4f62f0 100644 --- a/controllers/tests/cruisecontroloperation_controller_test.go +++ b/controllers/tests/cruisecontroloperation_controller_test.go @@ -24,8 +24,12 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/mock/gomock" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/banzaicloud/koperator/controllers" "github.com/banzaicloud/koperator/controllers/tests/mocks" corev1 "k8s.io/api/core/v1" @@ -43,6 +47,151 @@ const ( operationExtendedTimeoutDuration = 15 * time.Second // extended timeout for complex operations ) +type operationRetrievalHintClient struct { + client.Client + hint *v1alpha1.CruiseControlOperation + miss bool +} + +func (c *operationRetrievalHintClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if operation, ok := obj.(*v1alpha1.CruiseControlOperation); ok { + if c.miss { + return apierrors.NewNotFound(schema.GroupResource{Group: "kafka.banzaicloud.io", Resource: "cruisecontroloperations"}, key.Name) + } + if c.hint != nil { + c.hint.DeepCopyInto(operation) + return nil + } + } + return c.Client.Get(ctx, key, obj, opts...) +} + +type operationRetrievalReader struct { + client.Reader + selectors []string + results []v1alpha1.CruiseControlOperationList + gets int +} + +func (r *operationRetrievalReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + r.gets++ + return r.Reader.Get(ctx, key, obj, opts...) +} + +func (r *operationRetrievalReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + options := (&client.ListOptions{}).ApplyOptions(opts) + selector := "" + if options.LabelSelector != nil { + selector = options.LabelSelector.String() + } + r.selectors = append(r.selectors, selector) + if err := r.Reader.List(ctx, list, opts...); err != nil { + return err + } + r.results = append(r.results, *list.(*v1alpha1.CruiseControlOperationList).DeepCopy()) + return nil +} + +var _ = Describe("CruiseControlOperation scoped retrieval", Serial, func() { + DescribeTable("uses fresh API objects and preserves compatible fallback", func(ctx SpecContext, mode string) { + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "operation-retrieval-"}} + Expect(k8sClient.Create(ctx, namespace)).To(Succeed()) + DeferCleanup(func(ctx SpecContext) { + Expect(k8sClient.DeleteAllOf(ctx, &v1alpha1.CruiseControlOperation{}, client.InNamespace(namespace.Name))).To(Succeed()) + Expect(k8sClient.Delete(ctx, namespace)).To(Succeed()) + }) + var current *v1alpha1.CruiseControlOperation + for _, name := range []string{"z-current", "a-peer", "unrelated"} { + cluster := "target" + if name == "unrelated" { + cluster = "old" + } + operation := generateCruiseControlOperation(name, namespace.Name, cluster) + Expect(k8sClient.Create(ctx, &operation)).To(Succeed()) + operation.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationStatus} + Expect(k8sClient.Status().Update(ctx, &operation)).To(Succeed()) + if name == "z-current" { + current = operation.DeepCopy() + } + } + cached := &operationRetrievalHintClient{Client: cruiseControlOperationReconciler.Client} + Eventually(ctx, func(g Gomega) { + observed := &v1alpha1.CruiseControlOperation{} + g.Expect(cached.Get(ctx, client.ObjectKeyFromObject(current), observed)).To(Succeed()) + g.Expect(observed.CurrentTaskOperation()).To(Equal(v1alpha1.OperationStatus)) + }).Should(Succeed()) + switch mode { + case "miss": + cached.miss = true + case "stale": + cached.hint = current.DeepCopy() + cached.hint.Labels[v1beta1.KafkaCRLabelKey] = "old" + case "unlabeled-hint": + cached.hint = current.DeepCopy() + cached.hint.Labels = nil + case "legacy": + current.Labels = nil + Expect(k8sClient.Update(ctx, current)).To(Succeed()) + current.Status.CurrentTask.Operation = v1alpha1.OperationAddBroker + Expect(k8sClient.Status().Update(ctx, current)).To(Succeed()) + cached.hint = current.DeepCopy() + case "recreated": + cached.hint = current.DeepCopy() + Expect(k8sClient.Delete(ctx, current)).To(Succeed()) + replacement := generateCruiseControlOperation(current.Name, namespace.Name, "target") + Expect(k8sClient.Create(ctx, &replacement)).To(Succeed()) + replacement.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationStatus} + Expect(k8sClient.Status().Update(ctx, &replacement)).To(Succeed()) + Expect(replacement.UID).NotTo(Equal(current.UID)) + current = replacement.DeepCopy() + } + baseline := &v1alpha1.CruiseControlOperationList{} + Expect(k8sClient.List(ctx, baseline, client.InNamespace(namespace.Name))).To(Succeed()) + reader := &operationRetrievalReader{Reader: k8sClient} + reconciler := &controllers.CruiseControlOperationReconciler{Client: cached, DirectClient: reader} + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(current)}) + if mode == "legacy" { + Expect(err).To(MatchError(ContainSubstring("could not find kafka cluster reference label"))) + } else { + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeZero()) + } + scopedSelector := v1beta1.KafkaCRLabelKey + "=target" + switch mode { + case "miss", "unlabeled-hint", "legacy": + Expect(reader.selectors).To(Equal([]string{""})) + case "stale": + Expect(reader.selectors).To(Equal([]string{v1beta1.KafkaCRLabelKey + "=old", ""})) + default: + Expect(reader.selectors).To(Equal([]string{scopedSelector})) + } + expected := make([]string, 0) + for _, operation := range baseline.Items { + if len(reader.selectors) != 1 || reader.selectors[0] == "" || operation.GetClusterRef() == "target" { + expected = append(expected, operation.Name) + } + } + last := reader.results[len(reader.results)-1] + observed := make([]string, 0, len(last.Items)) + for _, operation := range last.Items { + observed = append(observed, operation.Name) + if operation.Name == current.Name { + Expect(operation.UID).To(Equal(current.UID)) + Expect(operation.Status).To(Equal(current.Status)) + } + } + Expect(observed).To(Equal(expected)) + Expect(reader.gets).To(BeZero()) + }, + Entry("warm cache", "cached"), + Entry("cache miss", "miss"), + Entry("reassigned label", "stale"), + Entry("unlabeled hint", "unlabeled-hint"), + Entry("unlabeled legacy object", "legacy"), + Entry("deleted and recreated object", "recreated"), + ) +}) + var _ = Describe("CruiseControlTaskReconciler", func() { var ( count uint64 = 0