diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index 0bf53b9e8..cec8b1684 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -83,8 +83,7 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques log := logr.FromContextOrDiscard(ctx) log.V(1).Info("reconciling CruiseControlOperation custom resources") - ccOperationListClusterWide := banzaiv1alpha1.CruiseControlOperationList{} - err := r.DirectClient.List(ctx, &ccOperationListClusterWide, client.ListOption(client.InNamespace(request.Namespace))) + ccOperationListClusterWide, err := r.listOperations(ctx, request.NamespacedName) if err != nil { return requeueWithError(log, err.Error(), err) } @@ -251,6 +250,27 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques return reconciled() } +func (r *CruiseControlOperationReconciler) listOperations(ctx context.Context, key client.ObjectKey) (banzaiv1alpha1.CruiseControlOperationList, error) { + operations := banzaiv1alpha1.CruiseControlOperationList{} + hint := &banzaiv1alpha1.CruiseControlOperation{} + // The cache supplies only a selector hint; scheduling always uses the direct LIST. + if err := r.Get(ctx, key, hint); err == nil && hint.GetClusterRef() != "" { + if err := r.DirectClient.List(ctx, &operations, client.InNamespace(key.Namespace), + client.MatchingLabels{banzaiv1beta1.KafkaCRLabelKey: hint.GetClusterRef()}); err != nil { + return operations, err + } + for i := range operations.Items { + if client.ObjectKeyFromObject(&operations.Items[i]) == key { + return operations, nil + } + } + } + + operations = banzaiv1alpha1.CruiseControlOperationList{} + err := r.DirectClient.List(ctx, &operations, client.InNamespace(key.Namespace)) + return operations, err +} + func (r *CruiseControlOperationReconciler) persistExecutionResult( ctx context.Context, log logr.Logger, diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index 770ffa962..c9a3e2610 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -18,6 +18,7 @@ package controllers import ( "context" "errors" + "fmt" "testing" "time" @@ -29,6 +30,7 @@ import ( v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -40,6 +42,324 @@ import ( "github.com/banzaicloud/koperator/pkg/scale" ) +func TestOperationRetrievalCompatibility(t *testing.T) { + for _, tt := range []struct { + name string + hint string + cacheMiss bool + cacheError bool + actualLabel string + deleted bool + legacy bool + listErrorAt int + wantSelectors []string + wantObjects []int + }{ + {name: "scoped list excludes unrelated history and namespaces", hint: "target", actualLabel: "target", wantSelectors: []string{"target"}, wantObjects: []int{3}}, + {name: "cache miss falls back", cacheMiss: true, actualLabel: "target", wantSelectors: []string{""}, wantObjects: []int{5}}, + {name: "cache error falls back", cacheError: true, actualLabel: "target", wantSelectors: []string{""}, wantObjects: []int{5}}, + {name: "unlabeled hint falls back", actualLabel: "target", wantSelectors: []string{""}, wantObjects: []int{5}}, + {name: "reassigned label falls back", hint: "old", actualLabel: "target", wantSelectors: []string{"old", ""}, wantObjects: []int{1, 5}}, + {name: "removed label retains legacy error", hint: "target", legacy: true, wantSelectors: []string{"target", ""}, wantObjects: []int{2, 5}}, + {name: "unlabeled legacy object retains error", legacy: true, wantSelectors: []string{""}, wantObjects: []int{5}}, + {name: "deleted object is not recovered from cache or another namespace", hint: "target", deleted: true, wantSelectors: []string{"target", ""}, wantObjects: []int{2, 4}}, + {name: "scoped list error propagates", hint: "target", actualLabel: "target", listErrorAt: 1, wantSelectors: []string{"target"}}, + {name: "fallback list error propagates", hint: "old", actualLabel: "target", listErrorAt: 2, wantSelectors: []string{"old", ""}, wantObjects: []int{1}}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(scheme)) + current := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{Name: "current", Namespace: "ns", UID: "replacement", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: tt.actualLabel}}, + Status: v1alpha1.CruiseControlOperationStatus{CurrentTask: &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationStatus}}, + } + if tt.legacy { + current.Labels = nil + current.OwnerReferences = []v1.OwnerReference{{Name: "target", Kind: "KafkaCluster"}} + current.Status.CurrentTask.Operation = v1alpha1.OperationAddBroker + } + hint := current.DeepCopy() + hint.UID = "old-uid" + hint.Labels = map[string]string{v1beta1.KafkaCRLabelKey: tt.hint} + hint.Status.CurrentTask.Operation = v1alpha1.OperationRemoveBroker + cacheBuilder := fake.NewClientBuilder().WithScheme(scheme) + if !tt.cacheMiss { + cacheBuilder.WithObjects(hint) + } + cacheFailure := errors.New("cache unavailable") + cachedReads := 0 + cachedClient := interceptor.NewClient(cacheBuilder.Build(), interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + cachedReads++ + if tt.cacheError { + return cacheFailure + } + return c.Get(ctx, key, obj, opts...) + }, + }) + var objects []client.Object + if !tt.deleted { + objects = append(objects, current) + } + for _, peer := range []struct{ name, namespace, label string }{ + {"z-peer", "ns", "target"}, {"a-peer", "ns", "target"}, + {"unrelated", "ns", "old"}, {"legacy", "ns", ""}, + {"current", "other-ns", "target"}, + } { + objects = append(objects, &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{Name: peer.name, Namespace: peer.namespace, + Labels: map[string]string{v1beta1.KafkaCRLabelKey: peer.label}}, + }) + } + var selectors []string + var objectCounts []int + listFailure := errors.New("direct list failed") + directClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). + WithInterceptorFuncs(interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + t.Error("operation retrieval must not issue a direct GET") + return errors.New("unexpected direct GET") + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + options := (&client.ListOptions{}).ApplyOptions(opts) + require.Equal(t, "ns", options.Namespace) + selector := "" + if options.LabelSelector != nil { + value, ok := options.LabelSelector.RequiresExactMatch(v1beta1.KafkaCRLabelKey) + require.True(t, ok) + selector = value + } + selectors = append(selectors, selector) + if len(selectors) == tt.listErrorAt { + return listFailure + } + if err := c.List(ctx, list, opts...); err != nil { + return err + } + objectCounts = append(objectCounts, len(list.(*v1alpha1.CruiseControlOperationList).Items)) + return nil + }, + }).Build() + r := &CruiseControlOperationReconciler{Client: cachedClient, DirectClient: directClient} + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(current)}) + switch { + case tt.listErrorAt != 0: + require.ErrorIs(t, err, listFailure) + case tt.legacy: + require.ErrorContains(t, err, "could not find kafka cluster reference label") + default: + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + } + assert.Equal(t, 1, cachedReads) + assert.Equal(t, tt.wantSelectors, selectors) + assert.Equal(t, tt.wantObjects, objectCounts) + }) + } +} + +func TestOperationRetrievalPreservesExecutionOrder(t *testing.T) { + for _, stopping := range []bool{false, true} { + t.Run(fmt.Sprintf("stopping=%t", stopping), func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "target", Namespace: "ns"}} + objects := []client.Object{cluster} + start := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + for i, name := range []string{"z-first", "a-second", "request", "unrelated"} { + op := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: name, Namespace: "ns", UID: "uid-" + types.UID(name), + CreationTimestamp: v1.NewTime(start.Add(time.Duration(900-i*100) * time.Millisecond)), + Labels: map[string]string{v1beta1.KafkaCRLabelKey: "target"}, + Finalizers: []string{ccOperationFinalizerGroup}, + }, + Status: v1alpha1.CruiseControlOperationStatus{CurrentTask: &v1alpha1.CruiseControlTask{ + Operation: v1alpha1.OperationAddBroker, Parameters: map[string]string{"brokerid": name}, + }}, + } + if name == "request" { + op.CreationTimestamp = v1.NewTime(start.Add(-time.Hour)) + op.Status.CurrentTask.Operation = v1alpha1.OperationRebalance + } + if name == "unrelated" { + op.Labels[v1beta1.KafkaCRLabelKey] = "other" + } + if stopping && name == "z-first" { + op.DeletionTimestamp = &v1.Time{Time: start} + op.Status.CurrentTask.ID = "running-task" + op.Status.CurrentTask.State = v1beta1.CruiseControlTaskInExecution + } + objects = append(objects, op) + } + kubeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objects...). + WithStatusSubresource(&v1alpha1.CruiseControlOperation{}).Build() + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + t.Fatal("first execution must not issue a direct GET") + return nil + }, + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := c.List(ctx, list, opts...); err != nil { + return err + } + operations := list.(*v1alpha1.CruiseControlOperationList) + items := operations.Items + operations.Items = nil + for _, name := range []string{"z-first", "unrelated", "a-second", "request"} { + for _, op := range items { + if op.Name == name { + operations.Items = append(operations.Items, op) + } + } + } + return nil + }, + }) + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(t)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: true, + }}, nil) + var taskIDs []any + var tasks []*scale.Result + if stopping { + taskIDs = []any{"running-task"} + tasks = []*scale.Result{{TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}} + } + scaler.EXPECT().UserTasks(gomock.Any(), taskIDs...).Return(tasks, nil) + result := &scale.Result{ + TaskID: "submitted", State: v1beta1.CruiseControlTaskInExecution, + StartedAt: "Mon, 02 Jan 2006 15:04:05 GMT", + } + if stopping { + scaler.EXPECT().StopExecution(gomock.Any()).Return(result, nil) + } else { + scaler.EXPECT().AddBrokersWithParams(gomock.Any(), map[string]string{"brokerid": "z-first"}).Return(result, nil) + } + r := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: directClient, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { return scaler, nil }, + } + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKey{Name: "request", Namespace: "ns"}}) + require.NoError(t, err) + persisted := &v1alpha1.CruiseControlOperation{} + require.NoError(t, kubeClient.Get(ctx, client.ObjectKey{Name: "z-first", Namespace: "ns"}, persisted)) + assert.Equal(t, "submitted", persisted.CurrentTaskID()) + }) + } +} + +func BenchmarkCruiseControlUnrelatedHistory(b *testing.B) { + for _, unrelatedClusters := range []int{0, 10, 100} { + b.Run(fmt.Sprintf("unrelated-clusters=%d_history-each=10", unrelatedClusters), func(b *testing.B) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(b, v1alpha1.AddToScheme(scheme)) + require.NoError(b, v1beta1.AddToScheme(scheme)) + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "target", Namespace: "ns"}} + operation := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: "active", Namespace: "ns", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: cluster.Name}, + Finalizers: []string{ccOperationFinalizerGroup}, + }, + Status: v1alpha1.CruiseControlOperationStatus{ + CurrentTask: &v1alpha1.CruiseControlTask{ + ID: "running-task", Operation: v1alpha1.OperationAddBroker, State: v1beta1.CruiseControlTaskInExecution, + }, + }, + } + objects := []client.Object{cluster, operation} + for clusterIndex := 0; clusterIndex <= unrelatedClusters; clusterIndex++ { + clusterName := cluster.Name + if clusterIndex != 0 { + clusterName = fmt.Sprintf("unrelated-%03d", clusterIndex) + } + for history := 0; history < 10; history++ { + completed := operation.DeepCopy() + completed.Name = fmt.Sprintf("%s-history-%02d", clusterName, history) + completed.Labels = map[string]string{v1beta1.KafkaCRLabelKey: clusterName} + completed.Finalizers = nil + completed.Status.CurrentTask.State = v1beta1.CruiseControlTaskCompleted + objects = append(objects, completed) + } + } + var lists, listedObjects, gets, writes int + kubeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(objects...).WithStatusSubresource(operation). + WithInterceptorFuncs(interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + lists++ + if err := c.List(ctx, list, opts...); err != nil { + return err + } + listedObjects += len(list.(*v1alpha1.CruiseControlOperationList).Items) + return nil + }, + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + gets++ + return c.Get(ctx, key, obj, opts...) + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + writes++ + return c.Update(ctx, obj, opts...) + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + writes++ + return c.SubResource(subresource).Update(ctx, obj, opts...) + }, + }).Build() + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(b)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: false, + }}, nil).Times(b.N) + scaler.EXPECT().UserTasks(gomock.Any(), "running-task").Return([]*scale.Result{ + {TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}, + }, nil).Times(b.N) + var directGets int + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + directGets++ + return c.Get(ctx, key, obj, opts...) + }, + }) + reconciler := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: directClient, Scheme: scheme, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { + return scaler, nil + }, + } + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(operation)} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + result, err := reconciler.Reconcile(ctx, request) + if err != nil { + b.Fatal(err) + } + if result.RequeueAfter != time.Duration(defaultRequeueIntervalInSeconds)*time.Second { + b.Fatalf("unexpected scheduling result: %+v", result) + } + } + b.StopTimer() + require.Equal(b, b.N, lists) + require.Equal(b, b.N*11, listedObjects) + require.Equal(b, b.N*2, gets) + require.Zero(b, directGets) + require.Zero(b, writes) + b.ReportMetric(float64(lists)/float64(b.N), "LIST/reconcile") + b.ReportMetric(float64(listedObjects)/float64(b.N), "objects/reconcile") + b.ReportMetric(float64(gets)/float64(b.N), "cachedGET/reconcile") + b.ReportMetric(float64(directGets)/float64(b.N), "directGET/reconcile") + b.ReportMetric(float64(writes)/float64(b.N), "writes/reconcile") + }) + } +} + func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { for _, tt := range []struct { name string @@ -53,10 +373,13 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { wantRetries int wantFailed int wantOp v1alpha1.CruiseControlTaskOperation + staleHint bool }{ {name: "first submission without conflict", wantOp: v1alpha1.OperationAddBroker}, {name: "retry without conflict preserves history once", retry: true, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, {name: "stop submission without conflict retains operation", deleting: true, wantOp: v1alpha1.OperationStopExecution}, + {name: "recreated operation uses fresh UID task and parameters", staleHint: true, wantOp: v1alpha1.OperationAddBroker}, + {name: "fresh deletion overrides stale cached task", deleting: true, staleHint: true, wantOp: v1alpha1.OperationStopExecution}, {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, @@ -174,8 +497,24 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { return c.Get(ctx, key, obj, opts...) }, }) + cachedClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + if op, ok := obj.(*v1alpha1.CruiseControlOperation); ok && tt.staleHint { + op.UID = "deleted-operation-uid" + op.DeletionTimestamp = nil + op.Status.CurrentTask = &v1alpha1.CruiseControlTask{ + ID: "stale-task", Operation: v1alpha1.OperationRemoveBroker, + Parameters: map[string]string{"brokerid": "stale"}, + } + } + return nil + }, + }) reconciler := &CruiseControlOperationReconciler{ - Client: kubeClient, DirectClient: directClient, Scheme: scheme, + Client: cachedClient, DirectClient: directClient, Scheme: scheme, ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { return scaler, nil }, diff --git a/controllers/tests/cruisecontroloperation_controller_test.go b/controllers/tests/cruisecontroloperation_controller_test.go index 49354f0f3..38a4f62f0 100644 --- a/controllers/tests/cruisecontroloperation_controller_test.go +++ b/controllers/tests/cruisecontroloperation_controller_test.go @@ -24,8 +24,12 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "go.uber.org/mock/gomock" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "github.com/banzaicloud/koperator/controllers" "github.com/banzaicloud/koperator/controllers/tests/mocks" corev1 "k8s.io/api/core/v1" @@ -43,6 +47,151 @@ const ( operationExtendedTimeoutDuration = 15 * time.Second // extended timeout for complex operations ) +type operationRetrievalHintClient struct { + client.Client + hint *v1alpha1.CruiseControlOperation + miss bool +} + +func (c *operationRetrievalHintClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if operation, ok := obj.(*v1alpha1.CruiseControlOperation); ok { + if c.miss { + return apierrors.NewNotFound(schema.GroupResource{Group: "kafka.banzaicloud.io", Resource: "cruisecontroloperations"}, key.Name) + } + if c.hint != nil { + c.hint.DeepCopyInto(operation) + return nil + } + } + return c.Client.Get(ctx, key, obj, opts...) +} + +type operationRetrievalReader struct { + client.Reader + selectors []string + results []v1alpha1.CruiseControlOperationList + gets int +} + +func (r *operationRetrievalReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + r.gets++ + return r.Reader.Get(ctx, key, obj, opts...) +} + +func (r *operationRetrievalReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + options := (&client.ListOptions{}).ApplyOptions(opts) + selector := "" + if options.LabelSelector != nil { + selector = options.LabelSelector.String() + } + r.selectors = append(r.selectors, selector) + if err := r.Reader.List(ctx, list, opts...); err != nil { + return err + } + r.results = append(r.results, *list.(*v1alpha1.CruiseControlOperationList).DeepCopy()) + return nil +} + +var _ = Describe("CruiseControlOperation scoped retrieval", Serial, func() { + DescribeTable("uses fresh API objects and preserves compatible fallback", func(ctx SpecContext, mode string) { + namespace := &corev1.Namespace{ObjectMeta: metav1.ObjectMeta{GenerateName: "operation-retrieval-"}} + Expect(k8sClient.Create(ctx, namespace)).To(Succeed()) + DeferCleanup(func(ctx SpecContext) { + Expect(k8sClient.DeleteAllOf(ctx, &v1alpha1.CruiseControlOperation{}, client.InNamespace(namespace.Name))).To(Succeed()) + Expect(k8sClient.Delete(ctx, namespace)).To(Succeed()) + }) + var current *v1alpha1.CruiseControlOperation + for _, name := range []string{"z-current", "a-peer", "unrelated"} { + cluster := "target" + if name == "unrelated" { + cluster = "old" + } + operation := generateCruiseControlOperation(name, namespace.Name, cluster) + Expect(k8sClient.Create(ctx, &operation)).To(Succeed()) + operation.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationStatus} + Expect(k8sClient.Status().Update(ctx, &operation)).To(Succeed()) + if name == "z-current" { + current = operation.DeepCopy() + } + } + cached := &operationRetrievalHintClient{Client: cruiseControlOperationReconciler.Client} + Eventually(ctx, func(g Gomega) { + observed := &v1alpha1.CruiseControlOperation{} + g.Expect(cached.Get(ctx, client.ObjectKeyFromObject(current), observed)).To(Succeed()) + g.Expect(observed.CurrentTaskOperation()).To(Equal(v1alpha1.OperationStatus)) + }).Should(Succeed()) + switch mode { + case "miss": + cached.miss = true + case "stale": + cached.hint = current.DeepCopy() + cached.hint.Labels[v1beta1.KafkaCRLabelKey] = "old" + case "unlabeled-hint": + cached.hint = current.DeepCopy() + cached.hint.Labels = nil + case "legacy": + current.Labels = nil + Expect(k8sClient.Update(ctx, current)).To(Succeed()) + current.Status.CurrentTask.Operation = v1alpha1.OperationAddBroker + Expect(k8sClient.Status().Update(ctx, current)).To(Succeed()) + cached.hint = current.DeepCopy() + case "recreated": + cached.hint = current.DeepCopy() + Expect(k8sClient.Delete(ctx, current)).To(Succeed()) + replacement := generateCruiseControlOperation(current.Name, namespace.Name, "target") + Expect(k8sClient.Create(ctx, &replacement)).To(Succeed()) + replacement.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationStatus} + Expect(k8sClient.Status().Update(ctx, &replacement)).To(Succeed()) + Expect(replacement.UID).NotTo(Equal(current.UID)) + current = replacement.DeepCopy() + } + baseline := &v1alpha1.CruiseControlOperationList{} + Expect(k8sClient.List(ctx, baseline, client.InNamespace(namespace.Name))).To(Succeed()) + reader := &operationRetrievalReader{Reader: k8sClient} + reconciler := &controllers.CruiseControlOperationReconciler{Client: cached, DirectClient: reader} + result, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(current)}) + if mode == "legacy" { + Expect(err).To(MatchError(ContainSubstring("could not find kafka cluster reference label"))) + } else { + Expect(err).NotTo(HaveOccurred()) + Expect(result.RequeueAfter).To(BeZero()) + } + scopedSelector := v1beta1.KafkaCRLabelKey + "=target" + switch mode { + case "miss", "unlabeled-hint", "legacy": + Expect(reader.selectors).To(Equal([]string{""})) + case "stale": + Expect(reader.selectors).To(Equal([]string{v1beta1.KafkaCRLabelKey + "=old", ""})) + default: + Expect(reader.selectors).To(Equal([]string{scopedSelector})) + } + expected := make([]string, 0) + for _, operation := range baseline.Items { + if len(reader.selectors) != 1 || reader.selectors[0] == "" || operation.GetClusterRef() == "target" { + expected = append(expected, operation.Name) + } + } + last := reader.results[len(reader.results)-1] + observed := make([]string, 0, len(last.Items)) + for _, operation := range last.Items { + observed = append(observed, operation.Name) + if operation.Name == current.Name { + Expect(operation.UID).To(Equal(current.UID)) + Expect(operation.Status).To(Equal(current.Status)) + } + } + Expect(observed).To(Equal(expected)) + Expect(reader.gets).To(BeZero()) + }, + Entry("warm cache", "cached"), + Entry("cache miss", "miss"), + Entry("reassigned label", "stale"), + Entry("unlabeled hint", "unlabeled-hint"), + Entry("unlabeled legacy object", "legacy"), + Entry("deleted and recreated object", "recreated"), + ) +}) + var _ = Describe("CruiseControlTaskReconciler", func() { var ( count uint64 = 0