diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index d7da19d39..0bf53b9e8 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -243,24 +243,41 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques } } - conflictRetryFunction := func() error { - if err = updateResult(log, cruseControlTaskResult, ccOperationExecution, true); err != nil { - return err - } - err = r.Status().Update(ctx, ccOperationExecution) - if apiErrors.IsConflict(err) { - err = r.Get(ctx, client.ObjectKey{Name: ccOperationExecution.GetName(), Namespace: ccOperationExecution.GetNamespace()}, ccOperationExecution) - } - return err - } - // Protection to avoid re-execution task when status update is failed - if err := util.RetryOnConflict(util.DefaultBackOffForConflict, conflictRetryFunction); err != nil { + // Retry persistence without resubmitting the external task. + if err := r.persistExecutionResult(ctx, log, ccOperationExecution, cruseControlTaskResult); err != nil { return requeueWithError(log, "could not update the result of the Cruise Control user task execution to the CruiseControlOperation status", err) } return reconciled() } +func (r *CruiseControlOperationReconciler) persistExecutionResult( + ctx context.Context, + log logr.Logger, + operation *banzaiv1alpha1.CruiseControlOperation, + result *scale.Result, +) error { + firstAttempt := true + return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { + latest := &banzaiv1alpha1.CruiseControlOperation{} + if firstAttempt { + latest = operation.DeepCopy() + } else if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + return err + } + firstAttempt = false + if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { + return errors.New("CruiseControlOperation identity or current task changed after external task submission") + } + // Preserve the stop operation selected in memory during finalization. + latest.Status.CurrentTask.Operation = operation.CurrentTaskOperation() + if err := updateResult(log, result, latest, true); err != nil { + return err + } + return r.Status().Update(ctx, latest) + }) +} + func (r *CruiseControlOperationReconciler) addFinalizer(ctx context.Context, currentCCOperation *banzaiv1alpha1.CruiseControlOperation) error { // examine DeletionTimestamp to determine if object is under deletion if currentCCOperation.DeletionTimestamp.IsZero() { diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index d9a68fd87..770ffa962 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -17,16 +17,22 @@ package controllers import ( "context" + "errors" "testing" "time" "github.com/go-logr/logr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" @@ -34,6 +40,183 @@ import ( "github.com/banzaicloud/koperator/pkg/scale" ) +func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { + for _, tt := range []struct { + name string + retry bool + deleting bool + replaced bool + taskRemoved bool + taskChanged bool + exhausted bool + conflicts int + wantRetries int + wantFailed int + wantOp v1alpha1.CruiseControlTaskOperation + }{ + {name: "first submission without conflict", wantOp: v1alpha1.OperationAddBroker}, + {name: "retry without conflict preserves history once", retry: true, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission without conflict retains operation", deleting: true, wantOp: v1alpha1.OperationStopExecution}, + {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, + {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, + {name: "replacement object rejects stale result", replaced: true, conflicts: 1}, + {name: "removed task rejects stale result", taskRemoved: true, conflicts: 1}, + {name: "newer task rejects stale result", taskChanged: true, conflicts: 1}, + {name: "exhausted conflicts return error", exhausted: true, conflicts: 1000}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := v1beta1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "cluster", Namespace: "ns"}} + operation := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: "operation", Namespace: "ns", UID: "operation-uid", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: "cluster"}, + }, + Spec: v1alpha1.CruiseControlOperationSpec{ErrorPolicy: v1alpha1.ErrorPolicyRetry}, + } + operation.Status.ErrorPolicy = v1alpha1.ErrorPolicyRetry + operation.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationAddBroker} + taskResult := &scale.Result{ + TaskID: "submitted-task", State: v1beta1.CruiseControlTaskInExecution, + StartedAt: "Mon, 02 Jan 2006 15:04:05 GMT", ResponseStatusCode: 202, + RequestURL: "http://cruise-control/add_broker", + } + var previous []*scale.Result + if tt.retry { + operation.Status.CurrentTask.ID = "failed-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskCompletedWithError + operation.Status.CurrentTask.Finished = &v1.Time{Time: time.Date(2006, 1, 2, 15, 5, 0, 0, time.UTC)} + operation.Status.RetryCount = 2 + operation.Status.FailedTasks = []v1alpha1.CruiseControlTask{{ID: "earlier-failed-task"}} + previous = []*scale.Result{{TaskID: "failed-task", State: v1beta1.CruiseControlTaskCompletedWithError}} + } + if tt.deleting { + operation.DeletionTimestamp = &v1.Time{Time: time.Now()} + operation.Finalizers = []string{ccOperationFinalizerGroup} + operation.Status.CurrentTask.ID = "running-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskInExecution + previous = []*scale.Result{{TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}} + } + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(t)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: true, + }}, nil) + var taskIDs []any + if len(previous) != 0 { + taskIDs = []any{previous[0].TaskID} + } + scaler.EXPECT().UserTasks(gomock.Any(), taskIDs...).Return(previous, nil) + if tt.deleting { + scaler.EXPECT().StopExecution(gomock.Any()).Return(taskResult, nil).Times(1) + } else { + scaler.EXPECT().AddBrokersWithParams(gomock.Any(), operation.CurrentTaskParameters()).Return(taskResult, nil).Times(1) + } + var resultWrites int + kubeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(cluster, operation). + WithStatusSubresource(operation). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if op, ok := obj.(*v1alpha1.CruiseControlOperation); ok && subresource == "status" && op.CurrentTaskID() == "submitted-task" { + resultWrites++ + if resultWrites <= tt.conflicts { + latest := &v1alpha1.CruiseControlOperation{} + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), latest); err != nil { + return err + } + latest.Annotations = map[string]string{"other-writer": "preserved"} + if err := c.Update(ctx, latest); err != nil { + return err + } + if tt.replaced { + latest.Finalizers = nil + if err := c.Update(ctx, latest); err != nil { + return err + } + if err := c.Delete(ctx, latest); err != nil { + return err + } + latest.ResourceVersion = "" + latest.UID = "replacement-uid" + if err := c.Create(ctx, latest); err != nil { + return err + } + } + if tt.taskRemoved || tt.taskChanged { + if tt.taskRemoved { + latest.Status.CurrentTask = nil + } else { + latest.Status.CurrentTask.ID = "newer-task" + } + if err := c.Status().Update(ctx, latest); err != nil { + return err + } + } + return apierrors.NewConflict(schema.GroupResource{Group: "kafka.banzaicloud.io", Resource: "cruisecontroloperations"}, obj.GetName(), errors.New("injected concurrent update")) + } + } + return c.SubResource(subresource).Update(ctx, obj, opts...) + }, + }).Build() + var directReads int + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + directReads++ + assert.Positive(t, resultWrites, "direct GET must follow a conflicting result write") + return c.Get(ctx, key, obj, opts...) + }, + }) + reconciler := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: directClient, Scheme: scheme, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { + return scaler, nil + }, + } + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(operation)}) + if err != nil && !tt.replaced && !tt.taskRemoved && !tt.taskChanged && !tt.exhausted { + t.Fatal(err) + } + persisted := &v1alpha1.CruiseControlOperation{} + if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(operation), persisted); err != nil { + t.Fatal(err) + } + if tt.replaced || tt.taskRemoved || tt.taskChanged || tt.exhausted { + if err == nil { + t.Error("reported success without persisting the result to the original task") + } + if persisted.CurrentTaskID() == "submitted-task" { + t.Error("attached stale external result to a replaced operation or removed task") + } + return + } + assert.Equal(t, "submitted-task", persisted.CurrentTaskID()) + assert.Equal(t, v1beta1.CruiseControlTaskInExecution, persisted.CurrentTaskState()) + assert.Equal(t, tt.wantOp, persisted.CurrentTaskOperation()) + task := persisted.CurrentTask() + require.NotNil(t, task) + assert.Equal(t, taskResult.RequestURL, task.HTTPRequest) + assert.Equal(t, &taskResult.ResponseStatusCode, task.HTTPResponseCode) + require.NotNil(t, task.Started) + assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) + assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) + assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) + if tt.conflicts > 0 { + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + } + assert.Equal(t, tt.conflicts+1, resultWrites) + assert.Equal(t, tt.conflicts, directReads) + }) + } +} + func createCCRetryExecutionOperation(createTime time.Time, id string, operation v1alpha1.CruiseControlTaskOperation) *v1alpha1.CruiseControlOperation { return &v1alpha1.CruiseControlOperation{ ObjectMeta: v1.ObjectMeta{ diff --git a/controllers/kafkauser_controller.go b/controllers/kafkauser_controller.go index 54c67774d..6372e6f05 100644 --- a/controllers/kafkauser_controller.go +++ b/controllers/kafkauser_controller.go @@ -31,6 +31,7 @@ import ( certv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "github.com/go-logr/logr" certsigningreqv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -63,6 +64,13 @@ func SetupKafkaUserWithManager(mgr ctrl.Manager, certSigningEnabled bool, certMa WithEventFilter(SkipClusterRegistryOwnedResourcePredicate{}). Named("KafkaUser") if certSigningEnabled { + // Self-generated Secret events can race the CSR annotation cache; only deletion needs recovery. + builder.Owns(&corev1.Secret{}, ctrlBuilder.WithPredicates(predicate.Funcs{ + CreateFunc: func(event.CreateEvent) bool { return false }, + UpdateFunc: func(event.UpdateEvent) bool { return false }, + DeleteFunc: func(event.DeleteEvent) bool { return true }, + GenericFunc: func(event.GenericEvent) bool { return false }, + })) csrMapper := csrMapper{ client: mgr.GetClient(), log: log, diff --git a/controllers/tests/kafkauser_controller_test.go b/controllers/tests/kafkauser_controller_test.go index be3293c9a..3ea11b0ba 100644 --- a/controllers/tests/kafkauser_controller_test.go +++ b/controllers/tests/kafkauser_controller_test.go @@ -16,7 +16,9 @@ package tests import ( + "context" "fmt" + "sync" "sync/atomic" "time" @@ -32,12 +34,56 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" "github.com/banzaicloud/koperator/pkg/util" + pkicommon "github.com/banzaicloud/koperator/pkg/util/pki" ) +type laggingUserSecretClient struct { + client.Client + mu sync.Mutex + key types.NamespacedName + snapshot *corev1.Secret +} + +func (c *laggingUserSecretClient) holdSecretUpdates(key types.NamespacedName) { + c.mu.Lock() + defer c.mu.Unlock() + c.key = key + c.snapshot = nil +} + +func (c *laggingUserSecretClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if secret, ok := obj.(*corev1.Secret); ok { + c.mu.Lock() + if key == c.key && c.snapshot != nil { + *secret = *c.snapshot.DeepCopy() + c.mu.Unlock() + return nil + } + c.mu.Unlock() + } + return c.Client.Get(ctx, key, obj, opts...) +} + +func (c *laggingUserSecretClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if err := c.Client.Create(ctx, obj, opts...); err != nil { + return err + } + if secret, ok := obj.(*corev1.Secret); ok { + c.mu.Lock() + defer c.mu.Unlock() + if client.ObjectKeyFromObject(secret) == c.key { + // Model an informer that has observed creation but not the subsequent CSR annotation update. + c.snapshot = secret.DeepCopy() + } + } + return nil +} + var _ = Describe("KafkaTopic", func() { var ( count uint64 = 0 @@ -241,6 +287,24 @@ var _ = Describe("KafkaTopic", func() { }, }, } + secretKey := types.NamespacedName{Name: user.Spec.SecretName, Namespace: user.Namespace} + kafkaUserClient.holdSecretUpdates(secretKey) + DeferCleanup(func() { + kafkaUserClient.holdSecretUpdates(types.NamespacedName{}) + }) + userCSRNames := func() ([]string, error) { + requests := &certsigningreqv1.CertificateSigningRequestList{} + if err := k8sClient.List(ctx, requests); err != nil { + return nil, err + } + var names []string + for _, request := range requests.Items { + if request.Annotations[pkicommon.KafkaUserAnnotationName] == client.ObjectKeyFromObject(&user).String() { + names = append(names, request.Name) + } + } + return names, nil + } err := k8sClient.Create(ctx, &user) Expect(err).NotTo(HaveOccurred()) @@ -256,6 +320,9 @@ var _ = Describe("KafkaTopic", func() { }, 5*time.Second, 100*time.Millisecond).Should(HaveKey(k8scsrpki.DependingCsrAnnotation)) csrName := secret.Annotations[k8scsrpki.DependingCsrAnnotation] + By("not creating duplicate CSRs from Secret self-writes while its annotation cache lags") + Consistently(ctx, userCSRNames, 2*time.Second, 100*time.Millisecond).Should(ConsistOf(csrName)) + kafkaUserClient.holdSecretUpdates(types.NamespacedName{}) csr := &certsigningreqv1.CertificateSigningRequest{} Eventually(ctx, func() error { err := k8sClient.Get(ctx, types.NamespacedName{ @@ -361,5 +428,47 @@ x0eepFeUNacaeg7O1ftIrzNlYsSLi2Qm+tnu7odyxafZ65GJ9lcSLUqXuNDCrNOl } return user.Status.State, nil }, 5*time.Second, 100*time.Millisecond).Should(Equal(v1alpha1.UserStateCreated)) + + By("waiting for outstanding certificate retries to drain after readiness") + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: user.Namespace, + Name: user.Name, + }, &user)).To(Succeed()) + readyVersion := user.ResourceVersion + Consistently(ctx, func() (string, error) { + current := &v1alpha1.KafkaUser{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: user.Namespace, + Name: user.Name, + }, current) + return current.ResourceVersion, err + }, 16*time.Second, 100*time.Millisecond).Should(Equal(readyVersion)) + + By("recovering an owned certificate Secret from its deletion event alone") + Expect(k8sClient.Get(ctx, types.NamespacedName{ + Namespace: user.Namespace, + Name: user.Spec.SecretName, + }, secret)).To(Succeed()) + originalUID := secret.UID + kafkaUserClient.holdSecretUpdates(secretKey) + Expect(k8sClient.Delete(ctx, secret)).To(Succeed()) + Eventually(ctx, func() (types.UID, error) { + recreated := &corev1.Secret{} + err := k8sClient.Get(ctx, types.NamespacedName{ + Namespace: user.Namespace, + Name: user.Spec.SecretName, + }, recreated) + return recreated.UID, err + }, 5*time.Second, 100*time.Millisecond).Should(And(Not(BeEmpty()), Not(Equal(originalUID)))) + Eventually(ctx, func() (string, error) { + err := k8sClient.Get(ctx, secretKey, secret) + return secret.Annotations[k8scsrpki.DependingCsrAnnotation], err + }, 5*time.Second, 100*time.Millisecond).Should(And(Not(BeEmpty()), Not(Equal(csrName)))) + recoveredCSRName := secret.Annotations[k8scsrpki.DependingCsrAnnotation] + recoveredCSR := &certsigningreqv1.CertificateSigningRequest{} + Expect(k8sClient.Get(ctx, types.NamespacedName{Name: recoveredCSRName}, recoveredCSR)).To(Succeed()) + Expect(recoveredCSR.Spec.Request).NotTo(BeEmpty()) + Consistently(ctx, userCSRNames, 2*time.Second, 100*time.Millisecond).Should(ConsistOf(csrName, recoveredCSRName)) + kafkaUserClient.holdSecretUpdates(types.NamespacedName{}) }) }) diff --git a/controllers/tests/suite_test.go b/controllers/tests/suite_test.go index 10ecaa739..b717f9f6a 100644 --- a/controllers/tests/suite_test.go +++ b/controllers/tests/suite_test.go @@ -69,6 +69,7 @@ import ( // http://onsi.github.io/ginkgo/ to learn more about Ginkgo. var k8sClient client.Client +var kafkaUserClient *laggingUserSecretClient var csrClient *csrclient.CertificatesV1Client var testEnv *envtest.Environment var mockKafkaClients map[types.NamespacedName]kafkaclient.KafkaClient @@ -175,8 +176,9 @@ var _ = BeforeSuite(func(ctx SpecContext) { Expect(err).NotTo(HaveOccurred()) // Create a new kafka user reconciler + kafkaUserClient = &laggingUserSecretClient{Client: mgr.GetClient()} kafkaUserReconciler := controllers.KafkaUserReconciler{ - Client: mgr.GetClient(), + Client: kafkaUserClient, Scheme: mgr.GetScheme(), } diff --git a/docs/monitoring.md b/docs/monitoring.md index b6f802f1c..8d917bade 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -37,4 +37,21 @@ sum by (verb) (rate(rest_client_rate_limiter_duration_seconds_sum[5m])) Record cluster size, resource counts, workload, test duration, leader changes, and error rates with each measurement. These metrics establish instrumentation, not evidence that an optimization improves performance. This change does not -alter metrics authentication, scraper routing, worker counts, or cache scope. \ No newline at end of file +alter metrics authentication, scraper routing, worker counts, or cache scope. + +## Cruise Control execution-result persistence + +After an external task is submitted, the operator retries conflicting status +writes using fresh API-server reads. These persistence retries do not submit +the external task again. They retain the task identity, response metadata, +retry history, and selected stop-operation type. A replaced operation, removed +current task, or different task identity produces an error rather than accepting +a stale result. + +This is not exactly-once execution. If all writes fail, or the process stops +after Cruise Control accepts a request but before its task ID is recorded, the +external task can outlive the recorded operation state. Recorded task IDs allow +subsequent reconciliation to query task progress. Automatic adoption of an +unrecorded submission is not implemented; inspect Cruise Control's task history +when investigating this window, and do not infer that a missing status means +the external operation never ran. \ No newline at end of file diff --git a/pkg/webhooks/kafkatopic_validator.go b/pkg/webhooks/kafkatopic_validator.go index d1e09e41a..02d6e6f78 100644 --- a/pkg/webhooks/kafkatopic_validator.go +++ b/pkg/webhooks/kafkatopic_validator.go @@ -55,8 +55,15 @@ func (s KafkaTopicValidator) ValidateCreate(ctx context.Context, kafkaTopic *ban return s.validate(ctx, kafkaTopic) } -func (s KafkaTopicValidator) ValidateUpdate(ctx context.Context, _, kafkaTopic *banzaicloudv1alpha1.KafkaTopic) (warnings admission.Warnings, err error) { - return s.validate(ctx, kafkaTopic) +func (s KafkaTopicValidator) ValidateUpdate(ctx context.Context, oldTopic, kafkaTopic *banzaicloudv1alpha1.KafkaTopic) (warnings admission.Warnings, err error) { + checkKafka := true + if oldTopic != nil { + oldManager, oldHasManager := oldTopic.Annotations[TopicManagedByAnnotationKey] + manager, hasManager := kafkaTopic.Annotations[TopicManagedByAnnotationKey] + checkKafka = !cmp.Equal(oldTopic.Spec, kafkaTopic.Spec) || + oldHasManager != hasManager || oldManager != manager + } + return s.validateWithKafkaCheck(ctx, kafkaTopic, checkKafka) } func (s KafkaTopicValidator) ValidateDelete(_ context.Context, _ *banzaicloudv1alpha1.KafkaTopic) (warnings admission.Warnings, err error) { @@ -64,9 +71,13 @@ func (s KafkaTopicValidator) ValidateDelete(_ context.Context, _ *banzaicloudv1a } func (s *KafkaTopicValidator) validate(ctx context.Context, kafkaTopic *banzaicloudv1alpha1.KafkaTopic) (warnings admission.Warnings, err error) { + return s.validateWithKafkaCheck(ctx, kafkaTopic, true) +} + +func (s *KafkaTopicValidator) validateWithKafkaCheck(ctx context.Context, kafkaTopic *banzaicloudv1alpha1.KafkaTopic, checkKafka bool) (warnings admission.Warnings, err error) { log := s.Log.WithValues("name", kafkaTopic.GetName(), "namespace", kafkaTopic.GetNamespace()) - fieldErrs, err := s.validateKafkaTopic(ctx, log, kafkaTopic) + fieldErrs, err := s.validateKafkaTopicWithKafkaCheck(ctx, log, kafkaTopic, checkKafka) if err != nil { log.Error(err, errorDuringValidationMsg) return nil, apierrors.NewInternalError(errors.WithMessage(err, errorDuringValidationMsg)) @@ -81,6 +92,10 @@ func (s *KafkaTopicValidator) validate(ctx context.Context, kafkaTopic *banzaicl } func (s *KafkaTopicValidator) validateKafkaTopic(ctx context.Context, log logr.Logger, topic *banzaicloudv1alpha1.KafkaTopic) (field.ErrorList, error) { + return s.validateKafkaTopicWithKafkaCheck(ctx, log, topic, true) +} + +func (s *KafkaTopicValidator) validateKafkaTopicWithKafkaCheck(ctx context.Context, log logr.Logger, topic *banzaicloudv1alpha1.KafkaTopic, checkKafka bool) (field.ErrorList, error) { var allErrs field.ErrorList var logMsg string // First check if the kafkatopic is valid @@ -136,6 +151,10 @@ func (s *KafkaTopicValidator) validateKafkaTopic(ctx context.Context, log logr.L allErrs = append(allErrs, fieldErr) } + if !checkKafka { + return allErrs, nil + } + fieldErrList, err := s.checkKafka(ctx, topic, cluster) if err != nil { return nil, err diff --git a/pkg/webhooks/kafkatopic_validator_test.go b/pkg/webhooks/kafkatopic_validator_test.go index 328cbd4c4..8d43cda14 100644 --- a/pkg/webhooks/kafkatopic_validator_test.go +++ b/pkg/webhooks/kafkatopic_validator_test.go @@ -17,9 +17,13 @@ package webhooks import ( "context" + "errors" + "fmt" "strings" "testing" + "github.com/IBM/sarama" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -91,6 +95,215 @@ func newMockClients(cluster *v1beta1.KafkaCluster) (runtimeClient.WithWatch, kaf return client, kafkaClient, returnMockedKafkaClient } +type admissionCountingKafkaClient struct { + kafkaclient.KafkaClient + topicReads int +} + +func (c *admissionCountingKafkaClient) GetTopic(name string) (*sarama.TopicDetail, error) { + c.topicReads++ + return c.KafkaClient.GetTopic(name) +} + +func TestKafkaTopicValidateUpdateDependencies(t *testing.T) { + testCases := []struct { + name string + change func(*v1alpha1.KafkaTopic) + validationRequired bool + withoutManager bool + }{ + {name: "finalizer addition", change: func(topic *v1alpha1.KafkaTopic) { + topic.Finalizers = append(topic.Finalizers, "example.com/cleanup") + }}, + {name: "finalizer removal", change: func(topic *v1alpha1.KafkaTopic) { + topic.Finalizers = nil + }}, + {name: "irrelevant label", change: func(topic *v1alpha1.KafkaTopic) { + topic.Labels = map[string]string{"example.com/team": "streaming"} + }}, + {name: "irrelevant annotation", change: func(topic *v1alpha1.KafkaTopic) { + topic.Annotations["example.com/note"] = "updated" + }}, + {name: "partition change", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Spec.Partitions++ + }}, + {name: "config change", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Spec.Config = map[string]string{"retention.ms": "1000"} + }}, + {name: "cluster namespace default", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Spec.ClusterRef.Namespace = "" + }}, + {name: "managedBy change", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Annotations[TopicManagedByAnnotationKey] = "external" + }}, + {name: "managedBy removal", validationRequired: true, change: func(topic *v1alpha1.KafkaTopic) { + delete(topic.Annotations, TopicManagedByAnnotationKey) + }}, + {name: "managedBy addition", validationRequired: true, withoutManager: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Annotations[TopicManagedByAnnotationKey] = TopicManagedByKoperatorAnnotationValue + }}, + {name: "empty managedBy addition", validationRequired: true, withoutManager: true, change: func(topic *v1alpha1.KafkaTopic) { + topic.Annotations[TopicManagedByAnnotationKey] = "" + }}, + } + for _, unavailable := range []bool{false, true} { + mode := "available" + if unavailable { + mode = "unavailable" + } + t.Run(mode, func(t *testing.T) { + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + cluster := newMockCluster() + client, broker, _ := newMockClients(cluster) + if err := client.Create(ctx, cluster); err != nil { + t.Fatal(err) + } + oldTopic := newMockTopic() + oldTopic.Spec.Partitions = 2 + oldTopic.Spec.ReplicationFactor = 1 + oldTopic.Finalizers = []string{"finalizer.kafkatopics.kafka.banzaicloud.io"} + oldTopic.Annotations = map[string]string{TopicManagedByAnnotationKey: TopicManagedByKoperatorAnnotationValue} + if tc.withoutManager { + delete(oldTopic.Annotations, TopicManagedByAnnotationKey) + } + if err := client.Create(ctx, oldTopic); err != nil { + t.Fatal(err) + } + newTopic := oldTopic.DeepCopy() + tc.change(newTopic) + countingBroker := &admissionCountingKafkaClient{KafkaClient: broker} + connections, closes := 0, 0 + validator := KafkaTopicValidator{ + Client: client, + Log: logr.Discard(), + NewKafkaFromCluster: func(runtimeClient.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + connections++ + if unavailable { + return nil, nil, errors.New("Kafka unavailable") + } + return countingBroker, func() { closes++ }, nil + }, + } + _, err := validator.ValidateUpdate(ctx, oldTopic, newTopic) + wantError := unavailable && tc.validationRequired + if (err != nil) != wantError { + t.Errorf("admission error = %v, want error = %v", err, wantError) + } + wantConnections, wantReads := 0, 0 + if tc.validationRequired { + wantConnections = 1 + if !unavailable { + wantReads = 1 + } + } + if connections != wantConnections || countingBroker.topicReads != wantReads || closes != wantReads { + t.Errorf("Kafka connections/GetTopic/closes = %d/%d/%d, want %d/%d/%d", + connections, countingBroker.topicReads, closes, wantConnections, wantReads, wantReads) + } + }) + } + }) + } +} + +func TestKafkaTopicValidateUpdateRetainsKubernetesValidation(t *testing.T) { + for _, invalidInput := range []string{"partitions", "missing cluster", "duplicate topic"} { + t.Run(invalidInput, func(t *testing.T) { + ctx := context.Background() + cluster := newMockCluster() + client, _, _ := newMockClients(cluster) + oldTopic := newMockTopic() + oldTopic.Spec.Partitions = 2 + oldTopic.Spec.ReplicationFactor = 1 + if invalidInput == "partitions" { + oldTopic.Spec.Partitions = 0 + } + if invalidInput != "missing cluster" { + if err := client.Create(ctx, cluster); err != nil { + t.Fatal(err) + } + } + if invalidInput == "duplicate topic" { + duplicate := oldTopic.DeepCopy() + duplicate.Name = "duplicate" + if err := client.Create(ctx, duplicate); err != nil { + t.Fatal(err) + } + } + if err := client.Create(ctx, oldTopic); err != nil { + t.Fatal(err) + } + newTopic := oldTopic.DeepCopy() + newTopic.Labels = map[string]string{"example.com/team": "streaming"} + connections := 0 + validator := KafkaTopicValidator{ + Client: client, + Log: logr.Discard(), + NewKafkaFromCluster: func(runtimeClient.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + connections++ + return nil, nil, errors.New("Kafka unavailable") + }, + } + _, err := validator.ValidateUpdate(ctx, oldTopic, newTopic) + if !apierrors.IsInvalid(err) { + t.Errorf("admission error = %v, want Kubernetes validation error", err) + } + if connections != 0 { + t.Errorf("Kafka connections = %d, want 0", connections) + } + }) + } +} + +func TestKafkaTopicValidateCreateAdoptionDependencies(t *testing.T) { + for _, unavailable := range []bool{false, true} { + for _, adopt := range []bool{false, true} { + name := fmt.Sprintf("unavailable=%t/adopt=%t", unavailable, adopt) + t.Run(name, func(t *testing.T) { + ctx := context.Background() + cluster := newMockCluster() + client, broker, _ := newMockClients(cluster) + if err := client.Create(ctx, cluster); err != nil { + t.Fatal(err) + } + topic := newMockTopic() + topic.Spec.Partitions = 2 + topic.Spec.ReplicationFactor = 1 + if adopt { + topic.Annotations = map[string]string{TopicManagedByAnnotationKey: TopicManagedByKoperatorAnnotationValue} + } + if err := broker.CreateTopic(&kafkaclient.CreateTopicOptions{ + Name: topic.Spec.Name, Partitions: 2, ReplicationFactor: 1, + }); err != nil { + t.Fatal(err) + } + connections := 0 + validator := KafkaTopicValidator{ + Client: client, + Log: logr.Discard(), + NewKafkaFromCluster: func(runtimeClient.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + connections++ + if unavailable { + return nil, nil, errors.New("Kafka unavailable") + } + return broker, func() {}, nil + }, + } + _, err := validator.ValidateCreate(ctx, topic) + wantError := unavailable || !adopt + if (err != nil) != wantError { + t.Errorf("adoption error = %v, want error = %v", err, wantError) + } + if connections != 1 { + t.Errorf("Kafka connections = %d, want 1", connections) + } + }) + } + } +} + func TestCheckKafkaTopicExist(t *testing.T) { cluster := newMockCluster() client, kafkaClient, returnMockedKafkaClient := newMockClients(cluster)