From da63dc3e3e39dc7a24b08810a3bf3ebf731abffc Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Mon, 16 Nov 2026 16:59:17 +0100 Subject: [PATCH 1/5] fix(controllers): persist Cruise Control results through status conflicts Retry fresh API reads and status writes without repeating external submission. Preserve stop-operation identity and retry history, reject stale object/task identities, add seven regression cases, and document the remaining submit-before-persist crash window. Refs #369. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cruisecontroloperation_controller.go | 37 ++-- .../cruisecontroloperation_controller_test.go | 169 ++++++++++++++++++ docs/monitoring.md | 19 +- 3 files changed, 212 insertions(+), 13 deletions(-) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index d7da19d39..68a343bb2 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -243,24 +243,37 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques } } - conflictRetryFunction := func() error { - if err = updateResult(log, cruseControlTaskResult, ccOperationExecution, true); err != nil { - return err - } - err = r.Status().Update(ctx, ccOperationExecution) - if apiErrors.IsConflict(err) { - err = r.Get(ctx, client.ObjectKey{Name: ccOperationExecution.GetName(), Namespace: ccOperationExecution.GetNamespace()}, ccOperationExecution) - } - return err - } - // Protection to avoid re-execution task when status update is failed - if err := util.RetryOnConflict(util.DefaultBackOffForConflict, conflictRetryFunction); err != nil { + // Retry persistence without resubmitting the external task. + if err := r.persistExecutionResult(ctx, log, ccOperationExecution, cruseControlTaskResult); err != nil { return requeueWithError(log, "could not update the result of the Cruise Control user task execution to the CruiseControlOperation status", err) } return reconciled() } +func (r *CruiseControlOperationReconciler) persistExecutionResult( + ctx context.Context, + log logr.Logger, + operation *banzaiv1alpha1.CruiseControlOperation, + result *scale.Result, +) error { + return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { + latest := &banzaiv1alpha1.CruiseControlOperation{} + if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + return err + } + if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { + return errors.New("CruiseControlOperation identity or current task changed after external task submission") + } + // Preserve the stop operation selected in memory during finalization. + latest.Status.CurrentTask.Operation = operation.CurrentTaskOperation() + if err := updateResult(log, result, latest, true); err != nil { + return err + } + return r.Status().Update(ctx, latest) + }) +} + func (r *CruiseControlOperationReconciler) addFinalizer(ctx context.Context, currentCCOperation *banzaiv1alpha1.CruiseControlOperation) error { // examine DeletionTimestamp to determine if object is under deletion if currentCCOperation.DeletionTimestamp.IsZero() { diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index d9a68fd87..0c25b5c50 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -17,16 +17,22 @@ package controllers import ( "context" + "errors" "testing" "time" "github.com/go-logr/logr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" @@ -34,6 +40,169 @@ import ( "github.com/banzaicloud/koperator/pkg/scale" ) +func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { + for _, tt := range []struct { + name string + retry bool + deleting bool + replaced bool + taskRemoved bool + taskChanged bool + exhausted bool + conflicts int + wantRetries int + wantFailed int + wantOp v1alpha1.CruiseControlTaskOperation + }{ + {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, + {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, + {name: "replacement object rejects stale result", replaced: true, conflicts: 1}, + {name: "removed task rejects stale result", taskRemoved: true, conflicts: 1}, + {name: "newer task rejects stale result", taskChanged: true, conflicts: 1}, + {name: "exhausted conflicts return error", exhausted: true, conflicts: 1000}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := v1beta1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "cluster", Namespace: "ns"}} + operation := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: "operation", Namespace: "ns", UID: "operation-uid", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: "cluster"}, + }, + Spec: v1alpha1.CruiseControlOperationSpec{ErrorPolicy: v1alpha1.ErrorPolicyRetry}, + } + operation.Status.ErrorPolicy = v1alpha1.ErrorPolicyRetry + operation.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationAddBroker} + taskResult := &scale.Result{ + TaskID: "submitted-task", State: v1beta1.CruiseControlTaskInExecution, + StartedAt: "Mon, 02 Jan 2006 15:04:05 GMT", ResponseStatusCode: 202, + RequestURL: "http://cruise-control/add_broker", + } + var previous []*scale.Result + if tt.retry { + operation.Status.CurrentTask.ID = "failed-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskCompletedWithError + operation.Status.CurrentTask.Finished = &v1.Time{Time: time.Date(2006, 1, 2, 15, 5, 0, 0, time.UTC)} + operation.Status.RetryCount = 2 + operation.Status.FailedTasks = []v1alpha1.CruiseControlTask{{ID: "earlier-failed-task"}} + previous = []*scale.Result{{TaskID: "failed-task", State: v1beta1.CruiseControlTaskCompletedWithError}} + } + if tt.deleting { + operation.DeletionTimestamp = &v1.Time{Time: time.Now()} + operation.Finalizers = []string{ccOperationFinalizerGroup} + operation.Status.CurrentTask.ID = "running-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskInExecution + previous = []*scale.Result{{TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}} + } + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(t)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: true, + }}, nil) + var taskIDs []any + if len(previous) != 0 { + taskIDs = []any{previous[0].TaskID} + } + scaler.EXPECT().UserTasks(gomock.Any(), taskIDs...).Return(previous, nil) + if tt.deleting { + scaler.EXPECT().StopExecution(gomock.Any()).Return(taskResult, nil).Times(1) + } else { + scaler.EXPECT().AddBrokersWithParams(gomock.Any(), operation.CurrentTaskParameters()).Return(taskResult, nil).Times(1) + } + var resultWrites int + kubeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(cluster, operation). + WithStatusSubresource(operation). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if op, ok := obj.(*v1alpha1.CruiseControlOperation); ok && subresource == "status" && op.CurrentTaskID() == "submitted-task" { + resultWrites++ + if resultWrites <= tt.conflicts { + latest := &v1alpha1.CruiseControlOperation{} + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), latest); err != nil { + return err + } + latest.Annotations = map[string]string{"other-writer": "preserved"} + if err := c.Update(ctx, latest); err != nil { + return err + } + if tt.replaced { + latest.Finalizers = nil + if err := c.Update(ctx, latest); err != nil { + return err + } + if err := c.Delete(ctx, latest); err != nil { + return err + } + latest.ResourceVersion = "" + latest.UID = "replacement-uid" + if err := c.Create(ctx, latest); err != nil { + return err + } + } + if tt.taskRemoved || tt.taskChanged { + if tt.taskRemoved { + latest.Status.CurrentTask = nil + } else { + latest.Status.CurrentTask.ID = "newer-task" + } + if err := c.Status().Update(ctx, latest); err != nil { + return err + } + } + return apierrors.NewConflict(schema.GroupResource{Group: "kafka.banzaicloud.io", Resource: "cruisecontroloperations"}, obj.GetName(), errors.New("injected concurrent update")) + } + } + return c.SubResource(subresource).Update(ctx, obj, opts...) + }, + }).Build() + reconciler := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: kubeClient, Scheme: scheme, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { + return scaler, nil + }, + } + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(operation)}) + if err != nil && !tt.replaced && !tt.taskRemoved && !tt.taskChanged && !tt.exhausted { + t.Fatal(err) + } + persisted := &v1alpha1.CruiseControlOperation{} + if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(operation), persisted); err != nil { + t.Fatal(err) + } + if tt.replaced || tt.taskRemoved || tt.taskChanged || tt.exhausted { + if err == nil { + t.Error("reported success without persisting the result to the original task") + } + if persisted.CurrentTaskID() == "submitted-task" { + t.Error("attached stale external result to a replaced operation or removed task") + } + return + } + assert.Equal(t, "submitted-task", persisted.CurrentTaskID()) + assert.Equal(t, v1beta1.CruiseControlTaskInExecution, persisted.CurrentTaskState()) + assert.Equal(t, tt.wantOp, persisted.CurrentTaskOperation()) + task := persisted.CurrentTask() + require.NotNil(t, task) + assert.Equal(t, taskResult.RequestURL, task.HTTPRequest) + assert.Equal(t, &taskResult.ResponseStatusCode, task.HTTPResponseCode) + require.NotNil(t, task.Started) + assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) + assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) + assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + assert.Equal(t, tt.conflicts+1, resultWrites) + }) + } +} + func createCCRetryExecutionOperation(createTime time.Time, id string, operation v1alpha1.CruiseControlTaskOperation) *v1alpha1.CruiseControlOperation { return &v1alpha1.CruiseControlOperation{ ObjectMeta: v1.ObjectMeta{ diff --git a/docs/monitoring.md b/docs/monitoring.md index b6f802f1c..8d917bade 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -37,4 +37,21 @@ sum by (verb) (rate(rest_client_rate_limiter_duration_seconds_sum[5m])) Record cluster size, resource counts, workload, test duration, leader changes, and error rates with each measurement. These metrics establish instrumentation, not evidence that an optimization improves performance. This change does not -alter metrics authentication, scraper routing, worker counts, or cache scope. \ No newline at end of file +alter metrics authentication, scraper routing, worker counts, or cache scope. + +## Cruise Control execution-result persistence + +After an external task is submitted, the operator retries conflicting status +writes using fresh API-server reads. These persistence retries do not submit +the external task again. They retain the task identity, response metadata, +retry history, and selected stop-operation type. A replaced operation, removed +current task, or different task identity produces an error rather than accepting +a stale result. + +This is not exactly-once execution. If all writes fail, or the process stops +after Cruise Control accepts a request but before its task ID is recorded, the +external task can outlive the recorded operation state. Recorded task IDs allow +subsequent reconciliation to query task progress. Automatic adoption of an +unrecorded submission is not implemented; inspect Cruise Control's task history +when investigating this window, and do not infer that a missing status means +the external operation never ran. \ No newline at end of file From f022f61913658b17b705a737c545c368e36fc831 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 08:37:01 +0100 Subject: [PATCH 2/5] perf(controllers): refresh Cruise Control results only on conflict Reuse the selected operation for the first status write and read directly only when retrying a conflict. Cover conflict-free submission, retry, and stop paths. Refs #369. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../cruisecontroloperation_controller.go | 6 +++++- .../cruisecontroloperation_controller_test.go | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index 68a343bb2..0bf53b9e8 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -257,11 +257,15 @@ func (r *CruiseControlOperationReconciler) persistExecutionResult( operation *banzaiv1alpha1.CruiseControlOperation, result *scale.Result, ) error { + firstAttempt := true return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { latest := &banzaiv1alpha1.CruiseControlOperation{} - if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + if firstAttempt { + latest = operation.DeepCopy() + } else if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { return err } + firstAttempt = false if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { return errors.New("CruiseControlOperation identity or current task changed after external task submission") } diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index 0c25b5c50..770ffa962 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -54,6 +54,9 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { wantFailed int wantOp v1alpha1.CruiseControlTaskOperation }{ + {name: "first submission without conflict", wantOp: v1alpha1.OperationAddBroker}, + {name: "retry without conflict preserves history once", retry: true, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission without conflict retains operation", deleting: true, wantOp: v1alpha1.OperationStopExecution}, {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, @@ -163,8 +166,16 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { return c.SubResource(subresource).Update(ctx, obj, opts...) }, }).Build() + var directReads int + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + directReads++ + assert.Positive(t, resultWrites, "direct GET must follow a conflicting result write") + return c.Get(ctx, key, obj, opts...) + }, + }) reconciler := &CruiseControlOperationReconciler{ - Client: kubeClient, DirectClient: kubeClient, Scheme: scheme, + Client: kubeClient, DirectClient: directClient, Scheme: scheme, ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { return scaler, nil }, @@ -197,8 +208,11 @@ func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) - assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + if tt.conflicts > 0 { + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + } assert.Equal(t, tt.conflicts+1, resultWrites) + assert.Equal(t, tt.conflicts, directReads) }) } } From b7659097606ea4f6f637a2516170006cbdf793a2 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 09:31:03 +0100 Subject: [PATCH 3/5] fix: preserve lifecycle cleanup ownership and bound topic deletion (#391) Persist finalizers and discoverable cleanup identity before provisioning. Preserve certificate and ACL cleanup ownership across interrupted provisioning. Submit Kafka topic deletion without waiting, retain finalizers while pending, and observe absence on a subsequent reconcile. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- controllers/finalizer_ordering_test.go | 616 ++++++++++++++++++ controllers/kafkacluster_controller.go | 14 +- controllers/kafkatopic_controller.go | 89 +-- controllers/kafkauser_controller.go | 126 +++- controllers/tests/suite_test.go | 2 +- docs/monitoring.md | 32 +- main.go | 10 +- pkg/pki/certmanagerpki/certmanager_user.go | 56 +- .../certmanagerpki/user_finalization_test.go | 81 +++ 9 files changed, 940 insertions(+), 86 deletions(-) create mode 100644 controllers/finalizer_ordering_test.go create mode 100644 pkg/pki/certmanagerpki/user_finalization_test.go diff --git a/controllers/finalizer_ordering_test.go b/controllers/finalizer_ordering_test.go new file mode 100644 index 000000000..915d27a84 --- /dev/null +++ b/controllers/finalizer_ordering_test.go @@ -0,0 +1,616 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "math/big" + "slices" + "sync" + "testing" + "time" + + certv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8sscheme "k8s.io/client-go/kubernetes/scheme" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/banzaicloud/koperator/api/v1alpha1" + "github.com/banzaicloud/koperator/api/v1beta1" + "github.com/banzaicloud/koperator/pkg/kafkaclient" + "github.com/banzaicloud/koperator/pkg/util" + certutil "github.com/banzaicloud/koperator/pkg/util/cert" + pkicommon "github.com/banzaicloud/koperator/pkg/util/pki" + "github.com/banzaicloud/koperator/pkg/webhooks" +) + +type protectedKafkaClient struct { + kafkaclient.KafkaClient + beforeMutation func() + afterMutationErr error + deletedUsers []string +} + +func (c *protectedKafkaClient) CreateTopic(opts *kafkaclient.CreateTopicOptions) error { + c.beforeMutation() + if err := c.KafkaClient.CreateTopic(opts); err != nil { + return err + } + return c.afterMutationErr +} + +func (c *protectedKafkaClient) CreateUserACLs(access v1alpha1.KafkaAccessType, pattern v1alpha1.KafkaPatternType, user, topic string) error { + c.beforeMutation() + if err := c.KafkaClient.CreateUserACLs(access, pattern, user, topic); err != nil { + return err + } + return c.afterMutationErr +} + +func (c *protectedKafkaClient) DeleteUserACLs(user string, pattern v1alpha1.KafkaPatternType) error { + c.deletedUsers = append(c.deletedUsers, user) + return c.KafkaClient.DeleteUserACLs(user, pattern) +} + +func TestFinalizersPrecedeProvisioning(t *testing.T) { + for _, kind := range []string{"cluster", "topic", "user-acls", "user-certificate"} { + for _, scenario := range []string{"first-reconcile", "failed-finalizer-write", "delete-during-finalizer-write", "restart-after-side-effect"} { + t.Run(kind+"/"+scenario, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, k8sscheme.AddToScheme(scheme)) + require.NoError(t, certv1.AddToScheme(scheme)) + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + + cluster := createTestKafkaCluster("protected-cluster", "default") + var primary client.Object + var finalizers []string + switch kind { + case "cluster": + primary = cluster + finalizers = []string{clusterFinalizer, clusterTopicsFinalizer, clusterUsersFinalizer} + case "topic": + primary = &v1alpha1.KafkaTopic{ + ObjectMeta: metav1.ObjectMeta{Name: "protected-topic", Namespace: cluster.Namespace}, + Spec: v1alpha1.KafkaTopicSpec{ + Name: "protected-topic", Partitions: 1, ReplicationFactor: 1, + ClusterRef: v1alpha1.ClusterReference{Name: cluster.Name}, + }, + } + finalizers = []string{topicFinalizer} + default: + primary = &v1alpha1.KafkaUser{ + ObjectMeta: metav1.ObjectMeta{Name: "protected-user", Namespace: cluster.Namespace}, + Spec: v1alpha1.KafkaUserSpec{ + SecretName: "protected-user", + ClusterRef: v1alpha1.ClusterReference{Name: cluster.Name}, + CreateCert: util.BoolPointer(kind == "user-certificate"), + }, + } + if kind == "user-certificate" { + cluster.Spec.ListenersConfig.SSLSecrets = &v1beta1.SSLSecrets{PKIBackend: v1beta1.PKIBackendCertManager} + } else { + primary.(*v1alpha1.KafkaUser).Spec.TopicGrants = []v1alpha1.UserTopicGrant{ + {TopicName: "protected-topic", AccessType: v1alpha1.KafkaAccessTypeRead}, + } + } + finalizers = []string{userFinalizer} + } + primary.SetFinalizers([]string{"example.com/unrelated"}) + objects := []client.Object{primary} + if kind != "cluster" { + objects = append(objects, cluster) + } + base := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(objects...).WithStatusSubresource(primary, cluster).Build() + sentinel := errors.New("injected finalizer write failure") + interrupted := errors.New("injected interruption after side effect") + mutations := 0 + guard := func() { + t.Helper() + mutations++ + persisted := primary.DeepCopyObject().(client.Object) + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(primary), persisted)) + for _, f := range finalizers { + require.Contains(t, persisted.GetFinalizers(), f, "side effect must have durable cleanup ownership") + } + require.Contains(t, persisted.GetFinalizers(), "example.com/unrelated") + if kind != "cluster" { + require.Equal(t, "protected-cluster.default", persisted.GetLabels()[clusterRefLabel], + "cluster deletion must be able to discover partially provisioned children") + } + require.Nil(t, persisted.GetDeletionTimestamp(), "must not provision a deleting resource") + require.NotEqual(t, "failed-finalizer-write", scenario) + require.NotEqual(t, "delete-during-finalizer-write", scenario) + } + deleted := false + wrapped := interceptor.NewClient(base, interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if client.ObjectKeyFromObject(obj) == client.ObjectKeyFromObject(primary) && + slices.Contains(obj.GetFinalizers(), finalizers[0]) { + if scenario == "failed-finalizer-write" { + return sentinel + } + if scenario == "delete-during-finalizer-write" && !deleted { + deleted = true + require.NoError(t, c.Delete(ctx, primary.DeepCopyObject().(client.Object))) + } + } + return c.Update(ctx, obj, opts...) + }, + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + guard() + err := c.Create(ctx, obj, opts...) + if err == nil && scenario == "restart-after-side-effect" { + return interrupted + } + return err + }, + }) + broker, closeBroker, err := kafkaclient.NewMockFromCluster(base, cluster) + require.NoError(t, err) + t.Cleanup(closeBroker) + protectedBroker := &protectedKafkaClient{KafkaClient: broker, beforeMutation: guard} + if scenario == "restart-after-side-effect" { + protectedBroker.afterMutationErr = interrupted + } + SetNewKafkaFromCluster(func(client.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + return protectedBroker, func() {}, nil + }) + t.Cleanup(func() { SetNewKafkaFromCluster(kafkaclient.NewFromCluster) }) + + newReconciler := func(c client.Client) reconcile.Reconciler { + switch kind { + case "cluster": + return &KafkaClusterReconciler{Client: c, DirectClient: base, KafkaClientProvider: kafkaclient.NewMockProvider()} + case "topic": + return &KafkaTopicReconciler{Client: c, Scheme: scheme} + default: + return &KafkaUserReconciler{Client: c, Scheme: scheme} + } + } + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(primary)} + _, err = newReconciler(wrapped).Reconcile(ctx, req) + switch scenario { + case "failed-finalizer-write": + require.ErrorIs(t, err, sentinel) + require.Zero(t, mutations) + case "delete-during-finalizer-write": + require.Error(t, err, "deletion must invalidate the in-flight metadata update") + require.Zero(t, mutations) + default: + require.Positive(t, mutations, "the first reconcile must continue after persisting finalizers") + persisted := primary.DeepCopyObject().(client.Object) + require.NoError(t, base.Get(ctx, req.NamespacedName, persisted)) + for _, f := range finalizers { + require.Contains(t, persisted.GetFinalizers(), f) + } + if scenario == "restart-after-side-effect" { + require.ErrorIs(t, err, interrupted) + require.NoError(t, base.Delete(ctx, persisted)) + result, err := newReconciler(base).Reconcile(ctx, req) + require.NoError(t, err) + if kind == "topic" { + require.Positive(t, result.RequeueAfter) + require.NoError(t, base.Get(ctx, req.NamespacedName, persisted)) + require.Contains(t, persisted.GetFinalizers(), topicFinalizer) + _, err = newReconciler(base).Reconcile(ctx, req) + require.NoError(t, err) + } + require.NoError(t, base.Get(ctx, req.NamespacedName, persisted), "unrelated finalizer retains the object") + for _, f := range finalizers { + require.NotContains(t, persisted.GetFinalizers(), f) + } + if kind == "topic" { + topic, err := broker.GetTopic("protected-topic") + require.NoError(t, err) + require.Nil(t, topic) + } + if kind == "user-acls" { + acls, err := broker.ListUserACLs() + require.NoError(t, err) + require.Empty(t, acls) + } + } + } + }) + } + } +} + +func TestUnmanagedTopicDoesNotAcquireCleanupOwnership(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + topic := &v1alpha1.KafkaTopic{ + ObjectMeta: metav1.ObjectMeta{ + Name: "external-topic", Namespace: "default", + Annotations: map[string]string{webhooks.TopicManagedByAnnotationKey: "external"}, + }, + Spec: v1alpha1.KafkaTopicSpec{ClusterRef: v1alpha1.ClusterReference{Name: "cluster"}}, + } + cluster := &v1beta1.KafkaCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "default"}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(topic, cluster).WithStatusSubresource(topic).Build() + SetNewKafkaFromCluster(func(client.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + t.Fatal("unmanaged topic must not require Kafka availability") + return nil, nil, nil + }) + t.Cleanup(func() { SetNewKafkaFromCluster(kafkaclient.NewFromCluster) }) + _, err := (&KafkaTopicReconciler{Client: c}).Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(topic)}) + require.NoError(t, err) + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(topic), topic)) + require.Empty(t, topic.Finalizers) + require.Equal(t, "external", topic.Status.ManagedBy) +} + +func TestDeletingUserDoesNotProvisionCertificate(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, k8sscheme.AddToScheme(scheme)) + require.NoError(t, certv1.AddToScheme(scheme)) + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + for _, withGrants := range []bool{false, true} { + t.Run(map[bool]string{false: "no-grants", true: "unsigned-certificate"}[withGrants], func(t *testing.T) { + user := &v1alpha1.KafkaUser{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pending-user", Namespace: "default", Finalizers: []string{userFinalizer}, + DeletionTimestamp: &metav1.Time{Time: metav1.Now().Time}, + }, + Spec: v1alpha1.KafkaUserSpec{ + SecretName: "pending-secret", ClusterRef: v1alpha1.ClusterReference{Name: "cluster"}, + }, + } + if withGrants { + user.Annotations = map[string]string{userACLPrincipalAnnotation: ""} + user.Spec.TopicGrants = []v1alpha1.UserTopicGrant{{TopicName: "topic", AccessType: v1alpha1.KafkaAccessTypeRead}} + } + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "default"}, + Spec: v1beta1.KafkaClusterSpec{ListenersConfig: v1beta1.ListenersConfig{ + SSLSecrets: &v1beta1.SSLSecrets{PKIBackend: v1beta1.PKIBackendCertManager}, + }}, + } + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "pending-secret", Namespace: "default"}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(user, cluster, secret).Build() + guarded := interceptor.NewClient(c, interceptor.Funcs{ + Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { + t.Fatal("deletion must not provision PKI resources") + return nil + }, + }) + _, err := (&KafkaUserReconciler{Client: guarded, Scheme: scheme}).Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(user)}) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(c.Get(ctx, client.ObjectKeyFromObject(user), user))) + }) + } +} + +func TestDeletingUserUsesExistingCertificateIdentity(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, k8sscheme.AddToScheme(scheme)) + require.NoError(t, certv1.AddToScheme(scheme)) + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "issued-principal", Organization: []string{"existing-organization"}}, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, publicKey, privateKey) + require.NoError(t, err) + + user := &v1alpha1.KafkaUser{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deleting-user", Namespace: "default", + Finalizers: []string{userFinalizer}, DeletionTimestamp: &metav1.Time{Time: metav1.Now().Time}, + }, + Spec: v1alpha1.KafkaUserSpec{ + SecretName: "existing-secret", ClusterRef: v1alpha1.ClusterReference{Name: "cluster"}, + TopicGrants: []v1alpha1.UserTopicGrant{{TopicName: "topic", AccessType: v1alpha1.KafkaAccessTypeRead}}, + }, + } + cluster := &v1beta1.KafkaCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "default"}} + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "existing-secret", Namespace: "default"}, + Data: map[string][]byte{corev1.TLSCertKey: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(user, cluster, secret).Build() + broker, closeBroker, err := kafkaclient.NewMockFromCluster(c, cluster) + require.NoError(t, err) + t.Cleanup(closeBroker) + protectedBroker := &protectedKafkaClient{KafkaClient: broker} + SetNewKafkaFromCluster(func(client.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + return protectedBroker, func() {}, nil + }) + t.Cleanup(func() { SetNewKafkaFromCluster(kafkaclient.NewFromCluster) }) + _, err = (&KafkaUserReconciler{Client: c, Scheme: scheme}).Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(user)}) + require.NoError(t, err) + require.Equal(t, []string{"CN=issued-principal,O=existing-organization"}, protectedBroker.deletedUsers) + require.True(t, apierrors.IsNotFound(c.Get(ctx, client.ObjectKeyFromObject(user), user))) +} + +func TestDeletionRetainsFinalizerOnClusterReadFailure(t *testing.T) { + for _, kind := range []string{"topic", "user"} { + t.Run(kind, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + meta := metav1.ObjectMeta{ + Name: "deleting", Namespace: "default", DeletionTimestamp: &metav1.Time{Time: metav1.Now().Time}, + } + var obj client.Object + if kind == "topic" { + meta.Finalizers = []string{topicFinalizer} + obj = &v1alpha1.KafkaTopic{ObjectMeta: meta, Spec: v1alpha1.KafkaTopicSpec{ + ClusterRef: v1alpha1.ClusterReference{Name: "cluster"}, + }} + } else { + meta.Finalizers = []string{userFinalizer} + obj = &v1alpha1.KafkaUser{ObjectMeta: meta, Spec: v1alpha1.KafkaUserSpec{ + ClusterRef: v1alpha1.ClusterReference{Name: "cluster"}, + }} + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(obj).Build() + readFailure := errors.New("temporary cluster read failure") + wrapped := interceptor.NewClient(c, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, object client.Object, opts ...client.GetOption) error { + if _, ok := object.(*v1beta1.KafkaCluster); ok { + return readFailure + } + return c.Get(ctx, key, object, opts...) + }, + }) + var r reconcile.Reconciler = &KafkaTopicReconciler{Client: wrapped} + if kind == "user" { + r = &KafkaUserReconciler{Client: wrapped} + } + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(obj)}) + require.ErrorIs(t, err, readFailure) + current := obj.DeepCopyObject().(client.Object) + require.NoError(t, c.Get(ctx, client.ObjectKeyFromObject(obj), current)) + require.Equal(t, meta.Finalizers, current.GetFinalizers()) + }) + } +} + +func TestDeletingUserAfterProvisioningFailsBeforeSecretCreation(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, k8sscheme.AddToScheme(scheme)) + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + user := &v1alpha1.KafkaUser{ + ObjectMeta: metav1.ObjectMeta{Name: "unprovisioned", Namespace: "default"}, + Spec: v1alpha1.KafkaUserSpec{ + SecretName: "never-created", ClusterRef: v1alpha1.ClusterReference{Name: "cluster"}, + TopicGrants: []v1alpha1.UserTopicGrant{{TopicName: "topic", AccessType: v1alpha1.KafkaAccessTypeRead}}, + }, + } + cluster := &v1beta1.KafkaCluster{ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "default"}} + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(user, cluster).WithStatusSubresource(user).Build() + r := &KafkaUserReconciler{Client: c, Scheme: scheme} + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(user)} + _, err := r.Reconcile(ctx, req) + require.Error(t, err, "cluster is not configured for certificate issuance") + require.NoError(t, c.Get(ctx, req.NamespacedName, user)) + require.Contains(t, user.Finalizers, userFinalizer) + require.NoError(t, c.Delete(ctx, user)) + _, err = (&KafkaUserReconciler{Client: c, Scheme: scheme}).Reconcile(ctx, req) + require.NoError(t, err) + require.True(t, apierrors.IsNotFound(c.Get(ctx, req.NamespacedName, user))) +} + +func TestDeletingUserAfterACLsCreatedAndStatusWriteFails(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, k8sscheme.AddToScheme(scheme)) + require.NoError(t, certv1.AddToScheme(scheme)) + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + user := &v1alpha1.KafkaUser{ + ObjectMeta: metav1.ObjectMeta{Name: "partial-user", Namespace: "default", UID: "partial-user-uid"}, + Spec: v1alpha1.KafkaUserSpec{ + SecretName: "partial-secret", ClusterRef: v1alpha1.ClusterReference{Name: "cluster"}, + TopicGrants: []v1alpha1.UserTopicGrant{{TopicName: "topic", AccessType: v1alpha1.KafkaAccessTypeRead}}, + }, + } + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "default"}, + Spec: v1beta1.KafkaClusterSpec{ListenersConfig: v1beta1.ListenersConfig{ + SSLSecrets: &v1beta1.SSLSecrets{PKIBackend: v1beta1.PKIBackendCertManager}, + }}, + } + certificatePEM, _, _, err := certutil.GenerateTestCert() + require.NoError(t, err) + certificate := &pkicommon.UserCertificate{Certificate: certificatePEM} + principal, err := certificate.GetDistinguishedName() + require.NoError(t, err) + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "partial-secret", Namespace: "default"}, + Data: map[string][]byte{ + corev1.TLSCertKey: certificatePEM, corev1.TLSPrivateKeyKey: []byte("key"), v1alpha1.CoreCACertKey: certificatePEM, + }, + } + cert := &certv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: user.Name, Namespace: user.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(user, v1alpha1.GroupVersion.WithKind("KafkaUser"))}, + }, + Spec: certv1.CertificateSpec{SecretName: user.Spec.SecretName}, + } + base := fake.NewClientBuilder().WithScheme(scheme).WithObjects(user, cluster, secret, cert).WithStatusSubresource(user).Build() + stale := false + writeFailure := errors.New("status write failed after ACL creation") + wrapped := interceptor.NewClient(base, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if err := c.Get(ctx, key, obj, opts...); err != nil { + return err + } + if s, ok := obj.(*corev1.Secret); ok && stale { + s.Data = nil + } + return nil + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if _, ok := obj.(*v1alpha1.KafkaUser); ok && stale { + return writeFailure + } + return c.SubResource(subresource).Update(ctx, obj, opts...) + }, + }) + broker, closeBroker, err := kafkaclient.NewMockFromCluster(base, cluster) + require.NoError(t, err) + t.Cleanup(closeBroker) + protectedBroker := &protectedKafkaClient{KafkaClient: broker, beforeMutation: func() { stale = true }} + SetNewKafkaFromCluster(func(client.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + return protectedBroker, func() {}, nil + }) + t.Cleanup(func() { SetNewKafkaFromCluster(kafkaclient.NewFromCluster) }) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(user)} + _, err = (&KafkaUserReconciler{Client: wrapped, Scheme: scheme}).Reconcile(ctx, req) + require.ErrorIs(t, err, writeFailure) + require.NoError(t, base.Get(ctx, req.NamespacedName, user)) + require.Empty(t, user.Status.ACLs) + require.NoError(t, base.Delete(ctx, user)) + _, err = (&KafkaUserReconciler{Client: wrapped, Scheme: scheme}).Reconcile(ctx, req) + require.NoError(t, err) + require.Equal(t, []string{principal}, protectedBroker.deletedUsers) +} + +type pendingTopicDeletionClient struct { + kafkaclient.KafkaClient + releaseWait <-chan struct{} + submitted chan<- string + deleteErr error +} + +func (c *pendingTopicDeletionClient) DeleteTopic(name string, wait bool) error { + c.submitted <- name + if c.deleteErr != nil { + return c.deleteErr + } + if wait { + <-c.releaseWait + } + return nil +} + +func TestDeletingTopicYieldsWhileDeletionPending(t *testing.T) { + for _, scenario := range []string{"stalled-wait", "accepted-but-still-visible", "deletion-error-then-retry"} { + t.Run(scenario, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + cluster := createTestKafkaCluster("pending-deletion-cluster", "default") + topic := &v1alpha1.KafkaTopic{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pending-topic", Namespace: cluster.Namespace, + Finalizers: []string{topicFinalizer, "example.com/unrelated"}, + }, + Spec: v1alpha1.KafkaTopicSpec{ + Name: "pending-topic", Partitions: 1, ReplicationFactor: 1, + ClusterRef: v1alpha1.ClusterReference{Name: cluster.Name}, + }, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(topic, cluster).Build() + require.NoError(t, c.Delete(ctx, topic)) + broker, closeBroker, err := kafkaclient.NewMockFromCluster(c, cluster) + require.NoError(t, err) + t.Cleanup(closeBroker) + require.NoError(t, broker.CreateTopic(&kafkaclient.CreateTopicOptions{ + Name: topic.Spec.Name, Partitions: 1, ReplicationFactor: 1, + })) + releaseWait := make(chan struct{}) + stopWaiting := sync.OnceFunc(func() { close(releaseWait) }) + t.Cleanup(stopWaiting) + if scenario != "stalled-wait" { + stopWaiting() + } + deletionFailure := errors.New("injected topic deletion failure") + var deleteErr error + if scenario == "deletion-error-then-retry" { + deleteErr = deletionFailure + } + submitted := make(chan string, 2) + SetNewKafkaFromCluster(func(client.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + return &pendingTopicDeletionClient{ + KafkaClient: broker, releaseWait: releaseWait, submitted: submitted, deleteErr: deleteErr, + }, func() {}, nil + }) + t.Cleanup(func() { SetNewKafkaFromCluster(kafkaclient.NewFromCluster) }) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(topic)} + type reconcileOutcome struct { + result ctrl.Result + err error + } + for attempt := 0; attempt < 2; attempt++ { + done := make(chan reconcileOutcome, 1) + var outcome reconcileOutcome + go func() { + result, err := (&KafkaTopicReconciler{Client: c, Scheme: scheme}).Reconcile(ctx, req) + done <- reconcileOutcome{result: result, err: err} + }() + select { + case outcome = <-done: + case <-time.After(time.Second): + stopWaiting() + <-done + t.Fatal("Reconcile held a worker while topic deletion remained pending") + } + require.NoError(t, c.Get(ctx, req.NamespacedName, topic)) + require.Contains(t, topic.Finalizers, topicFinalizer, "restart must retain cleanup ownership until observed absence") + require.Contains(t, topic.Finalizers, "example.com/unrelated") + if deleteErr != nil { + require.ErrorIs(t, outcome.err, deletionFailure) + } else { + require.NoError(t, outcome.err) + require.Positive(t, outcome.result.RequeueAfter, "pending deletion must schedule another observation") + } + require.Len(t, submitted, 1, "each pending reconcile must submit a bounded deletion request") + require.Equal(t, "pending-topic", <-submitted) + deleteErr = nil + } + + require.NoError(t, broker.DeleteTopic(topic.Spec.Name, false)) + result, err := (&KafkaTopicReconciler{Client: c, Scheme: scheme}).Reconcile(ctx, req) + require.NoError(t, err) + require.Zero(t, result.RequeueAfter) + require.NoError(t, c.Get(ctx, req.NamespacedName, topic)) + require.Equal(t, []string{"example.com/unrelated"}, topic.Finalizers) + require.Empty(t, submitted, "observed absence must not submit another deletion") + }) + } +} diff --git a/controllers/kafkacluster_controller.go b/controllers/kafkacluster_controller.go index 16e988253..0d44b427d 100644 --- a/controllers/kafkacluster_controller.go +++ b/controllers/kafkacluster_controller.go @@ -112,6 +112,10 @@ func (r *KafkaClusterReconciler) Reconcile(ctx context.Context, request ctrl.Req return r.checkFinalizers(ctx, instance) } + if instance, err = r.ensureFinalizers(ctx, instance); err != nil { + return requeueWithError(log, "failed to ensure finalizers on kafkacluster instance", err) + } + if instance.Status.State != v1beta1.KafkaClusterRollingUpgrading { if err := k8sutil.UpdateCRStatus(r.Client, instance, v1beta1.KafkaClusterReconciling, log); err != nil { return requeueWithError(log, err.Error(), err) @@ -183,11 +187,6 @@ func (r *KafkaClusterReconciler) Reconcile(ctx context.Context, request ctrl.Req } } - log.Info("ensuring finalizers on kafkacluster") - if instance, err = r.ensureFinalizers(ctx, instance); err != nil { - return requeueWithError(log, "failed to ensure finalizers on kafkacluster instance", err) - } - // Update rolling upgrade last successful state if instance.Status.State == v1beta1.KafkaClusterRollingUpgrading { if err := k8sutil.UpdateRollingUpgradeState(r.Client, instance, time.Now(), log); err != nil { @@ -332,11 +331,16 @@ func topicListToStrSlice(list v1alpha1.KafkaTopicList) []string { func (r *KafkaClusterReconciler) ensureFinalizers(ctx context.Context, cluster *v1beta1.KafkaCluster) (updated *v1beta1.KafkaCluster, err error) { finalizers := []string{clusterFinalizer, clusterTopicsFinalizer, clusterUsersFinalizer} + changed := false for _, finalizer := range finalizers { if apiutil.StringSliceContains(cluster.GetFinalizers(), finalizer) { continue } cluster.SetFinalizers(append(cluster.GetFinalizers(), finalizer)) + changed = true + } + if !changed { + return cluster, nil } return r.updateAndFetchLatest(ctx, cluster) } diff --git a/controllers/kafkatopic_controller.go b/controllers/kafkatopic_controller.go index 4a23c8c4f..ae0076175 100644 --- a/controllers/kafkatopic_controller.go +++ b/controllers/kafkatopic_controller.go @@ -21,6 +21,7 @@ import ( "fmt" "reflect" "strings" + "time" "github.com/go-logr/logr" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -98,7 +99,7 @@ func (r *KafkaTopicReconciler) Reconcile(ctx context.Context, request reconcile. var cluster *v1beta1.KafkaCluster if cluster, err = k8sutil.LookupKafkaCluster(ctx, r.Client, instance.Spec.ClusterRef.Name, clusterNamespace); err != nil { // This shouldn't trigger anymore, but leaving it here as a safetybelt - if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { + if apierrors.IsNotFound(err) && k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { reqLogger.Info("Cluster is already gone, there is nothing we can do") if err = r.removeFinalizer(ctx, instance); err != nil { return requeueWithError(reqLogger, "failed to remove finalizer", err) @@ -110,6 +111,21 @@ func (r *KafkaTopicReconciler) Reconcile(ctx context.Context, request reconcile. return requeueWithError(reqLogger, "failed to lookup referenced cluster", err) } + if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { + if !apiutil.StringSliceContains(instance.Finalizers, topicFinalizer) { + return reconciled() + } + if !isTopicManagedByKoperator(instance) { + return ctrl.Result{}, r.removeFinalizer(ctx, instance) + } + broker, close, err := newKafkaFromCluster(r.Client, cluster) + if err != nil { + return checkBrokerConnectionError(reqLogger, err) + } + defer close() + return r.checkFinalizers(ctx, broker, instance) + } + // Set managed status based on KafkaTopic managedBy annotation managedByStatus := webhooks.TopicManagedByKoperatorAnnotationValue if !isTopicManagedByKoperator(instance) { @@ -123,24 +139,28 @@ func (r *KafkaTopicReconciler) Reconcile(ctx context.Context, request reconcile. } } - // Get a kafka connection - broker, close, err := newKafkaFromCluster(r.Client, cluster) - if err != nil { - return checkBrokerConnectionError(reqLogger, err) - } - defer close() - - // Check if marked for deletion and if so run finalizers - if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { - return r.checkFinalizers(ctx, broker, instance) - } - // No need to do anything when the kafka topic is not managed by Koperator if !isTopicManagedByKoperator(instance) { reqLogger.Info(fmt.Sprintf("topic '%s' is not managed by %s it is managed by '%s' ==> nothing to reconcile here", instance.Spec.Name, webhooks.TopicManagedByKoperatorAnnotationValue, managedByStatus)) return reconciled() } + if instance, err = r.ensureClusterLabel(ctx, cluster, instance); err != nil { + return requeueWithError(reqLogger, "failed to ensure kafkacluster label on topic", err) + } + if !apiutil.StringSliceContains(instance.GetFinalizers(), topicFinalizer) { + instance.SetFinalizers(append(instance.GetFinalizers(), topicFinalizer)) + if instance, err = r.updateAndFetchLatest(ctx, instance); err != nil { + return requeueWithError(reqLogger, "failed to add Finalizer to KafkaTopic", err) + } + } + + broker, close, err := newKafkaFromCluster(r.Client, cluster) + if err != nil { + return checkBrokerConnectionError(reqLogger, err) + } + defer close() + // Check if the topic already exists existing, err := broker.GetTopic(instance.Spec.Name) if err != nil { @@ -177,20 +197,6 @@ func (r *KafkaTopicReconciler) Reconcile(ctx context.Context, request reconcile. return requeueWithError(reqLogger, "failed to create kafka topic", err) } - // ensure kafkaCluster label - if instance, err = r.ensureClusterLabel(ctx, cluster, instance); err != nil { - return requeueWithError(reqLogger, "failed to ensure kafkacluster label on topic", err) - } - - // ensure a finalizer for cleanup on deletion - if !apiutil.StringSliceContains(instance.GetFinalizers(), topicFinalizer) { - reqLogger.Info("Adding Finalizer for the KafkaTopic") - instance.SetFinalizers(append(instance.GetFinalizers(), topicFinalizer)) - if instance, err = r.updateAndFetchLatest(ctx, instance); err != nil { - return requeueWithError(reqLogger, "failed to add Finalizer to KafkaTopic", err) - } - } - // set topic status as created if instance.Status.State != v1alpha1.TopicStateCreated { instance.Status.State = v1alpha1.TopicStateCreated @@ -230,9 +236,13 @@ func (r *KafkaTopicReconciler) checkFinalizers(ctx context.Context, broker kafka if apiutil.StringSliceContains(topic.GetFinalizers(), topicFinalizer) { // Remove topic from Kafka cluster when it is managed by Koperator if isTopicManagedByKoperator(topic) { - if err = r.finalizeKafkaTopic(reqLogger, broker, topic); err != nil { + done, err := r.finalizeKafkaTopic(reqLogger, broker, topic) + if err != nil { return requeueWithError(reqLogger, "failed to finalize kafkatopic", err) } + if !done { + return ctrl.Result{RequeueAfter: 3 * time.Second}, nil + } } if err = r.removeFinalizer(ctx, topic); err != nil { return requeueWithError(reqLogger, "failed to remove finalizer from kafkatopic", err) @@ -247,22 +257,17 @@ func (r *KafkaTopicReconciler) removeFinalizer(ctx context.Context, topic *v1alp return err } -func (r *KafkaTopicReconciler) finalizeKafkaTopic(reqLogger logr.Logger, broker kafkaclient.KafkaClient, topic *v1alpha1.KafkaTopic) error { +func (r *KafkaTopicReconciler) finalizeKafkaTopic(reqLogger logr.Logger, broker kafkaclient.KafkaClient, topic *v1alpha1.KafkaTopic) (bool, error) { exists, err := broker.GetTopic(topic.Spec.Name) if err != nil { - return err + return false, err } - if exists != nil { - // DeleteTopic with wait to make sure it goes down fully in case of cluster - // deletion. - // TODO (tinyzimmer): Perhaps this should only wait when it's the cluster - // being deleted, and use false when it's just the topic itself. Also, - // if delete.topic.enable=false this may hang forever, so should maybe - // check if that's the case during a wait. - if err = broker.DeleteTopic(topic.Spec.Name, true); err != nil { - return err - } - reqLogger.Info("Deleted topic") + if exists == nil { + return true, nil + } + if err = broker.DeleteTopic(topic.Spec.Name, false); err != nil { + return false, err } - return nil + reqLogger.Info("Submitted topic deletion") + return false, nil } diff --git a/controllers/kafkauser_controller.go b/controllers/kafkauser_controller.go index 54c67774d..c02d5e38d 100644 --- a/controllers/kafkauser_controller.go +++ b/controllers/kafkauser_controller.go @@ -31,7 +31,9 @@ import ( certv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" "github.com/go-logr/logr" certsigningreqv1 "k8s.io/api/certificates/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -55,6 +57,8 @@ import ( var userFinalizer = "finalizer.kafkausers.kafka.banzaicloud.io" +const userACLPrincipalAnnotation = "kafka.banzaicloud.io/acl-principal" + // SetupKafkaUserWithManager registers KafkaUser controller to the manager func SetupKafkaUserWithManager(mgr ctrl.Manager, certSigningEnabled bool, certManagerEnabled bool) *ctrl.Builder { log := mgr.GetLogger() @@ -179,13 +183,14 @@ func (r *KafkaUserReconciler) Reconcile(ctx context.Context, request reconcile.R // Error reading the object - requeue the request. return requeueWithError(reqLogger, err.Error(), err) } + instance.SetGroupVersionKind(v1alpha1.GroupVersion.WithKind("KafkaUser")) // Get the referenced kafkacluster clusterNamespace := getClusterRefNamespace(instance.Namespace, instance.Spec.ClusterRef) var cluster *v1beta1.KafkaCluster if cluster, err = k8sutil.LookupKafkaCluster(ctx, r.Client, instance.Spec.ClusterRef.Name, clusterNamespace); err != nil { // This shouldn't trigger anymore, but leaving it here as a safetybelt - if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { + if apierrors.IsNotFound(err) && k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { reqLogger.Info("Cluster is gone already, there is nothing we can do") if err = r.removeFinalizer(ctx, instance); err != nil { return requeueWithError(reqLogger, "failed to remove finalizer from kafkauser", err) @@ -195,6 +200,17 @@ func (r *KafkaUserReconciler) Reconcile(ctx context.Context, request reconcile.R return requeueWithError(reqLogger, "failed to lookup referenced cluster", err) } + if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { + return r.finalizeUser(ctx, cluster, instance) + } + + if instance, err = r.ensureClusterLabel(ctx, cluster, instance); err != nil { + return requeueWithError(reqLogger, "failed to ensure kafkacluster label on user", err) + } + if instance, err = r.ensureUserFinalizer(ctx, instance); err != nil { + return requeueWithError(reqLogger, "failed to update kafkauser with finalizer", err) + } + var kafkaUser string if instance.Spec.GetIfCertShouldBeCreated() { @@ -235,13 +251,6 @@ func (r *KafkaUserReconciler) Reconcile(ctx context.Context, request reconcile.R RequeueAfter: time.Duration(15) * time.Second, }, nil default: - if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { - reqLogger.Info("Kafka user marked for deletion before creating certificates") - if err = r.removeFinalizer(ctx, instance); err != nil { - return requeueWithError(reqLogger, "failed to remove finalizer from kafkauser", err) - } - return reconciled() - } return requeueWithError(reqLogger, "failed to reconcile user secret", err) } } @@ -250,29 +259,18 @@ func (r *KafkaUserReconciler) Reconcile(ctx context.Context, request reconcile.R reqLogger.Error(err, "could not get Distinguished Name from the generated TLS certificate", "cert", string(user.Certificate)) return ctrl.Result{}, err } - // check if marked for deletion and remove created certs - if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { - reqLogger.Info("Kafka user is marked for deletion, revoking certificates") - if err = pkiManager.FinalizeUserCertificate(ctx, instance); err != nil { - return requeueWithError(reqLogger, "failed to finalize user certificate", err) - } - } } else { kafkaUser = fmt.Sprintf("CN=%s", instance.Name) } - // check if marked for deletion and remove kafka ACLs - if k8sutil.IsMarkedForDeletion(instance.ObjectMeta) { - return r.checkFinalizers(ctx, cluster, instance, kafkaUser) - } - - // ensure a kafkaCluster label - if instance, err = r.ensureClusterLabel(ctx, cluster, instance); err != nil { - return requeueWithError(reqLogger, "failed to ensure kafkacluster label on user", err) - } - // If topic grants supplied, grab a broker connection and set ACLs if len(instance.Spec.TopicGrants) > 0 { + if instance.Annotations[userACLPrincipalAnnotation] != kafkaUser { + metav1.SetMetaDataAnnotation(&instance.ObjectMeta, userACLPrincipalAnnotation, kafkaUser) + if instance, err = r.updateAndFetchLatest(ctx, instance); err != nil { + return requeueWithError(reqLogger, "failed to record ACL cleanup identity", err) + } + } broker, close, err := newKafkaFromCluster(r.Client, cluster) if err != nil { return checkBrokerConnectionError(reqLogger, err) @@ -289,14 +287,6 @@ func (r *KafkaUserReconciler) Reconcile(ctx context.Context, request reconcile.R } } - // ensure a finalizer for cleanup on deletion - if !apiutil.StringSliceContains(instance.GetFinalizers(), userFinalizer) { - r.addFinalizer(reqLogger, instance) - if instance, err = r.updateAndFetchLatest(ctx, instance); err != nil { - return requeueWithError(reqLogger, "failed to update kafkauser with finalizer", err) - } - } - // set user status instance.Status = v1alpha1.KafkaUserStatus{ State: v1alpha1.UserStateCreated, @@ -311,6 +301,76 @@ func (r *KafkaUserReconciler) Reconcile(ctx context.Context, request reconcile.R return reconciled() } +func (r *KafkaUserReconciler) finalizeUser(ctx context.Context, cluster *v1beta1.KafkaCluster, user *v1alpha1.KafkaUser) (reconcile.Result, error) { + if !apiutil.StringSliceContains(user.Finalizers, userFinalizer) { + return reconciled() + } + kafkaUser := fmt.Sprintf("CN=%s", user.Name) + if user.Spec.GetIfCertShouldBeCreated() { + if len(user.Spec.TopicGrants) > 0 && !k8sutil.IsMarkedForDeletion(cluster.ObjectMeta) { + var err error + kafkaUser, err = r.userACLPrincipal(ctx, user) + if err != nil { + return ctrl.Result{}, err + } + } + if cluster.Spec.ListenersConfig.SSLSecrets != nil || user.Spec.PKIBackendSpec != nil { + backend := v1beta1.PKIBackendProvided + if user.Spec.PKIBackendSpec != nil { + backend = v1beta1.PKIBackend(user.Spec.PKIBackendSpec.PKIBackend) + } + if err := pki.GetPKIManager(r.Client, cluster, backend).FinalizeUserCertificate(ctx, user); err != nil { + if errors.As(err, &errorfactory.ResourceNotReady{}) { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + return ctrl.Result{}, err + } + } + } + if kafkaUser == "" { + return ctrl.Result{}, r.removeFinalizer(ctx, user) + } + return r.checkFinalizers(ctx, cluster, user, kafkaUser) +} + +func (r *KafkaUserReconciler) userACLPrincipal(ctx context.Context, user *v1alpha1.KafkaUser) (string, error) { + if principal, recorded := user.Annotations[userACLPrincipalAnnotation]; recorded { + return principal, nil + } + secret := &corev1.Secret{} + if err := r.Client.Get(ctx, client.ObjectKey{Namespace: user.Namespace, Name: user.Spec.SecretName}, secret); err != nil { + return "", err + } + certificate := &pkicommon.UserCertificate{Certificate: secret.Data[corev1.TLSCertKey]} + return certificate.GetDistinguishedName() +} + +func (r *KafkaUserReconciler) ensureUserFinalizer(ctx context.Context, user *v1alpha1.KafkaUser) (*v1alpha1.KafkaUser, error) { + if apiutil.StringSliceContains(user.Finalizers, userFinalizer) { + return user, nil + } + if _, recorded := user.Annotations[userACLPrincipalAnnotation]; !recorded && user.Status.State == "" && len(user.Status.ACLs) == 0 { + principal := "" + if user.Spec.GetIfCertShouldBeCreated() && len(user.Spec.TopicGrants) > 0 { + secret := &corev1.Secret{} + err := r.Client.Get(ctx, client.ObjectKey{Namespace: user.Namespace, Name: user.Spec.SecretName}, secret) + if client.IgnoreNotFound(err) != nil { + return nil, err + } + if len(secret.Data[corev1.TLSCertKey]) > 0 { + certificate := &pkicommon.UserCertificate{Certificate: secret.Data[corev1.TLSCertKey]} + if principal, err = certificate.GetDistinguishedName(); err != nil { + return nil, err + } + } + } + // Empty records that no ACL submission has started; absence is legacy/unknown. + metav1.SetMetaDataAnnotation(&user.ObjectMeta, userACLPrincipalAnnotation, principal) + } + r.addFinalizer(logr.FromContextOrDiscard(ctx), user) + return r.updateAndFetchLatest(ctx, user) +} + func (r *KafkaUserReconciler) ensureClusterLabel(ctx context.Context, cluster *v1beta1.KafkaCluster, user *v1alpha1.KafkaUser) (*v1alpha1.KafkaUser, error) { labels := applyClusterRefLabel(cluster, user.GetLabels()) if !reflect.DeepEqual(labels, user.GetLabels()) { diff --git a/controllers/tests/suite_test.go b/controllers/tests/suite_test.go index 10ecaa739..0d993672e 100644 --- a/controllers/tests/suite_test.go +++ b/controllers/tests/suite_test.go @@ -176,7 +176,7 @@ var _ = BeforeSuite(func(ctx SpecContext) { // Create a new kafka user reconciler kafkaUserReconciler := controllers.KafkaUserReconciler{ - Client: mgr.GetClient(), + Client: k8sClient, Scheme: mgr.GetScheme(), } diff --git a/docs/monitoring.md b/docs/monitoring.md index 8d917bade..18925b00b 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -54,4 +54,34 @@ external task can outlive the recorded operation state. Recorded task IDs allow subsequent reconciliation to query task progress. Automatic adoption of an unrecorded submission is not implemented; inspect Cruise Control's task history when investigating this window, and do not infer that a missing status means -the external operation never ran. \ No newline at end of file +the external operation never ran. + +## Cleanup ownership during provisioning + +KafkaCluster, managed KafkaTopic, and KafkaUser resources acquire cleanup +finalizers before provisioning. A failed finalizer write prevents protected +side effects. Reconciliation continues after a successful write rather than +depending on a finalizer-only watch event. + +KafkaUser records `kafka.banzaicloud.io/acl-principal` before attempting ACL +creation. A nonempty value is the certificate-derived cleanup identity; an +empty value records that ACL submission has not started. An absent annotation +is legacy/unknown state, not evidence that no ACLs exist. Do not remove or edit +this annotation. KafkaUser uses an uncached Kubernetes client so certificate +and Secret absence used during initialization and cleanup is authoritative. +This adds direct reads; it is a correctness change, not an API-traffic +optimization. + +Deletion does not create certificates to discover a user's principal. +Recorded principals survive a failed status write or an incomplete Secret +cache. Legacy users without a recorded principal still require their existing +certificate for ACL cleanup; if it is missing or invalid, restore the cleanup +identity rather than removing the finalizer and abandoning ACLs. The annotation +does not recover external work whose identity was never recorded by an older +operator, and rolling back to an older operator stops recording these markers. + +For cert-manager users, cleanup stops the owned Certificate before deleting +user-owned Secrets. Regenerated Secrets without owner references are adopted +for cleanup only when they identify that owned Certificate. Shared or +externally owned Secrets are preserved. Cleanup retries while the Certificate +is still deleting; it does not wait synchronously for issuance or approval. \ No newline at end of file diff --git a/main.go b/main.go index a854dd863..22d01a304 100644 --- a/main.go +++ b/main.go @@ -48,6 +48,7 @@ import ( clientgoscheme "k8s.io/client-go/kubernetes/scheme" _ "k8s.io/client-go/plugin/pkg/client/auth/gcp" ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" contour "github.com/projectcontour/contour/apis/projectcontour/v1" @@ -207,9 +208,16 @@ func main() { os.Exit(1) } + // User cleanup makes absence-based decisions about certificates and Secrets. + userClient, err := client.New(mgr.GetConfig(), client.Options{Scheme: mgr.GetScheme()}) + if err != nil { + setupLog.Error(err, "unable to create direct KafkaUser client") + os.Exit(1) + } + // Create a new kafka user reconciler kafkaUserReconciler := &controllers.KafkaUserReconciler{ - Client: mgr.GetClient(), + Client: userClient, Scheme: mgr.GetScheme(), } diff --git a/pkg/pki/certmanagerpki/certmanager_user.go b/pkg/pki/certmanagerpki/certmanager_user.go index b693716ed..a3cffa875 100644 --- a/pkg/pki/certmanagerpki/certmanager_user.go +++ b/pkg/pki/certmanagerpki/certmanager_user.go @@ -36,11 +36,61 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" ) -// FinalizeUserCertificate for cert-manager backend auto returns because controller references handle cleanup -func (c *certManager) FinalizeUserCertificate(_ context.Context, _ *v1alpha1.KafkaUser) (err error) { - return +// FinalizeUserCertificate stops issuance before removing generated user Secrets. +func (c *certManager) FinalizeUserCertificate(ctx context.Context, user *v1alpha1.KafkaUser) error { + cert, err := c.getUserCertificate(ctx, user) + if apierrors.IsNotFound(err) { + return c.deleteOwnedUserSecret(ctx, user) + } + if err != nil { + return err + } + if !metav1.IsControlledBy(cert, user) { + return nil + } + secret, err := c.getUserSecretForCleanup(ctx, user) + if err != nil { + return err + } + if secret != nil && len(secret.OwnerReferences) == 0 && secret.Annotations[certv1.CertificateNameKey] == cert.Name { + secret.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(user, v1alpha1.GroupVersion.WithKind("KafkaUser"))} + if err = c.client.Update(ctx, secret); err != nil { + return err + } + } + if err = c.client.Delete(ctx, cert, client.Preconditions{UID: &cert.UID, ResourceVersion: &cert.ResourceVersion}); client.IgnoreNotFound(err) != nil { + return err + } + if _, err = c.getUserCertificate(ctx, user); err == nil { + return errorfactory.New(errorfactory.ResourceNotReady{}, errors.New("certificate is still deleting"), "waiting for user certificate deletion") + } else if !apierrors.IsNotFound(err) { + return err + } + return c.deleteOwnedUserSecret(ctx, user) +} + +func (c *certManager) getUserSecretForCleanup(ctx context.Context, user *v1alpha1.KafkaUser) (*corev1.Secret, error) { + secret := &corev1.Secret{} + err := c.client.Get(ctx, types.NamespacedName{Name: user.Spec.SecretName, Namespace: user.Namespace}, secret) + if apierrors.IsNotFound(err) { + return nil, nil + } + return secret, err +} + +func (c *certManager) deleteOwnedUserSecret(ctx context.Context, user *v1alpha1.KafkaUser) error { + secret, err := c.getUserSecretForCleanup(ctx, user) + if err != nil || secret == nil { + return err + } + if !metav1.IsControlledBy(secret, user) || len(secret.OwnerReferences) != 1 { + return nil + } + err = c.client.Delete(ctx, secret, client.Preconditions{UID: &secret.UID, ResourceVersion: &secret.ResourceVersion}) + return client.IgnoreNotFound(err) } // ReconcileUserCertificate ensures a certificate/secret combination using cert-manager diff --git a/pkg/pki/certmanagerpki/user_finalization_test.go b/pkg/pki/certmanagerpki/user_finalization_test.go new file mode 100644 index 000000000..e564402f8 --- /dev/null +++ b/pkg/pki/certmanagerpki/user_finalization_test.go @@ -0,0 +1,81 @@ +// Copyright 2026 Adobe. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package certmanagerpki + +import ( + "context" + "testing" + + certv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + "github.com/banzaicloud/koperator/api/v1alpha1" +) + +func TestFinalizeUserCertificateCleansRegeneratedSecret(t *testing.T) { + for _, owner := range []string{"generated", "external", "shared"} { + t.Run(owner, func(t *testing.T) { + ctx := context.Background() + manager, err := newMock(newMockCluster()) + require.NoError(t, err) + user := newMockUser() + user.UID = "user-uid" + certificate := &certv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: user.Name, Namespace: user.Namespace, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(user, v1alpha1.GroupVersion.WithKind("KafkaUser"))}, + }, + Spec: certv1.CertificateSpec{SecretName: user.Spec.SecretName}, + } + secret := newMockUserSecret() + secret.Annotations = map[string]string{certv1.CertificateNameKey: user.Name} + if owner != "generated" { + secret.OwnerReferences = []metav1.OwnerReference{{ + APIVersion: "v1", Kind: "ConfigMap", Name: "external-owner", UID: "external-uid", + }} + if owner == "shared" { + secret.OwnerReferences = append(secret.OwnerReferences, *metav1.NewControllerRef(user, v1alpha1.GroupVersion.WithKind("KafkaUser"))) + } + } + require.NoError(t, manager.client.Create(ctx, certificate)) + require.NoError(t, manager.client.Create(ctx, secret)) + base := manager.client + manager.client = interceptor.NewClient(base.(client.WithWatch), interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*corev1.Secret); ok { + err := c.Get(ctx, client.ObjectKeyFromObject(certificate), &certv1.Certificate{}) + require.True(t, apierrors.IsNotFound(err), "stop the certificate producer before deleting its Secret") + } + return c.Delete(ctx, obj, opts...) + }, + }) + require.NoError(t, manager.FinalizeUserCertificate(ctx, user)) + require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(certificate), certificate))) + err = base.Get(ctx, client.ObjectKeyFromObject(secret), secret) + if owner != "generated" { + require.NoError(t, err, "shared or externally owned Secrets must be preserved") + require.Equal(t, "external-uid", string(secret.OwnerReferences[0].UID)) + } else { + require.True(t, apierrors.IsNotFound(err), "generated private-key Secret must not leak") + } + require.NoError(t, manager.FinalizeUserCertificate(ctx, user), "cleanup must be replay-safe") + }) + } +} From 823d5adf428ba4ae89fcebe96f0279ac3d30cada Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 09:40:15 +0100 Subject: [PATCH 4/5] fix: checkpoint legacy ACL identity before PKI cleanup (#391) Persist a recovered legacy principal before deleting its owned Certificate or Secret, and stop cleanup when that metadata write fails. Cover failed ACL cleanup followed by restart and failed identity checkpoint writes without losing cleanup ownership. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- controllers/finalizer_ordering_test.go | 119 +++++++++++++++++++++++++ controllers/kafkauser_controller.go | 6 ++ 2 files changed, 125 insertions(+) diff --git a/controllers/finalizer_ordering_test.go b/controllers/finalizer_ordering_test.go index 915d27a84..599dd2f67 100644 --- a/controllers/finalizer_ordering_test.go +++ b/controllers/finalizer_ordering_test.go @@ -55,6 +55,7 @@ type protectedKafkaClient struct { beforeMutation func() afterMutationErr error deletedUsers []string + deleteUserErr error } func (c *protectedKafkaClient) CreateTopic(opts *kafkaclient.CreateTopicOptions) error { @@ -75,6 +76,9 @@ func (c *protectedKafkaClient) CreateUserACLs(access v1alpha1.KafkaAccessType, p func (c *protectedKafkaClient) DeleteUserACLs(user string, pattern v1alpha1.KafkaPatternType) error { c.deletedUsers = append(c.deletedUsers, user) + if c.deleteUserErr != nil { + return c.deleteUserErr + } return c.KafkaClient.DeleteUserACLs(user, pattern) } @@ -360,6 +364,121 @@ func TestDeletingUserUsesExistingCertificateIdentity(t *testing.T) { require.True(t, apierrors.IsNotFound(c.Get(ctx, client.ObjectKeyFromObject(user), user))) } +func TestDeletingLegacyUserCheckpointsACLIdentity(t *testing.T) { + publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "legacy-principal", Organization: []string{"legacy-organization"}}, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, publicKey, privateKey) + require.NoError(t, err) + const principal = "CN=legacy-principal,O=legacy-organization" + + for _, scenario := range []string{"acl-failure-then-restart", "checkpoint-failure"} { + t.Run(scenario, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, k8sscheme.AddToScheme(scheme)) + require.NoError(t, certv1.AddToScheme(scheme)) + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + user := &v1alpha1.KafkaUser{ + ObjectMeta: metav1.ObjectMeta{ + Name: "legacy-user", Namespace: "default", UID: "legacy-user-uid", + Finalizers: []string{userFinalizer, "example.com/unrelated"}, DeletionTimestamp: &metav1.Time{Time: metav1.Now().Time}, + }, + Spec: v1alpha1.KafkaUserSpec{ + SecretName: "legacy-secret", ClusterRef: v1alpha1.ClusterReference{Name: "cluster"}, + CreateCert: util.BoolPointer(true), + TopicGrants: []v1alpha1.UserTopicGrant{{ + TopicName: "topic", AccessType: v1alpha1.KafkaAccessTypeRead, PatternType: v1alpha1.KafkaPatternTypeLiteral, + }}, + }, + Status: v1alpha1.KafkaUserStatus{State: v1alpha1.UserStateCreated}, + } + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "default"}, + Spec: v1beta1.KafkaClusterSpec{ListenersConfig: v1beta1.ListenersConfig{ + SSLSecrets: &v1beta1.SSLSecrets{PKIBackend: v1beta1.PKIBackendCertManager}, + }}, + } + owner := *metav1.NewControllerRef(user, v1alpha1.GroupVersion.WithKind("KafkaUser")) + cert := &certv1.Certificate{ + ObjectMeta: metav1.ObjectMeta{Name: user.Name, Namespace: user.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: certv1.CertificateSpec{SecretName: user.Spec.SecretName}, + } + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: user.Spec.SecretName, Namespace: user.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}, + Data: map[string][]byte{corev1.TLSCertKey: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der})}, + } + base := fake.NewClientBuilder().WithScheme(scheme).WithObjects(user, cluster, cert, secret).WithStatusSubresource(user).Build() + checkpointFailure := errors.New("injected ACL identity checkpoint failure") + failCheckpoint := scenario == "checkpoint-failure" + wrapped := interceptor.NewClient(base, interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if _, ok := obj.(*v1alpha1.KafkaUser); ok && failCheckpoint && obj.GetAnnotations()[userACLPrincipalAnnotation] == principal { + return checkpointFailure + } + return c.Update(ctx, obj, opts...) + }, + }) + broker, closeBroker, err := kafkaclient.NewMockFromCluster(base, cluster) + require.NoError(t, err) + t.Cleanup(closeBroker) + require.NoError(t, broker.CreateUserACLs(v1alpha1.KafkaAccessTypeRead, v1alpha1.KafkaPatternTypeLiteral, principal, "topic")) + aclFailure := errors.New("injected ACL deletion failure") + protectedBroker := &protectedKafkaClient{KafkaClient: broker} + if scenario == "acl-failure-then-restart" { + protectedBroker.deleteUserErr = aclFailure + } + SetNewKafkaFromCluster(func(client.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + return protectedBroker, func() {}, nil + }) + t.Cleanup(func() { SetNewKafkaFromCluster(kafkaclient.NewFromCluster) }) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(user)} + _, err = (&KafkaUserReconciler{Client: wrapped, Scheme: scheme}).Reconcile(ctx, req) + if failCheckpoint { + require.ErrorIs(t, err, checkpointFailure) + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(cert), cert), "failed checkpoint must preserve the Certificate") + require.NoError(t, base.Get(ctx, client.ObjectKeyFromObject(secret), secret), "failed checkpoint must preserve the identity Secret") + require.Empty(t, protectedBroker.deletedUsers, "failed checkpoint must stop ACL cleanup") + } else { + require.ErrorIs(t, err, aclFailure) + require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(cert), cert))) + require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(secret), secret))) + } + require.NoError(t, base.Get(ctx, req.NamespacedName, user)) + require.Contains(t, user.Finalizers, userFinalizer) + require.Contains(t, user.Finalizers, "example.com/unrelated") + if failCheckpoint { + require.NotContains(t, user.Annotations, userACLPrincipalAnnotation) + } + acls, err := broker.ListUserACLs() + require.NoError(t, err) + require.NotEmpty(t, acls, "failed cleanup must leave ACLs available for retry") + + failCheckpoint = false + protectedBroker.deleteUserErr = nil + _, err = (&KafkaUserReconciler{Client: wrapped, Scheme: scheme}).Reconcile(ctx, req) + require.NoError(t, err, "restart must recover the principal even after the owned Secret was deleted") + require.NoError(t, base.Get(ctx, req.NamespacedName, user)) + require.Equal(t, principal, user.Annotations[userACLPrincipalAnnotation]) + require.Equal(t, []string{"example.com/unrelated"}, user.Finalizers) + if scenario == "acl-failure-then-restart" { + require.Equal(t, []string{principal, principal}, protectedBroker.deletedUsers) + } else { + require.Equal(t, []string{principal}, protectedBroker.deletedUsers) + } + acls, err = broker.ListUserACLs() + require.NoError(t, err) + require.Empty(t, acls) + require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(cert), cert))) + require.True(t, apierrors.IsNotFound(base.Get(ctx, client.ObjectKeyFromObject(secret), secret))) + }) + } +} + func TestDeletionRetainsFinalizerOnClusterReadFailure(t *testing.T) { for _, kind := range []string{"topic", "user"} { t.Run(kind, func(t *testing.T) { diff --git a/controllers/kafkauser_controller.go b/controllers/kafkauser_controller.go index c02d5e38d..f8e99432f 100644 --- a/controllers/kafkauser_controller.go +++ b/controllers/kafkauser_controller.go @@ -313,6 +313,12 @@ func (r *KafkaUserReconciler) finalizeUser(ctx context.Context, cluster *v1beta1 if err != nil { return ctrl.Result{}, err } + if _, recorded := user.Annotations[userACLPrincipalAnnotation]; !recorded { + metav1.SetMetaDataAnnotation(&user.ObjectMeta, userACLPrincipalAnnotation, kafkaUser) + if user, err = r.updateAndFetchLatest(ctx, user); err != nil { + return ctrl.Result{}, err + } + } } if cluster.Spec.ListenersConfig.SSLSecrets != nil || user.Spec.PKIBackendSpec != nil { backend := v1beta1.PKIBackendProvided From 535de8ff83cb4f01d6c73a7e9461394a7d9e72b5 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Tue, 17 Nov 2026 21:16:58 +0100 Subject: [PATCH 5/5] test(controllers): cover ACL identity checkpoint after async certificate issuance (#391) Reconcile a cert-manager KafkaUser with topic grants before its Secret is issued, then after issuance, and verify the issued principal is recorded before ACL creation and used for ACL cleanup on deletion. --- controllers/finalizer_ordering_test.go | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/controllers/finalizer_ordering_test.go b/controllers/finalizer_ordering_test.go index 599dd2f67..de261d7d7 100644 --- a/controllers/finalizer_ordering_test.go +++ b/controllers/finalizer_ordering_test.go @@ -628,6 +628,79 @@ func TestDeletingUserAfterACLsCreatedAndStatusWriteFails(t *testing.T) { require.Equal(t, []string{principal}, protectedBroker.deletedUsers) } +func TestDeletingUserAfterAsynchronousCertificateIssuance(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + require.NoError(t, k8sscheme.AddToScheme(scheme)) + require.NoError(t, certv1.AddToScheme(scheme)) + require.NoError(t, v1alpha1.AddToScheme(scheme)) + require.NoError(t, v1beta1.AddToScheme(scheme)) + // Subject of the fixture returned by certutil.GenerateTestCert. + const principal = "CN=test-cn,O=test-ou" + user := &v1alpha1.KafkaUser{ + ObjectMeta: metav1.ObjectMeta{Name: "async-user", Namespace: "default", UID: "async-user-uid"}, + Spec: v1alpha1.KafkaUserSpec{ + SecretName: "async-secret", ClusterRef: v1alpha1.ClusterReference{Name: "cluster"}, + TopicGrants: []v1alpha1.UserTopicGrant{{TopicName: "topic", AccessType: v1alpha1.KafkaAccessTypeRead}}, + }, + } + cluster := &v1beta1.KafkaCluster{ + ObjectMeta: metav1.ObjectMeta{Name: "cluster", Namespace: "default"}, + Spec: v1beta1.KafkaClusterSpec{ListenersConfig: v1beta1.ListenersConfig{ + SSLSecrets: &v1beta1.SSLSecrets{PKIBackend: v1beta1.PKIBackendCertManager}, + }}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(user, cluster).WithStatusSubresource(user).Build() + broker, closeBroker, err := kafkaclient.NewMockFromCluster(c, cluster) + require.NoError(t, err) + t.Cleanup(closeBroker) + req := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(user)} + protectedBroker := &protectedKafkaClient{KafkaClient: broker, beforeMutation: func() { + persisted := &v1alpha1.KafkaUser{} + require.NoError(t, c.Get(ctx, req.NamespacedName, persisted)) + require.Equal(t, principal, persisted.Annotations[userACLPrincipalAnnotation], + "ACL creation must follow a durable record of the issued principal") + }} + SetNewKafkaFromCluster(func(client.Client, *v1beta1.KafkaCluster) (kafkaclient.KafkaClient, func(), error) { + return protectedBroker, func() {}, nil + }) + t.Cleanup(func() { SetNewKafkaFromCluster(kafkaclient.NewFromCluster) }) + r := &KafkaUserReconciler{Client: c, Scheme: scheme} + + // cert-manager has not issued the Secret yet. + result, err := r.Reconcile(ctx, req) + require.NoError(t, err) + require.Positive(t, result.RequeueAfter) + acls, err := broker.ListUserACLs() + require.NoError(t, err) + require.Empty(t, acls) + + certificatePEM, _, _, err := certutil.GenerateTestCert() + require.NoError(t, err) + require.NoError(t, c.Create(ctx, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: user.Spec.SecretName, Namespace: user.Namespace}, + Data: map[string][]byte{ + corev1.TLSCertKey: certificatePEM, corev1.TLSPrivateKeyKey: []byte("key"), v1alpha1.CoreCACertKey: certificatePEM, + }, + })) + _, err = r.Reconcile(ctx, req) + require.NoError(t, err) + acls, err = broker.ListUserACLs() + require.NoError(t, err) + require.NotEmpty(t, acls) + + require.NoError(t, c.Delete(ctx, user)) + for attempt := 0; attempt < 3 && !apierrors.IsNotFound(c.Get(ctx, req.NamespacedName, user)); attempt++ { + _, err = r.Reconcile(ctx, req) + require.NoError(t, err) + } + require.True(t, apierrors.IsNotFound(c.Get(ctx, req.NamespacedName, user))) + require.Equal(t, []string{principal}, protectedBroker.deletedUsers) + acls, err = broker.ListUserACLs() + require.NoError(t, err) + require.Empty(t, acls, "deleting an asynchronously issued user must not leak its ACLs") +} + type pendingTopicDeletionClient struct { kafkaclient.KafkaClient releaseWait <-chan struct{}