diff --git a/controllers/envtest_test.go b/controllers/envtest_test.go index 0fed29d2..c8cc8f56 100644 --- a/controllers/envtest_test.go +++ b/controllers/envtest_test.go @@ -12,7 +12,9 @@ import ( "time" api "github.com/pravega/zookeeper-operator/api/v1beta1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -22,7 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/envtest" ) -func newEnvtestCluster(t *testing.T) (context.Context, client.Client, *api.ZookeeperCluster) { +func newEnvtestCluster(t *testing.T) (context.Context, client.WithWatch, *api.ZookeeperCluster) { t.Helper() environment := &envtest.Environment{ CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, @@ -41,10 +43,16 @@ func newEnvtestCluster(t *testing.T) (context.Context, client.Client, *api.Zooke if err := corev1.AddToScheme(scheme); err != nil { t.Fatal(err) } + if err := appsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := policyv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } if err := api.AddToScheme(scheme); err != nil { t.Fatal(err) } - c, err := client.New(cfg, client.Options{Scheme: scheme}) + c, err := client.NewWithWatch(cfg, client.Options{Scheme: scheme}) if err != nil { t.Fatal(err) } @@ -102,6 +110,11 @@ func newEnvtestCluster(t *testing.T) (context.Context, client.Client, *api.Zooke return ctx, c, cluster } +func TestEnvtestResourceWriteGuards(t *testing.T) { + _, c, cluster := newEnvtestCluster(t) + exerciseResourceWriteGuards(t, c, cluster) +} + func TestEnvtestStatusSubresource(t *testing.T) { ctx, c, cluster := newEnvtestCluster(t) cluster.Status.ReadyReplicas = 1 diff --git a/controllers/zookeepercluster_controller.go b/controllers/zookeepercluster_controller.go index e3cd00c7..139a7624 100644 --- a/controllers/zookeepercluster_controller.go +++ b/controllers/zookeepercluster_controller.go @@ -231,11 +231,13 @@ func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, i } else if err != nil { return err } else { - foundServiceAccount.ImagePullSecrets = serviceAccount.ImagePullSecrets - ctrl.LoggerFrom(ctx).Info("Updating ServiceAccount", "ServiceAccount.Namespace", serviceAccount.Namespace, "ServiceAccount.Name", serviceAccount.Name) - err = r.Client.Update(ctx, foundServiceAccount) - if err != nil { - return err + if !equality.Semantic.DeepEqual(foundServiceAccount.ImagePullSecrets, serviceAccount.ImagePullSecrets) { + foundServiceAccount.ImagePullSecrets = serviceAccount.ImagePullSecrets + ctrl.LoggerFrom(ctx).Info("Updating ServiceAccount", "ServiceAccount.Namespace", serviceAccount.Namespace, "ServiceAccount.Name", serviceAccount.Name) + err = r.Client.Update(ctx, foundServiceAccount) + if err != nil { + return err + } } } } @@ -576,6 +578,10 @@ func (r *ZookeeperClusterReconciler) reconcileConfigMap(ctx context.Context, ins } else if err != nil { return err } else { + if equality.Semantic.DeepEqual(foundCm.Data, cm.Data) && + equality.Semantic.DeepEqual(foundCm.BinaryData, cm.BinaryData) { + return nil + } ctrl.LoggerFrom(ctx).Info("Updating existing config-map", "ConfigMap.Namespace", foundCm.Namespace, "ConfigMap.Name", foundCm.Name) diff --git a/controllers/zookeepercluster_controller_test.go b/controllers/zookeepercluster_controller_test.go index 5ac1657e..87162250 100644 --- a/controllers/zookeepercluster_controller_test.go +++ b/controllers/zookeepercluster_controller_test.go @@ -13,6 +13,7 @@ package controllers import ( "context" "errors" + "fmt" "os" "reflect" "strconv" @@ -678,6 +679,194 @@ func (c *cancellationSessionClient) Connect(ctx context.Context, _ string) error func (c *cancellationSessionClient) Close() { close(c.closed) } +func TestResourceWriteGuards(t *testing.T) { + r, z, _ := newScaleMetadataTestReconciler(t, 1) + exerciseResourceWriteGuards(t, r.Client.(client.WithWatch), z) +} + +func exerciseResourceWriteGuards(t *testing.T, cl client.WithWatch, z *v1beta1.ZookeeperCluster) { + t.Helper() + ctx := t.Context() + z.Spec.Pod.ServiceAccountName = "example-account" + z.Spec.Pod.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "old-secret"}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + resource client.Object + reconcile func(*ZookeeperClusterReconciler, context.Context, *v1beta1.ZookeeperCluster) error + }{ + {"ConfigMap", &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: z.ConfigMapName(), Namespace: z.Namespace}}, (*ZookeeperClusterReconciler).reconcileConfigMap}, + {"ServiceAccount", &corev1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Name: z.Spec.Pod.ServiceAccountName, Namespace: z.Namespace}}, (*ZookeeperClusterReconciler).reconcileStatefulSet}, + } { + t.Run(tc.name, func(t *testing.T) { + r := &ZookeeperClusterReconciler{Client: cl, APIReader: cl, Scheme: cl.Scheme(), NewZkClient: func() zk.ZookeeperClient { return &MockZookeeperClient{} }, Log: log} + if err := tc.reconcile(r, ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(tc.resource), tc.resource); err != nil { + t.Fatal(err) + } + tc.resource.SetAnnotations(map[string]string{"external": "keep"}) + if err := cl.Update(ctx, tc.resource); err != nil { + t.Fatal(err) + } + t.Run("unchanged", func(t *testing.T) { + r.Client = rejectResourceWrites(cl, tc.resource) + defer func() { r.Client = cl }() + for range 2 { + if err := tc.reconcile(r, ctx, z); err != nil { + t.Fatal(err) + } + } + }) + t.Run("change and conflict retry", func(t *testing.T) { + switch tc.resource.(type) { + case *corev1.ConfigMap: + z.Spec.Conf.TickTime = 4000 + case *corev1.ServiceAccount: + z.Spec.Pod.ImagePullSecrets = []corev1.LocalObjectReference{{Name: "new-secret"}} + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + concurrentWrite := func(ctx context.Context, c client.WithWatch, obj client.Object) error { + if reflect.TypeOf(obj) == reflect.TypeOf(tc.resource) { + if err := c.Get(ctx, client.ObjectKeyFromObject(tc.resource), tc.resource); err != nil { + return err + } + tc.resource.GetAnnotations()["external"] = "concurrent" + return c.Update(ctx, tc.resource) + } + return nil + } + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if err := concurrentWrite(ctx, c, obj); err != nil { + return err + } + return c.Update(ctx, obj, opts...) + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := concurrentWrite(ctx, c, obj); err != nil { + return err + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + defer func() { r.Client = cl }() + if err := tc.reconcile(r, ctx, z); !apierrors.IsConflict(err) { + t.Fatalf("stale write must conflict, got %v", err) + } + r.Client = cl + if err := tc.reconcile(r, ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(tc.resource), tc.resource); err != nil { + t.Fatal(err) + } + if tc.resource.GetAnnotations()["external"] != "concurrent" { + t.Fatal("retry overwrote the concurrent annotation") + } + switch resource := tc.resource.(type) { + case *corev1.ConfigMap: + if !strings.Contains(resource.Data["zoo.cfg"], "tickTime=4000\n") { + t.Fatal("changed configuration did not reach the ConfigMap") + } + case *corev1.ServiceAccount: + if len(resource.ImagePullSecrets) != 1 || resource.ImagePullSecrets[0].Name != "new-secret" { + t.Fatalf("imagePullSecrets were not updated: %v", resource.ImagePullSecrets) + } + } + }) + t.Run("owned field removal", func(t *testing.T) { + switch resource := tc.resource.(type) { + case *corev1.ConfigMap: + resource.BinaryData = map[string][]byte{"obsolete": []byte("old")} + if err := cl.Update(ctx, resource); err != nil { + t.Fatal(err) + } + case *corev1.ServiceAccount: + z.Spec.Pod.ImagePullSecrets = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + } + if err := tc.reconcile(r, ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(tc.resource), tc.resource); err != nil { + t.Fatal(err) + } + switch resource := tc.resource.(type) { + case *corev1.ConfigMap: + if len(resource.BinaryData) != 0 { + t.Fatal("obsolete binary data was retained") + } + case *corev1.ServiceAccount: + if len(resource.ImagePullSecrets) != 0 { + t.Fatal("removed imagePullSecrets were retained") + } + } + r.Client = rejectResourceWrites(cl, tc.resource) + defer func() { r.Client = cl }() + if err := tc.reconcile(r, ctx, z); err != nil { + t.Fatal(err) + } + }) + }) + } +} + +func TestServiceAccountNoChangeStillScalesStatefulSet(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 3) + z.Spec.Pod.ServiceAccountName = "example-account" + if err := r.Client.Update(t.Context(), z); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), z); err != nil { + t.Fatal(err) + } + z.Spec.Replicas = 5 + if err := r.Client.Update(t.Context(), z); err != nil { + t.Fatal(err) + } + r.Client = rejectResourceWrites(r.Client.(client.WithWatch), &corev1.ServiceAccount{}) + if err := r.reconcileStatefulSet(t.Context(), z); err != nil { + t.Fatal(err) + } + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if *sts.Spec.Replicas != 5 { + t.Fatalf("unchanged ServiceAccount stopped StatefulSet scaling: replicas=%d", *sts.Spec.Replicas) + } +} + +func rejectResourceWrites(cl client.WithWatch, resource client.Object) client.WithWatch { + reject := func(obj client.Object) error { + if reflect.TypeOf(obj) == reflect.TypeOf(resource) { + return fmt.Errorf("unchanged %T/%s received a write", obj, obj.GetName()) + } + return nil + } + return interceptor.NewClient(cl, interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if err := reject(obj); err != nil { + return err + } + return c.Update(ctx, obj, opts...) + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := reject(obj); err != nil { + return err + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) +} + func TestScaleMetadataStatefulSetConflictRetries(t *testing.T) { r, z, sts := newScaleMetadataTestReconciler(t, 3) z.Spec.Replicas = 5 diff --git a/main_test.go b/main_test.go index 11ba6960..db3d27de 100644 --- a/main_test.go +++ b/main_test.go @@ -78,7 +78,7 @@ func TestOperatorHealthLifecycle(t *testing.T) { } if err := (&controllers.ZookeeperClusterReconciler{ Client: mgr.GetClient(), Scheme: scheme, Log: ctrl.Log, - ZkClient: new(zkClient.DefaultZookeeperClient), + NewZkClient: func() zkClient.ZookeeperClient { return new(zkClient.DefaultZookeeperClient) }, }).SetupWithManager(mgr); err != nil { t.Fatal(err) }