diff --git a/controllers/compatibility_envtest_test.go b/controllers/compatibility_envtest_test.go new file mode 100644 index 000000000..c74939777 --- /dev/null +++ b/controllers/compatibility_envtest_test.go @@ -0,0 +1,736 @@ +//go:build envtest + +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "maps" + "slices" + "testing" + + api "github.com/pravega/zookeeper-operator/api/v1beta1" + "github.com/pravega/zookeeper-operator/pkg/zk" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func TestEnvtestMutableApplyPreservesDifferentImmutableFields(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + sts := zk.MakeStatefulSet(z).DeepCopy() + zk.EnsureLogbackProjection(z, nil, sts) + sts.Spec.ServiceName = "existing-headless" + sts.Spec.Selector.MatchLabels["kind"] = "ZookeeperMember" + sts.Spec.PodManagementPolicy = appsv1.ParallelPodManagement + storage := resource.MustParse("32Gi") + sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = storage + if err := controllerutil.SetControllerReference(z, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, sts, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + z.Spec.Image.Tag = "updated" + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + if err := convergenceReconciler(cl).reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + spec := sts.Spec + if spec.ServiceName != "existing-headless" || spec.PodManagementPolicy != appsv1.ParallelPodManagement || spec.Selector.MatchLabels["kind"] != "ZookeeperMember" || !spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().Equal(storage) { + t.Fatal("StatefulSet immutable fields were not preserved") + } + if spec.Template.Spec.Containers[0].Image != z.Spec.Image.ToString() { + t.Fatal("mutable template change did not take effect") + } +} + +func TestEnvtestPreservesNamedContainerPortAliases(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Ports = append(z.Spec.Ports, corev1.ContainerPort{Name: "client-alt", ContainerPort: 2181}) + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + hasAlias := func(name string) bool { + return slices.ContainsFunc(sts.Spec.Template.Spec.Containers[0].Ports, func(port corev1.ContainerPort) bool { + return port.Name == name && port.ContainerPort == 2181 && port.Protocol == corev1.ProtocolTCP + }) + } + if len(sts.Spec.Template.Spec.Containers[0].Ports) != len(z.Spec.Ports) || !hasAlias("client") || !hasAlias("client-alt") { + t.Fatal("an accepted port alias was dropped") + } + z.Spec.Ports = z.Spec.Ports[:len(z.Spec.Ports)-1] + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + if len(sts.Spec.Template.Spec.Containers[0].Ports) != len(z.Spec.Ports) || !hasAlias("client") || hasAlias("client-alt") { + t.Fatal("expected only the canonical client port after retiring the alias") + } +} + +func TestEnvtestRepairsNamedPortKeyDrift(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + svc := &corev1.Service{} + svcKey := client.ObjectKey{Namespace: z.Namespace, Name: z.GetClientServiceName()} + if err := cl.Get(ctx, svcKey, svc); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = 2199 + sts.Spec.Template.Spec.Containers[0].Ports = append(sts.Spec.Template.Spec.Containers[0].Ports, corev1.ContainerPort{Name: "extra", ContainerPort: 9090}) + svc.Spec.Ports[0].Port = 2199 + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Name: "extra", Port: 9090}) + for _, obj := range []client.Object{sts, svc} { + if err := cl.Update(ctx, obj, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, svcKey, svc); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + ports := sts.Spec.Template.Spec.Containers[0].Ports + for name, port := range map[string]int32{"client": 2181, "extra": 9090} { + if !slices.ContainsFunc(ports, func(p corev1.ContainerPort) bool { return p.Name == name && p.ContainerPort == port }) { + t.Fatalf("expected %s/%d in container ports: %+v", name, port, ports) + } + } + for name, port := range map[string]int32{"tcp-client": 2181, "extra": 9090} { + if !slices.ContainsFunc(svc.Spec.Ports, func(p corev1.ServicePort) bool { return p.Name == name && p.Port == port }) { + t.Fatalf("expected %s/%d in Service ports: %+v", name, port, svc.Spec.Ports) + } + } + previous := sts.DeepCopy() + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + if sts.Generation != previous.Generation { + t.Fatalf("repair was followed by another rollout: ports before=%+v after=%+v", previous.Spec.Template.Spec.Containers[0].Ports, sts.Spec.Template.Spec.Containers[0].Ports) + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestReplacesMutuallyExclusiveSources(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Pod.Env = []corev1.EnvVar{{Name: "SETTING", Value: "desired"}} + z.Spec.Volumes = []corev1.Volume{{Name: "input", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + env := sts.Spec.Template.Spec.Containers[0].Env + env[0] = corev1.EnvVar{Name: "ENVOY_SIDECAR_STATUS", Value: "different"} + env[1] = corev1.EnvVar{Name: "SETTING", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}} + sts.Spec.Template.Spec.Volumes[0].VolumeSource = corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "another"}}} + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + env = sts.Spec.Template.Spec.Containers[0].Env + vol := sts.Spec.Template.Spec.Volumes[0] + if env[0].Value != "" || env[0].ValueFrom == nil || env[1].Value != "desired" || env[1].ValueFrom != nil || vol.EmptyDir == nil || vol.ConfigMap != nil { + t.Fatal("operator did not replace the incompatible environment and volume sources") + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestRetainsVolumeUntilIndependentMountIsRemoved(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Volumes = []corev1.Volume{{Name: "shared", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, corev1.Container{ + Name: "independent", Image: "busybox:1", + VolumeMounts: []corev1.VolumeMount{{Name: "shared", MountPath: "/shared"}}, + }) + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + z.Spec.Volumes = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + for _, mounted := range []bool{true, false} { + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + pod := &sts.Spec.Template.Spec + i := slices.IndexFunc(pod.Containers, func(c corev1.Container) bool { return c.Name == "independent" }) + if i < 0 || slices.ContainsFunc(pod.Volumes, func(v corev1.Volume) bool { return v.Name == "shared" }) != mounted || (len(pod.Containers[i].VolumeMounts) != 0) != mounted { + t.Fatalf("expected the volume to survive only while the independent container mounts it: mounted=%v", mounted) + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) + if mounted { + pod.Containers[i].VolumeMounts = nil + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + } + } +} + +func TestEnvtestRetainsCompleteSourceOfSharedVolume(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Volumes = []corev1.Volume{{Name: "shared", VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "shared-config"}}, + }}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + // An independent applier owns the mount and defaultMode, but not the ConfigMap name. + intent := map[string]any{ + "apiVersion": "apps/v1", "kind": "StatefulSet", + "metadata": map[string]string{"name": z.Name, "namespace": z.Namespace}, + "spec": map[string]any{"template": map[string]any{"spec": map[string]any{ + "containers": []corev1.Container{{Name: "independent", Image: "busybox:1", VolumeMounts: []corev1.VolumeMount{{Name: "shared", MountPath: "/shared"}}}}, + "volumes": []map[string]any{{"name": "shared", "configMap": map[string]int{"defaultMode": 0o640}}}, + }}}, + } + payload, err := json.Marshal(intent) + if err != nil { + t.Fatal(err) + } + if err := cl.Patch(ctx, sts, client.RawPatch(types.ApplyPatchType, payload), client.FieldOwner("independent"), client.ForceOwnership); err != nil { + t.Fatal(err) + } + z.Spec.Volumes = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + pod := &sts.Spec.Template.Spec + if !slices.ContainsFunc(pod.Volumes, func(v corev1.Volume) bool { + return v.Name == "shared" && v.ConfigMap != nil && v.ConfigMap.Name == "shared-config" && v.ConfigMap.DefaultMode != nil && *v.ConfigMap.DefaultMode == 0o640 + }) { + t.Fatalf("shared volume lost its ConfigMap name or independently owned mode: %+v", pod.Volumes) + } + i := slices.IndexFunc(pod.Containers, func(c corev1.Container) bool { return c.Name == "independent" }) + if i < 0 || !slices.ContainsFunc(pod.Containers[i].VolumeMounts, func(m corev1.VolumeMount) bool { return m.Name == "shared" }) { + t.Fatal("independent mount was not retained") + } + pod.Containers[i].VolumeMounts = nil + if err := cl.Update(ctx, sts, client.FieldOwner("independent")); err != nil { + t.Fatal(err) + } + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + if slices.ContainsFunc(sts.Spec.Template.Spec.Volumes, func(v corev1.Volume) bool { return v.Name == "shared" }) { + t.Fatal("partially shared volume was not pruned after its last mount disappeared") + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestRestoresDuplicateEnvWithoutMovingIndependentVariables(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Pod.Env = []corev1.EnvVar{ + {Name: "DATA_DIR", Value: "$(EXTERNAL_BASE)/data"}, + {Name: "WRAPPED", Value: "first"}, + {Name: "WRAPPED", Value: "$(WRAPPED)/second"}, + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + c := &sts.Spec.Template.Spec.Containers[0] + c.Env = slices.Insert(c.Env, 1, corev1.EnvVar{Name: "EXTERNAL_BASE", Value: "/srv"}) + c.Env = append(c.Env, corev1.EnvVar{Name: "EXTERNAL_TRAILING", Value: "$(WRAPPED)"}) + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + var names []string + for _, env := range sts.Spec.Template.Spec.Containers[0].Env { + names = append(names, env.Name) + } + want := []string{"ENVOY_SIDECAR_STATUS", "EXTERNAL_BASE", "DATA_DIR", "WRAPPED", "WRAPPED", "EXTERNAL_TRAILING"} + if !slices.Equal(names, want) { + t.Fatalf("expected the injected variables to keep their positions: got %v, want %v", names, want) + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestClearsRenderedEmptyEnvironmentLiteral(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Pod.Env = []corev1.EnvVar{{Name: "SETTING"}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + c := &sts.Spec.Template.Spec.Containers[0] + c.Env[1].Value = "out-of-date" + c.Env = append(c.Env, corev1.EnvVar{Name: "UNRELATED", Value: "keep"}) + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + env := sts.Spec.Template.Spec.Containers[0].Env + if !slices.ContainsFunc(env, func(e corev1.EnvVar) bool { return e.Name == "SETTING" && e.Value == "" && e.ValueFrom == nil }) || !slices.ContainsFunc(env, func(e corev1.EnvVar) bool { return e.Name == "UNRELATED" && e.Value == "keep" }) { + t.Fatalf("expected the rendered empty value and preserved unrelated value: %+v", env) + } +} + +func TestEnvtestRepairsRenderedProbeAndLifecycleActions(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + container := &sts.Spec.Template.Spec.Containers[0] + container.ReadinessProbe.ProbeHandler = corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/health", Port: intstr.FromInt32(8080)}} + container.Lifecycle.PreStop = &corev1.LifecycleHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/stop", Port: intstr.FromInt32(8080)}} + container.Lifecycle.PostStart = &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"true"}}} + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + container = &sts.Spec.Template.Spec.Containers[0] + if container.ReadinessProbe.Exec == nil || container.ReadinessProbe.HTTPGet != nil || container.Lifecycle.PreStop.Exec == nil || container.Lifecycle.PreStop.HTTPGet != nil || container.Lifecycle.PostStart == nil { + t.Fatal("requested actions did not replace incompatible variants while preserving the unrelated hook") + } +} + +func TestEnvtestRetiresSharedContainerAsAUnit(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Containers = []corev1.Container{{Name: "retired", Image: "busybox:1"}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + intent := map[string]any{ + "apiVersion": "apps/v1", "kind": "StatefulSet", + "metadata": map[string]string{"name": z.Name, "namespace": z.Namespace}, + "spec": map[string]any{"template": map[string]any{"spec": map[string]any{"containers": []corev1.Container{ + {Name: "retired", Args: []string{"extra"}}, + {Name: "independent", Image: "busybox:1"}, + }}}}, + } + data, err := json.Marshal(intent) + if err != nil { + t.Fatal(err) + } + if err := cl.Patch(ctx, sts, client.RawPatch(types.ApplyPatchType, data), client.FieldOwner("injector")); err != nil { + t.Fatal(err) + } + z.Spec.Containers = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + containers := sts.Spec.Template.Spec.Containers + if len(containers) != 2 || !slices.ContainsFunc(containers, func(c corev1.Container) bool { return c.Name == "independent" }) || slices.ContainsFunc(containers, func(c corev1.Container) bool { return c.Name == "retired" }) { + t.Fatalf("expected the retired container removed and the independent one retained: %+v", containers) + } +} + +func TestEnvtestPullPolicyFollowsImageButPreservesExplicitOwners(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Containers = []corev1.Container{ + {Name: "defaulted", Image: "busybox:1"}, + {Name: "external-policy", Image: "busybox:1"}, + {Name: "explicit-policy", Image: "busybox:1", ImagePullPolicy: corev1.PullIfNotPresent}, + } + z.Spec.InitContainers = []corev1.Container{{Name: "defaulted-init", Image: "busybox:1"}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[1].ImagePullPolicy = corev1.PullNever + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + for i := range z.Spec.Containers { + z.Spec.Containers[i].Image = "busybox:latest" + } + z.Spec.InitContainers[0].Image = "busybox" + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + for i, want := range []corev1.PullPolicy{corev1.PullAlways, corev1.PullNever, corev1.PullIfNotPresent} { + if got := sts.Spec.Template.Spec.Containers[i].ImagePullPolicy; got != want { + t.Errorf("container %d pull policy = %s, want %s", i, got, want) + } + } + if sts.Spec.Template.Spec.InitContainers[0].ImagePullPolicy != corev1.PullAlways { + t.Error("an untagged init image should default to Always") + } +} + +func TestEnvtestAdminServiceTransitionsAreAtomic(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKey{Namespace: z.Namespace, Name: z.GetAdminServerServiceName()} + svc := &corev1.Service{} + if err := cl.Get(ctx, key, svc); err != nil { + t.Fatal(err) + } + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Name: "internal-extra", Port: 9090}) + svc.Annotations = map[string]string{"external": "preserve"} + if err := cl.Update(ctx, svc, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + z.Spec.AdminServerService.External = true + const frontend = "networking.gke.io/load-balancer-type" + const ranges = "service.beta.kubernetes.io/load-balancer-source-ranges" + z.Spec.AdminServerService.Annotations = map[string]string{frontend: "Internal"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + for _, external := range []bool{true, false} { + writes := 0 + r.Client = observeServicePatches(cl, key, func(live *corev1.Service) { + writes++ + wantType := corev1.ServiceTypeClusterIP + if external { + wantType = corev1.ServiceTypeLoadBalancer + } + if live.Spec.Type != wantType || len(live.Spec.Ports) != 1 || live.Spec.Ports[0].Name != "tcp-admin-server" || live.Annotations["external"] != "preserve" { + t.Fatalf("unexpected persisted admin Service state: type=%s ports=%+v", live.Spec.Type, live.Spec.Ports) + } + if external && live.Annotations[frontend] != "Internal" || !external && (live.Annotations[ranges] != "" || len(live.Spec.LoadBalancerSourceRanges) != 0) { + t.Fatal("Service type and exposure annotations must transition together") + } + }) + if err := r.reconcileAdminServerService(ctx, z); err != nil { + t.Fatal(err) + } + if writes != 1 { + t.Fatalf("transition issued %d Service patches, want 1", writes) + } + if external { + if err := cl.Get(ctx, key, svc); err != nil { + t.Fatal(err) + } + svc.Annotations[ranges] = "192.0.2.0/24" + svc.Spec.LoadBalancerSourceRanges = []string{"198.51.100.0/24"} + if err := cl.Update(ctx, svc, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + z.Spec.AdminServerService.External = false + z.Spec.AdminServerService.Annotations = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + } + } +} + +func TestEnvtestAdminPortChangeRetainsManuallyAllocatedNodePort(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.AdminServerService.External = true + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKey{Namespace: z.Namespace, Name: z.GetAdminServerServiceName()} + svc := &corev1.Service{} + if err := cl.Get(ctx, key, svc); err != nil { + t.Fatal(err) + } + nodePort := svc.Spec.Ports[0].NodePort + if nodePort == 0 { + t.Fatal("expected the test Service to have an allocated NodePort") + } + allocate := false + svc.Spec.AllocateLoadBalancerNodePorts = &allocate + svc.Spec.LoadBalancerSourceRanges = []string{"192.0.2.0/24"} + if err := cl.Update(ctx, svc, client.FieldOwner("network-config")); err != nil { + t.Fatal(err) + } + const newPort = 8081 + for i := range z.Spec.Ports { + if z.Spec.Ports[i].Name == "admin-server" { + z.Spec.Ports[i].ContainerPort = newPort + } + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + for i := range 3 { + if err := r.reconcileAdminServerService(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, key, svc); err != nil { + t.Fatal(err) + } + if ports := svc.Spec.Ports; len(ports) != 1 || ports[0].Port != newPort || ports[0].NodePort != nodePort { + t.Fatalf("reconcile %d: expected port %d to retain NodePort %d, got %+v", i+1, newPort, nodePort, ports) + } + if svc.Spec.AllocateLoadBalancerNodePorts == nil || *svc.Spec.AllocateLoadBalancerNodePorts || !slices.Equal(svc.Spec.LoadBalancerSourceRanges, []string{"192.0.2.0/24"}) { + t.Fatal("external access settings changed during the port update") + } + } +} + +func TestEnvtestHeadlessServicePortSwapIsAtomic(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKeyFromObject(zk.MakeHeadlessService(z)) + z.Spec.Ports[0].ContainerPort, z.Spec.Ports[1].ContainerPort = z.Spec.Ports[1].ContainerPort, z.Spec.Ports[0].ContainerPort + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + desired := zk.MakeHeadlessService(z) + writes := 0 + r.Client = observeServicePatches(cl, key, func(live *corev1.Service) { + writes++ + for _, wanted := range desired.Spec.Ports { + if !slices.ContainsFunc(live.Spec.Ports, func(p corev1.ServicePort) bool { return p.Name == wanted.Name && p.Port == wanted.Port }) { + t.Fatalf("persisted headless Service lacks %s/%d", wanted.Name, wanted.Port) + } + } + }) + if err := r.reconcileHeadlessService(ctx, z); err != nil { + t.Fatal(err) + } + if writes != 1 { + t.Fatalf("port swap issued %d Service patches, want 1", writes) + } +} + +func observeServicePatches(cl client.WithWatch, key client.ObjectKey, observe func(*corev1.Service)) client.WithWatch { + return interceptor.NewClient(cl, interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := c.Patch(ctx, obj, patch, opts...); err != nil { + return err + } + if svc, ok := obj.(*corev1.Service); ok && client.ObjectKeyFromObject(svc) == key { + observe(svc) + } + return nil + }, + }) +} + +func TestEnvtestEnforcesRequestedZooKeeperIdentity(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKeyFromObject(z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + oldUser, requestedUser, readOnly := int64(2000), int64(1000), true + sts.Spec.Template.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{RunAsUser: &oldUser, ReadOnlyRootFilesystem: &readOnly} + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + z.Spec.Pod.SecurityContext = &corev1.PodSecurityContext{RunAsUser: &requestedUser} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + pod, container := sts.Spec.Template.Spec.SecurityContext, sts.Spec.Template.Spec.Containers[0].SecurityContext + if pod == nil || pod.RunAsUser == nil || *pod.RunAsUser != requestedUser || container == nil || container.RunAsUser != nil || container.ReadOnlyRootFilesystem == nil || !*container.ReadOnlyRootFilesystem { + t.Fatal("operator must enforce the requested identity without dropping other security settings") + } +} + +func TestEnvtestSeccompProfileReplacesConflictingLegacyAnnotation(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKeyFromObject(z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Annotations[corev1.SeccompPodAnnotationKey] = "localhost/previous.json" + sts.Spec.Template.Annotations[corev1.SeccompContainerAnnotationKeyPrefix+"zookeeper"] = "localhost/previous.json" + sts.Spec.Template.Annotations["example.com/unrelated"] = "keep" + if err := cl.Update(ctx, sts, client.FieldOwner("profile-config")); err != nil { + t.Fatal(err) + } + z.Spec.Pod.SecurityContext = &corev1.PodSecurityContext{ + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + template := sts.Spec.Template + templateGeneration := sts.Generation + if template.Spec.SecurityContext == nil || template.Spec.SecurityContext.SeccompProfile == nil || template.Spec.SecurityContext.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Fatal("requested seccomp profile was not applied") + } + for _, key := range []string{corev1.SeccompPodAnnotationKey, corev1.SeccompContainerAnnotationKeyPrefix + "zookeeper"} { + if value, ok := template.Annotations[key]; ok && value != corev1.SeccompProfileRuntimeDefault && value != corev1.DeprecatedSeccompProfileDockerDefault { + t.Fatalf("conflicting legacy annotation %s retained: %s", key, value) + } + } + if template.Annotations["example.com/unrelated"] != "keep" { + t.Fatal("unrelated annotation was removed") + } + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + if sts.Generation != templateGeneration { + t.Fatal("completing the ownership migration triggered a second template change") + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestMigratesLegacyLocalRunner(t *testing.T) { + for _, initialLogging := range []string{"INFO", "WARN"} { + t.Run(initialLogging, func(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Logging = api.LoggingConfig{Level: initialLogging} + z.Spec.Pod.NodeSelector = map[string]string{"pool": "old"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + sts := zk.MakeStatefulSet(z) + sts.Labels = maps.Clone(sts.Labels) + sts.Labels["owner-rv"] = z.ResourceVersion + if err := controllerutil.SetControllerReference(z, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, sts, client.FieldOwner("main")); err != nil { + t.Fatal(err) + } + z.Spec.Pod.NodeSelector = nil + if initialLogging == "INFO" { + z.Spec.Logging.Level = "WARN" + } else { + z.Spec.Logging.Level = "INFO" + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + if err := convergenceReconciler(cl).reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + template := sts.Spec.Template + if len(template.Spec.NodeSelector) != 0 || template.Spec.Volumes[0].Projected == nil { + t.Fatal("local runner did not converge scheduling and logging fields") + } + if _, present := template.Annotations["zookeeper.pravega.io/logback-hash"]; present != (z.Spec.Logging.Level == "WARN") { + t.Fatal("logging hash did not follow the requested configuration") + } + }) + } +} diff --git a/controllers/convergence_test.go b/controllers/convergence_test.go new file mode 100644 index 000000000..f9eed4173 --- /dev/null +++ b/controllers/convergence_test.go @@ -0,0 +1,588 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "testing" + + api "github.com/pravega/zookeeper-operator/api/v1beta1" + "github.com/pravega/zookeeper-operator/pkg/zk" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// workloadWrites are the StatefulSet and Service applies that every reconcile issues. +// The API server does not persist an apply that changes nothing; envtest checks this. +var workloadWrites = []string{"apply *v1.StatefulSet/", "apply *v1.Service/"} + +func TestReconcileSettledClusterSkipsStatusAndConfigWrites(t *testing.T) { + r, z, _ := newScaleMetadataTestReconciler(t, 1) + cl := r.Client.(client.WithWatch) + ctx := t.Context() + z.Spec.Replicas = 4 + z.Spec.Pod.ServiceAccountName = "example-account" + z.Spec.Conf.AdditionalConfig = map[string]string{"custom.a": "one", "custom.b": "two", "custom.c": "three"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r = convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + sts.Status.Replicas, sts.Status.ReadyReplicas = z.Spec.Replicas, 2 + sts.Status.CurrentRevision, sts.Status.UpdateRevision = "settled", "settled" + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + for i, name := range []string{"member-a", "member-b", "member-c", "member-d"} { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: z.Namespace, Labels: map[string]string{"app": z.Name}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "zookeeper", Image: "example:1"}}}, + } + if err := cl.Create(ctx, pod); err != nil { + t.Fatal(err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "zookeeper", Ready: i < 2}} + if err := cl.Status().Update(ctx, pod); err != nil { + t.Fatal(err) + } + } + settleCluster(t, ctx, r, z) + if !slices.Equal(z.Status.Members.Ready, []string{"member-a", "member-b"}) || !slices.Equal(z.Status.Members.Unready, []string{"member-c", "member-d"}) { + t.Fatalf("expected both ready and unready members in canonical order: %+v", z.Status.Members) + } + var writes []string + reverse := false + r.Client = observeMutations(interceptor.NewClient(cl, interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := c.List(ctx, list, opts...); err != nil { + return err + } + if pods, ok := list.(*corev1.PodList); ok && len(pods.Items) == 4 { + reverse = !reverse + if reverse { + slices.Reverse(pods.Items) + } + } + return nil + }, + }), &writes) + for range 5 { + runReconcile(t, ctx, r, z) + } + if unexpected := writesExcept(writes, workloadWrites...); len(unexpected) != 0 { + t.Fatalf("settled cluster must not rewrite status, config or metadata: %v", unexpected) + } +} + +func TestReconcileConflictRetries(t *testing.T) { + r, z, _ := newScaleMetadataTestReconciler(t, 1) + exerciseReconcileConflictRetries(t, t.Context(), r.Client.(client.WithWatch), z, "StatefulSet", "Service", "PodDisruptionBudget", "status") +} + +func TestUnchangedStatusRejectsStaleCluster(t *testing.T) { + r, z, _ := newScaleMetadataTestReconciler(t, 1) + key := client.ObjectKeyFromObject(z) + if err := r.Client.Get(t.Context(), key, z); err != nil { + t.Fatal(err) + } + newer := z.DeepCopy() + newer.Spec.AdminServerService.External = true + if err := r.Client.Update(t.Context(), newer); err != nil { + t.Fatal(err) + } + var writes []string + r.Client = observeMutations(r.Client.(client.WithWatch), &writes) + if err := r.patchClusterStatus(t.Context(), z); !apierrors.IsConflict(err) || len(writes) != 0 { + t.Fatalf("stale snapshot with unchanged status must conflict without writing: err=%v writes=%v", err, writes) + } +} + +func TestReconcilePreservesUnownedContainerFields(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + settleCluster(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Args = []string{"external-argument"} + if err := r.Client.Update(t.Context(), sts); err != nil { + t.Fatal(err) + } + z.Spec.Pod.NodeSelector = map[string]string{"pool": "new"} + if err := r.Client.Update(t.Context(), z); err != nil { + t.Fatal(err) + } + runReconcile(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if sts.Spec.Template.Spec.NodeSelector["pool"] != "new" { + t.Fatal("desired node selector was not applied") + } + if args := sts.Spec.Template.Spec.Containers[0].Args; len(args) != 1 || args[0] != "external-argument" { + t.Fatal("Pod-template apply replaced unowned container arguments") + } +} + +func TestReconcileAppliesOnlyRenderedFields(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + settleCluster(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Args = []string{"external-argument"} + if err := r.Client.Update(t.Context(), sts); err != nil { + t.Fatal(err) + } + z.Spec.Image.Tag = "next" + if err := r.Client.Update(t.Context(), z); err != nil { + t.Fatal(err) + } + applied := false + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok { + if patch.Type() != types.ApplyPatchType { + t.Errorf("StatefulSet must be written with server-side apply, got %s", patch.Type()) + } + data, err := patch.Data(obj) + if err != nil { + return err + } + applied = true + var body map[string]any + if err := json.Unmarshal(data, &body); err != nil { + return err + } + if _, ok := body["status"]; ok || strings.Contains(string(data), `"args"`) { + t.Errorf("apply included status or unowned container arguments: %s", data) + } + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + runReconcile(t, t.Context(), r, z) + if !applied { + t.Fatal("image change did not apply the StatefulSet") + } +} + +func TestReconcileRepairsOwnedFieldDrift(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + settleCluster(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Image = "external:drift" + if err := r.Client.Update(t.Context(), sts); err != nil { + t.Fatal(err) + } + runReconcile(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if sts.Spec.Template.Spec.Containers[0].Image != z.Spec.Image.ToString() { + t.Fatal("owned-field drift was not corrected") + } +} + +func TestReconcileRemovesOnlyPreviouslyAppliedMetadata(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + cl := r.Client.(client.WithWatch) + ctx := t.Context() + z.Spec.Pod.Annotations = map[string]string{"requested": "old"} + z.Spec.Pod.Labels["custom"] = "old" + z.Spec.ClientService.Annotations = map[string]string{"requested": "old"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + settleCluster(t, ctx, r, z) + svc := &corev1.Service{} + if err := cl.Get(ctx, client.ObjectKey{Name: z.GetClientServiceName(), Namespace: z.Namespace}, svc); err != nil { + t.Fatal(err) + } + svc.Annotations["external"] = "keep" + if err := cl.Update(ctx, svc); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Annotations["external"] = "keep" + sts.Spec.Template.Labels["external"] = "keep" + if err := cl.Update(ctx, sts); err != nil { + t.Fatal(err) + } + z.Spec.Pod.Annotations = nil + delete(z.Spec.Pod.Labels, "custom") + z.Spec.ClientService.Annotations = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil { + t.Fatal(err) + } + if _, exists := sts.Spec.Template.Annotations["requested"]; exists { + t.Error("removed CR annotation remains on Pod template") + } + if _, exists := sts.Spec.Template.Labels["custom"]; exists { + t.Error("removed CR label remains on Pod template") + } + if _, exists := svc.Annotations["requested"]; exists { + t.Error("removed CR annotation remains on Service") + } + if sts.Spec.Template.Annotations["external"] != "keep" || + sts.Spec.Template.Labels["external"] != "keep" || svc.Annotations["external"] != "keep" { + t.Fatal("removing applied keys erased unrelated metadata") + } +} + +// exerciseReconcileConflictRetries injects a concurrent write before the reconciler's write to each kind +// and checks that the stale write conflicts and the retry keeps both the concurrent and the desired change. +func exerciseReconcileConflictRetries(t *testing.T, ctx context.Context, cl client.WithWatch, z *api.ZookeeperCluster, kinds ...string) { + t.Helper() + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + for _, kind := range kinds { + t.Run(kind, func(t *testing.T) { + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + switch kind { + case "StatefulSet": + z.Spec.Pod.NodeSelector = map[string]string{"pool": "new"} + case "Service": + z.Spec.ClientService.Annotations = map[string]string{"requested": "new"} + case "PodDisruptionBudget": + z.Spec.MaxUnavailableReplicas = 2 + case "status": + z.Status.InternalClientEndpoint = "stale" + if err := cl.Status().Update(ctx, z); err != nil { + t.Fatal(err) + } + } + if kind != "status" { + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + } + injected := false + concurrent := func(ctx context.Context, c client.Client, obj client.Object, status bool) error { + matches := false + switch obj.(type) { + case *appsv1.StatefulSet: + matches = kind == "StatefulSet" + case *corev1.Service: + matches = kind == "Service" + case *policyv1.PodDisruptionBudget: + matches = kind == "PodDisruptionBudget" + case *api.ZookeeperCluster: + matches = kind == "status" && status + } + if !matches || injected { + return nil + } + injected = true + fresh := obj.DeepCopyObject().(client.Object) + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), fresh); err != nil { + return err + } + annotations := fresh.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations["external"] = "concurrent" + fresh.SetAnnotations(annotations) + return c.Update(ctx, fresh) + } + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if err := concurrent(ctx, c, obj, false); err != nil { + return err + } + return c.Update(ctx, obj, opts...) + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := concurrent(ctx, c, obj, false); err != nil { + return err + } + return c.Patch(ctx, obj, patch, opts...) + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if err := concurrent(ctx, c, obj, true); err != nil { + return err + } + return c.SubResource(sub).Update(ctx, obj, opts...) + }, + SubResourcePatch: func(ctx context.Context, c client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + if err := concurrent(ctx, c, obj, true); err != nil { + return err + } + return c.SubResource(sub).Patch(ctx, obj, patch, opts...) + }, + }) + _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(z)}) + if !injected || !apierrors.IsConflict(err) { + t.Fatalf("stale %s write must conflict: injected=%v err=%v", kind, injected, err) + } + r.Client = cl + runReconcile(t, ctx, r, z) + var obj client.Object + key := client.ObjectKeyFromObject(z) + switch kind { + case "StatefulSet": + obj = &appsv1.StatefulSet{} + case "Service": + obj = &corev1.Service{} + key.Name = z.GetClientServiceName() + case "PodDisruptionBudget": + obj = &policyv1.PodDisruptionBudget{} + default: + obj = &api.ZookeeperCluster{} + } + if err := cl.Get(ctx, key, obj); err != nil { + t.Fatal(err) + } + if obj.GetAnnotations()["external"] != "concurrent" { + t.Fatal("retry overwrote concurrent metadata") + } + switch got := obj.(type) { + case *appsv1.StatefulSet: + if got.Spec.Template.Spec.NodeSelector["pool"] != "new" { + t.Fatal("retry lost desired node selector") + } + case *corev1.Service: + if got.Annotations["requested"] != "new" { + t.Fatal("retry lost desired Service annotation") + } + case *policyv1.PodDisruptionBudget: + if got.Spec.MaxUnavailable.IntValue() != 2 { + t.Fatal("retry lost desired disruption budget") + } + case *api.ZookeeperCluster: + if got.Status.InternalClientEndpoint == "stale" { + t.Fatal("retry did not persist observed endpoint") + } + } + }) + } +} + +func TestLoggingAndUpgradeStatusRemainCorrect(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + ctx := t.Context() + cl := r.Client.(client.WithWatch) + r.APIReader = cl + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Status.Replicas, sts.Status.ReadyReplicas = 1, 1 + sts.Status.CurrentRevision, sts.Status.UpdateRevision = "old", "logging" + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + initialImage := z.Spec.Image.Tag + z.Status.SetPodsReadyConditionTrue() + z.Status.CurrentVersion = initialImage + if err := cl.Status().Update(ctx, z); err != nil { + t.Fatal(err) + } + for _, logging := range []api.LoggingConfig{{Level: "WARN", Format: "json"}, {Level: "INFO", Format: "text"}} { + z.Spec.Logging = logging + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + config := sts.Spec.Template.Annotations["zookeeper.pravega.io/logback-config"] + if !strings.Contains(config, ``) { + t.Fatalf("logging change did not reach rollout annotation: %q", config) + } + if logging.Level == "INFO" && sts.Spec.Template.Annotations["zookeeper.pravega.io/logback-hash"] != "" { + t.Fatal("logging rollback retained the non-default hash") + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + if z.Status.IsClusterInUpgradingState() || z.Status.CurrentVersion != initialImage || z.Status.TargetVersion != "" { + t.Fatalf("logging-only rollout changed image upgrade state: %+v", z.Status) + } + } + z.Spec.Image.Tag = "next" + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + sts.Status.CurrentRevision, sts.Status.UpdateRevision = "old", "new" + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + // A later failure must not discard the intentional intermediate upgrade status. + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Service); ok { + return errors.New("later Service read failed") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + if _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(z)}); err == nil { + t.Fatal("expected later read failure") + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + if !z.Status.IsClusterInUpgradingState() || z.Status.TargetVersion != "next" || z.Status.IsClusterInReadyState() { + t.Fatalf("intermediate upgrade transition was lost: %+v", z.Status) + } + r.Client = cl + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Status.CurrentRevision, sts.Status.UpdateRevision = "new", "new" + sts.Status.Replicas, sts.Status.ReadyReplicas = 1, 1 + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + if z.Status.IsClusterInUpgradingState() || z.Status.CurrentVersion != "next" || z.Status.TargetVersion != "" { + t.Fatalf("upgrade completion was lost: %+v", z.Status) + } + var writes []string + r.Client = observeMutations(cl, &writes) + runReconcile(t, ctx, r, z) + if unexpected := writesExcept(writes, workloadWrites...); len(unexpected) != 0 { + t.Fatalf("completed upgrade must settle: %v", unexpected) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + z.Status.CurrentVersion, z.Status.TargetVersion = "old", "next" + z.Status.SetUpgradingConditionTrue(api.UpdatingZookeeperReason, "0") + z.Status.SetErrorConditionTrue("UpgradeFailed", "deadline exceeded") + z.Status.SetPodsReadyConditionFalse() + if err := cl.Status().Update(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Status.ReadyReplicas = 0 + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + writes = nil + runReconcile(t, ctx, r, z) + if unexpected := writesExcept(writes, workloadWrites...); len(unexpected) != 0 { + t.Fatalf("unchanged failed upgrade must not write: %v", unexpected) + } + sts.Status.ReadyReplicas = 1 + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + r.Client = cl + settleCluster(t, ctx, r, z) + if z.Status.IsClusterInUpgradeFailedState() || z.Status.IsClusterInUpgradingState() || + z.Status.CurrentVersion != "next" || z.Status.TargetVersion != "" { + t.Fatalf("failed upgrade recovery was lost: %+v", z.Status) + } +} + +func convergenceReconciler(cl client.WithWatch) *ZookeeperClusterReconciler { + return &ZookeeperClusterReconciler{ + Client: cl, APIReader: cl, Scheme: cl.Scheme(), Log: log, + NewZkClient: func() zk.ZookeeperClient { return &MockZookeeperClient{} }, + } +} + +func runReconcile(t *testing.T, ctx context.Context, r *ZookeeperClusterReconciler, z *api.ZookeeperCluster) { + t.Helper() + if _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(z)}); err != nil { + t.Fatal(err) + } +} + +func settleCluster(t *testing.T, ctx context.Context, r *ZookeeperClusterReconciler, z *api.ZookeeperCluster) { + t.Helper() + for range 4 { + runReconcile(t, ctx, r, z) + } + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } +} + +func writesExcept(writes []string, ignored ...string) []string { + var remaining []string + for _, write := range writes { + skip := false + for _, prefix := range ignored { + skip = skip || strings.HasPrefix(write, prefix) + } + if !skip { + remaining = append(remaining, write) + } + } + return remaining +} + +func observeMutations(cl client.WithWatch, writes *[]string) client.WithWatch { + record := func(verb string, obj client.Object) { + *writes = append(*writes, fmt.Sprintf("%s %T/%s", verb, obj, obj.GetName())) + } + return interceptor.NewClient(cl, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + record("create", obj) + return c.Create(ctx, obj, opts...) + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + record("update", obj) + return c.Update(ctx, obj, opts...) + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + verb := "patch" + if patch.Type() == types.ApplyPatchType { + verb = "apply" + } + record(verb, obj) + return c.Patch(ctx, obj, patch, opts...) + }, + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + record("delete", obj) + return c.Delete(ctx, obj, opts...) + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + record(sub+" update", obj) + return c.SubResource(sub).Update(ctx, obj, opts...) + }, + SubResourcePatch: func(ctx context.Context, c client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + record(sub+" patch", obj) + return c.SubResource(sub).Patch(ctx, obj, patch, opts...) + }, + }) +} diff --git a/controllers/envtest_test.go b/controllers/envtest_test.go index ac4a99af7..1c132876c 100644 --- a/controllers/envtest_test.go +++ b/controllers/envtest_test.go @@ -7,46 +7,80 @@ package controllers import ( "context" + "errors" + "fmt" "os" "path/filepath" + "reflect" + "slices" + "strings" + "sync" "testing" "time" api "github.com/pravega/zookeeper-operator/api/v1beta1" + "github.com/pravega/zookeeper-operator/pkg/zk" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/yaml" ) +var defaultEnvtest = envtest.Environment{ + UseExistingCluster: new(bool), + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, +} + +var startDefaultEnvtest = sync.OnceValues(defaultEnvtest.Start) + +func TestMain(m *testing.M) { + code := m.Run() + if defaultEnvtest.Config != nil { + if err := defaultEnvtest.Stop(); err != nil { + fmt.Fprintf(os.Stderr, "stop shared envtest API server: %v\n", err) + code = 1 + } + } + os.Exit(code) +} + func newEnvtestCluster(t *testing.T, crds ...*apiextensionsv1.CustomResourceDefinition) (context.Context, client.WithWatch, *api.ZookeeperCluster) { t.Helper() - environment := &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - if len(crds) > 0 { - environment.CRDDirectoryPaths = nil - environment.CRDs = crds + var cfg *rest.Config + var err error + if len(crds) == 0 { + cfg, err = startDefaultEnvtest() + } else { + environment := &envtest.Environment{UseExistingCluster: new(bool), CRDs: crds} + cfg, err = environment.Start() + if err == nil { + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Errorf("stop isolated API server: %v", err) + } + }) + } } - cfg, err := environment.Start() if err != nil { t.Fatalf("start isolated API server (run make test-envtest to provision assets): %v", err) } - t.Cleanup(func() { - if err := environment.Stop(); err != nil { - t.Errorf("stop isolated API server: %v", err) - } - }) scheme := runtime.NewScheme() if err := corev1.AddToScheme(scheme); err != nil { t.Fatal(err) @@ -93,6 +127,25 @@ func newEnvtestCluster(t *testing.T, crds ...*apiextensionsv1.CustomResourceDefi t.Error(err) } } + for _, list := range []client.ObjectList{ + &appsv1.StatefulSetList{}, &corev1.ServiceList{}, &policyv1.PodDisruptionBudgetList{}, &corev1.PodList{}, + &corev1.ConfigMapList{}, &corev1.ServiceAccountList{}, &corev1.PersistentVolumeClaimList{}, + } { + if err := c.List(ctx, list, client.InNamespace(namespace.Name)); err != nil { + t.Error(err) + continue + } + items, err := meta.ExtractList(list) + if err != nil { + t.Error(err) + continue + } + for _, item := range items { + if err := client.IgnoreNotFound(c.Delete(ctx, item.(client.Object), client.GracePeriodSeconds(0))); err != nil { + t.Error(err) + } + } + } if err := c.Delete(ctx, namespace); err != nil { t.Error(err) return @@ -351,6 +404,733 @@ func TestEnvtestLegacyStorageObjectRemainsUpdatable(t *testing.T) { } } +func TestEnvtestReconcileSettles(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Replicas = 2 + z.Spec.Conf.AdditionalConfig = map[string]string{"custom.a": "one", "custom.b": "two", "custom.c": "three"} + z.Spec.Pod.Annotations = map[string]string{"custom": "set"} + z.Spec.ClientService.Annotations = map[string]string{"custom": "set"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 5) +} + +func TestEnvtestReconcileConflictRetries(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + exerciseReconcileConflictRetries(t, ctx, cl, z, "StatefulSet", "Service", "PodDisruptionBudget", "status") +} + +func TestEnvtestStatefulSetCacheMissDoesNotBypassScaleMetadata(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + key := client.ObjectKeyFromObject(z) + readStatefulSet := func() *appsv1.StatefulSet { + t.Helper() + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + return sts + } + before := readStatefulSet() + metadataErr := errors.New("metadata unavailable") + zkClient := &scaleMetadataClient{updateErr: metadataErr} + metadataAttempts := 0 + r.NewZkClient = func() zk.ZookeeperClient { + metadataAttempts++ + return zkClient + } + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, name client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok && name == key { + return apierrors.NewNotFound(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, name.Name) + } + return c.Get(ctx, name, obj, opts...) + }, + }) + if err := r.reconcileStatefulSet(ctx, z); !apierrors.IsAlreadyExists(err) { + t.Fatalf("expected the create to fail with AlreadyExists, got %v", err) + } + if got := readStatefulSet(); got.ResourceVersion != before.ResourceVersion || metadataAttempts != 0 { + t.Fatalf("cache miss changed the existing StatefulSet or attempted to update ZooKeeper: replicas=%d, metadata attempts=%d", *got.Spec.Replicas, metadataAttempts) + } + + r.Client = cl + if err := r.reconcileStatefulSet(ctx, z); !errors.Is(err, metadataErr) { + t.Fatalf("retry should attempt to update ZooKeeper before scaling: %v", err) + } + if got := readStatefulSet(); got.ResourceVersion != before.ResourceVersion { + t.Fatalf("StatefulSet changed despite the failed ZooKeeper metadata update: replicas=%d", *got.Spec.Replicas) + } + zkClient.updateErr = nil + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if got := readStatefulSet(); *got.Spec.Replicas != z.Spec.Replicas || metadataAttempts != 2 { + t.Fatalf("successful metadata retry did not scale as expected: replicas=%d, metadata attempts=%d", *got.Spec.Replicas, metadataAttempts) + } +} + +func TestEnvtestStatefulSetRejectsStaleParentSpec(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + fresh := z.DeepCopy() + fresh.Spec.Replicas = 1 + if err := cl.Update(ctx, fresh); err != nil { + t.Fatal(err) + } + var writes []string + r.Client = observeMutations(cl, &writes) + r.NewZkClient = func() zk.ZookeeperClient { + t.Fatal("stale parent must not start a resize") + return nil + } + if err := r.reconcileStatefulSet(ctx, z); !apierrors.IsConflict(err) || len(writes) != 0 { + t.Fatalf("stale parent must conflict without writing: err=%v writes=%v", err, writes) + } +} + +func TestEnvtestResizeRechecksParentWithinZooKeeperBudget(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + var sessionContext context.Context + r.NewZkClient = func() zk.ZookeeperClient { + return &resizeMetadataProbe{exists: func(zkCtx context.Context) error { + sessionContext = zkCtx + newer := z.DeepCopy() + newer.Spec.Replicas = 3 + return cl.Update(zkCtx, newer) + }, update: func() { t.Error("stale intent must not update ZooKeeper metadata") }} + } + r.APIReader = interceptor.NewClient(cl, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*api.ZookeeperCluster); ok && sessionContext != nil && ctx != sessionContext { + t.Error("post-NodeExists freshness check did not use the bounded ZooKeeper context") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + var writes []string + r.Client = observeMutations(cl, &writes) + if err := r.reconcileStatefulSet(ctx, z); sessionContext == nil || !apierrors.IsConflict(err) || len(writes) != 0 { + t.Fatalf("stale resize must conflict after NodeExists without writing: err=%v writes=%v", err, writes) + } +} + +func TestEnvtestResizeRejectsChangedStatefulSetBeforeMetadataWrite(t *testing.T) { + for _, change := range []string{"deleted", "terminating", "replaced", "updated"} { + t.Run(change, func(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + if change == "terminating" { + sts.Finalizers = []string{"example.com/hold"} + if err := cl.Update(ctx, sts); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := cl.Get(ctx, key, sts); err != nil { + t.Error(err) + return + } + sts.Finalizers = nil + if err := cl.Update(ctx, sts); err != nil { + t.Error(err) + } + }) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + changed, metadataWritten := false, false + r.NewZkClient = func() zk.ZookeeperClient { + return &resizeMetadataProbe{exists: func(ctx context.Context) error { + if change == "updated" { + sts.Annotations = map[string]string{"example.com/changed": "true"} + if err := cl.Update(ctx, sts); err != nil { + return err + } + } else if err := cl.Delete(ctx, sts); err != nil { + return err + } + if change == "replaced" { + replacement := zk.MakeStatefulSet(z) + if err := controllerutil.SetControllerReference(z, replacement, cl.Scheme()); err != nil { + return err + } + if err := cl.Create(ctx, replacement); err != nil { + return err + } + } + changed = true + return nil + }, update: func() { metadataWritten = true }} + } + err := r.reconcileStatefulSet(ctx, z) + if !changed || metadataWritten || !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) { + t.Fatalf("changed StatefulSet must stop resize before writing metadata: changed=%v metadataWritten=%v err=%v", changed, metadataWritten, err) + } + }) + } +} + +type resizeMetadataProbe struct { + MockZookeeperClient + exists func(context.Context) error + update func() +} + +func (c *resizeMetadataProbe) NodeExists(ctx context.Context, _ string) (int32, error) { + return 0, c.exists(ctx) +} + +func (c *resizeMetadataProbe) UpdateNode(context.Context, string, string, int32) error { + c.update() + return nil +} + +func TestEnvtestApplyDoesNotRecreateDeletedStatefulSet(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKeyFromObject(z) + deleted := false + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok && patch.Type() == types.ApplyPatchType && !deleted { + if err := c.Delete(ctx, obj); err != nil { + return err + } + deleted = true + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + if err := r.reconcileStatefulSet(ctx, z); err == nil || !deleted { + t.Fatalf("apply must not create after deletion: deleted=%v err=%v", deleted, err) + } + if err := cl.Get(ctx, key, &appsv1.StatefulSet{}); !apierrors.IsNotFound(err) { + t.Fatalf("StatefulSet should stay deleted until the next reconcile: %v", err) + } + r.Client = cl + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, key, &appsv1.StatefulSet{}); err != nil { + t.Fatalf("create path did not recreate the StatefulSet: %v", err) + } +} + +func TestEnvtestReconcileRejectsStatefulSetOwnershipConflict(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + sts := zk.MakeStatefulSet(z) + if err := cl.Create(ctx, sts, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + var metadataAttempts int + r.NewZkClient = func() zk.ZookeeperClient { + metadataAttempts++ + return &MockZookeeperClient{} + } + for _, scenario := range []string{"no controller", "previous cluster UID", "failed upgrade"} { + t.Run(scenario, func(t *testing.T) { + switch scenario { + case "previous cluster UID": + previous := z.DeepCopy() + previous.UID = "previous-cluster" + if err := controllerutil.SetControllerReference(previous, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Update(ctx, sts); err != nil { + t.Fatal(err) + } + case "failed upgrade": + z.Status.SetErrorConditionTrue("UpgradeFailed", "upgrade timed out") + if err := cl.Status().Update(ctx, z); err != nil { + t.Fatal(err) + } + } + if err := r.reconcileStatefulSet(ctx, z); err == nil || !strings.Contains(err.Error(), "is not controlled by ZookeeperCluster") { + t.Fatalf("expected an ownership error, got %v", err) + } + got := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), got); err != nil { + t.Fatal(err) + } + if got.ResourceVersion != sts.ResourceVersion || metadataAttempts != 0 { + t.Fatalf("ownership conflict caused a StatefulSet write or ZooKeeper metadata update: metadata attempts=%d", metadataAttempts) + } + if scenario == "failed upgrade" && !z.Status.IsClusterInUpgradeFailedState() { + t.Fatal("an unrelated StatefulSet cleared the failed upgrade state") + } + }) + } +} + +func TestEnvtestReconcileRejectsServiceNameCollision(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + for _, test := range []struct { + name string + svc *corev1.Service + reconcile reconcileFun + }{ + {"unowned client Service", zk.MakeClientService(z), r.reconcileClientService}, + {"headless Service from previous cluster UID", zk.MakeHeadlessService(z), r.reconcileHeadlessService}, + {"concurrently created admin Service", zk.MakeAdminServerService(z), r.reconcileAdminServerService}, + } { + t.Run(test.name, func(t *testing.T) { + svc := test.svc + svc.Spec.Selector["app"] = "another-app" + key := client.ObjectKeyFromObject(svc) + if svc.Spec.ClusterIP == corev1.ClusterIPNone { + previous := z.DeepCopy() + previous.UID = "previous-cluster" + if err := controllerutil.SetControllerReference(previous, svc, cl.Scheme()); err != nil { + t.Fatal(err) + } + } + concurrent := svc.Name == z.GetAdminServerServiceName() + if concurrent { + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, name client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if name != key { + return c.Get(ctx, name, obj, opts...) + } + if err := c.Create(ctx, svc, client.FieldOwner("zookeeper-operator")); err != nil { + return err + } + return apierrors.NewNotFound(schema.GroupResource{Resource: "services"}, name.Name) + }, + }) + defer func() { r.Client = cl }() + } else if err := cl.Create(ctx, svc, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + err := test.reconcile(ctx, z) + if concurrent { + if !apierrors.IsAlreadyExists(err) || svc.ResourceVersion == "" { + t.Fatalf("concurrent create must return AlreadyExists, got %v", err) + } + } else if err == nil || !strings.Contains(err.Error(), "is not controlled by ZookeeperCluster") { + t.Fatalf("expected an ownership error, got %v", err) + } + got := &corev1.Service{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatal(err) + } + if got.ResourceVersion != svc.ResourceVersion { + t.Fatalf("Service changed despite the name collision: selector %v, owners %v", got.Spec.Selector, got.OwnerReferences) + } + }) + } +} + +func TestEnvtestReconcileZookeeperEnvOverride(t *testing.T) { + for _, mode := range []string{"new", "legacy"} { + t.Run(mode, func(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + optional := true + env := []corev1.EnvVar{ + {Name: "ORIGINAL_STATUS", Value: "$(ENVOY_SIDECAR_STATUS)"}, + {Name: "BASE", Value: "/srv"}, + {Name: "BASE", Value: "$(BASE)/data"}, + {Name: "ENVOY_SIDECAR_STATUS", ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "optional-config"}, Key: "status", Optional: &optional, + }}}, + } + if mode == "legacy" { + sts := zk.MakeStatefulSet(z) + container := &sts.Spec.Template.Spec.Containers[0] + // The previous operator appended user variables to the built-in variable. + container.Env = append(container.Env, env...) + if err := controllerutil.SetControllerReference(z, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, sts, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + } + z.Spec.Pod.Env = env + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + got := sts.Spec.Template.Spec.Containers[0].Env + if len(got) != len(env)+1 || got[0].Name != "ENVOY_SIDECAR_STATUS" || got[0].ValueFrom == nil || got[0].ValueFrom.FieldRef == nil || !reflect.DeepEqual(got[1:], env) { + t.Fatalf("expected the generated fallback and the exact ordered env overrides: got %+v", got) + } + fallback := got[0] + z.Spec.Pod.Env = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + if got := sts.Spec.Template.Spec.Containers[0].Env; !reflect.DeepEqual(got, []corev1.EnvVar{fallback}) { + t.Fatalf("removing duplicate overrides should leave only the generated fallback: got %+v", got) + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + }) + } +} + +func TestEnvtestReconcileRestoresInternalAdminService(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + admin := &corev1.Service{} + key := client.ObjectKey{Namespace: z.Namespace, Name: z.GetAdminServerServiceName()} + if err := cl.Get(ctx, key, admin); err != nil { + t.Fatal(err) + } + admin.Spec.Type = corev1.ServiceTypeLoadBalancer + admin.Spec.LoadBalancerSourceRanges = []string{"192.0.2.0/24"} + if err := cl.Update(ctx, admin, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, admin); err != nil { + t.Fatal(err) + } + if admin.Spec.Type != corev1.ServiceTypeClusterIP || admin.Spec.Ports[0].NodePort != 0 || len(admin.Spec.LoadBalancerSourceRanges) != 0 { + t.Fatalf("expected an internal admin Service without a NodePort or load balancer source ranges, got %+v", admin.Spec) + } +} + +func TestEnvtestReconcileDefaultsConverge(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Containers = []corev1.Container{{ + Name: "sidecar", Image: "busybox:1", + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1.1m"), + }}, + StartupProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{ + GRPC: &corev1.GRPCAction{Port: 9090}, + }}, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromInt32(8080)}, + }}, + }} + z.Spec.Volumes = []corev1.Volume{{ + Name: "secret", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "example"}, + }, + }, { + Name: "empty", + }} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + patches := map[types.PatchType]int{} + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok { + patches[patch.Type()]++ + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + const passes = 3 + assertEnvtestSettled(t, ctx, r, cl, z, passes) + if compatibility, apply := patches[types.MergePatchType], patches[types.ApplyPatchType]; compatibility != 0 || apply != passes { + t.Fatalf("settled reconciles sent %d StatefulSet compatibility patches and %d applies, want 0 and %d", compatibility, apply, passes) + } + r.Client = cl + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + if sts.Spec.Template.Spec.DNSPolicy != corev1.DNSClusterFirst || + sts.Spec.Template.Spec.Containers[1].TerminationMessagePath != "/dev/termination-log" { + t.Fatal("test requires real API-defaulted Pod-template fields") + } + if !slices.ContainsFunc(sts.Spec.Template.Spec.Volumes, func(v corev1.Volume) bool { + return v.Name == "empty" && v.EmptyDir != nil + }) { + t.Fatal("test requires the API to default the sourceless volume to EmptyDir") + } + svc := &corev1.Service{} + svcKey := types.NamespacedName{Name: z.GetClientServiceName(), Namespace: z.Namespace} + if err := cl.Get(ctx, svcKey, svc); err != nil { + t.Fatal(err) + } + if svc.Spec.Ports[0].Protocol != corev1.ProtocolTCP || svc.Spec.Ports[0].TargetPort.IntVal != 2181 { + t.Fatal("test requires real API-defaulted Service ports") + } + for _, external := range []bool{true, false} { + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + z.Spec.AdminServerService.External = external + z.Spec.DomainName = "" + if external { + z.Spec.DomainName = "example.test" + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + admin := &corev1.Service{} + if err := cl.Get(ctx, client.ObjectKey{Name: z.GetAdminServerServiceName(), Namespace: z.Namespace}, admin); err != nil { + t.Fatal(err) + } + if external { + if admin.Spec.Type != corev1.ServiceTypeLoadBalancer || admin.Spec.Ports[0].NodePort == 0 { + t.Fatal("external admin service did not get an allocated NodePort") + } + } else if admin.Spec.Type != corev1.ServiceTypeClusterIP || admin.Spec.Ports[0].NodePort != 0 { + t.Fatal("removing the external service did not clear its NodePort") + } + headless := &corev1.Service{} + if err := cl.Get(ctx, client.ObjectKey{Name: z.Name + "-headless", Namespace: z.Namespace}, headless); err != nil { + t.Fatal(err) + } + if _, hasDNS := headless.Annotations["external-dns.alpha.kubernetes.io/hostname"]; hasDNS != external { + t.Fatal("external-DNS annotation change or removal did not converge") + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + } +} + +func TestEnvtestReconcileAdoptsLegacyResources(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + createLegacyResources(t, ctx, cl, z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Args = []string{"external-argument"} + if err := cl.Update(ctx, sts, client.FieldOwner("external-injector")); err != nil { + t.Fatal(err) + } + clientSvc := &corev1.Service{} + clientKey := client.ObjectKey{Name: z.GetClientServiceName(), Namespace: z.Namespace} + if err := cl.Get(ctx, clientKey, clientSvc); err != nil { + t.Fatal(err) + } + clientSvc.Annotations = map[string]string{"external-dns.alpha.kubernetes.io/hostname": "zk.user.example"} + clientSvc.Spec.Ports = append(clientSvc.Spec.Ports, corev1.ServicePort{Name: "extra", Port: 8081}) + if err := cl.Update(ctx, clientSvc, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + generation, clusterIP := sts.Generation, clientSvc.Spec.ClusterIP + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if sts.Generation != generation { + t.Fatalf("adopting a legacy StatefulSet changed its spec: generation %d -> %d", generation, sts.Generation) + } + if _, ok := sts.Labels["owner-rv"]; ok { + t.Error("legacy owner-rv label was not removed") + } + if args := sts.Spec.Template.Spec.Containers[0].Args; len(args) != 1 || args[0] != "external-argument" { + t.Error("adoption removed a field owned by another manager") + } + for _, entry := range sts.ManagedFields { + if legacyFieldManagers.Has(entry.Manager) { + t.Errorf("legacy field manager %q was not migrated", entry.Manager) + } + } + if err := cl.Get(ctx, clientKey, clientSvc); err != nil { + t.Fatal(err) + } + if clientSvc.Spec.ClusterIP != clusterIP { + t.Error("adoption changed the allocated cluster IP") + } + if clientSvc.Annotations["external-dns.alpha.kubernetes.io/hostname"] != "zk.user.example" { + t.Error("adoption removed a user-owned external-DNS annotation") + } + if !slices.ContainsFunc(clientSvc.Spec.Ports, func(port corev1.ServicePort) bool { + return port.Name == "extra" && port.Port == 8081 + }) { + t.Error("adoption removed a user-owned Service port") + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestReconcileLegacyOwnedFieldRemoval(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Pod.NodeSelector = map[string]string{"pool": "old"} + z.Spec.Pod.Env = []corev1.EnvVar{{Name: "OWNED", Value: "remove"}} + z.Spec.Pod.Annotations = map[string]string{"custom": "remove"} + z.Spec.Containers = []corev1.Container{ + {Name: "sidecar", Image: "busybox:1"}, + {Name: "retired", Image: "busybox:1"}, + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + createLegacyResources(t, ctx, cl, z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + for i := range sts.Spec.Template.Spec.Containers { + container := &sts.Spec.Template.Spec.Containers[i] + if container.Name == "sidecar" { + container.Args = []string{"external-argument"} + } + if container.Name == "zookeeper" { + container.Env = append(container.Env, corev1.EnvVar{Name: "EXTERNAL", Value: "keep"}) + } + } + if err := cl.Update(ctx, sts, client.FieldOwner("external-injector")); err != nil { + t.Fatal(err) + } + // The CR changes before the new operator first reconciles the legacy StatefulSet. + z.Spec.Pod.NodeSelector = nil + z.Spec.Pod.Env = nil + z.Spec.Pod.Annotations = nil + z.Spec.Containers = []corev1.Container{{Name: "sidecar", Image: "busybox:2"}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if len(sts.Spec.Template.Spec.NodeSelector) != 0 || len(sts.Spec.Template.Spec.Containers) != 2 { + t.Fatal("fields the legacy operator rendered were not removed") + } + if _, ok := sts.Spec.Template.Annotations["custom"]; ok { + t.Error("Pod annotation the legacy operator rendered was not removed") + } + externalEnv := false + for _, container := range sts.Spec.Template.Spec.Containers { + if container.Name == "sidecar" && + (container.Image != "busybox:2" || len(container.Args) != 1 || container.Args[0] != "external-argument") { + t.Error("sidecar image change erased an unowned container field") + } + for _, env := range container.Env { + if env.Name == "OWNED" { + t.Error("environment entry the legacy operator rendered was not removed") + } + externalEnv = externalEnv || env.Name == "EXTERNAL" + } + } + if !externalEnv { + t.Error("removing an owned environment entry erased an unowned entry") + } + for _, logging := range []api.LoggingConfig{{Level: "WARN", Format: "json"}, {Level: "INFO", Format: "text"}} { + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + z.Spec.Logging = logging + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +// createLegacyResources creates the StatefulSet and Services as earlier operator versions did: +// with Create requests from the legacy field manager and the owner-rv label. +func createLegacyResources(t *testing.T, ctx context.Context, cl client.Client, z *api.ZookeeperCluster) { + t.Helper() + sts := zk.MakeStatefulSet(z) + zk.EnsureLogbackProjection(z, nil, sts) + sts.Labels = map[string]string{"owner-rv": z.ResourceVersion} + for key, value := range z.Spec.Labels { + sts.Labels[key] = value + } + objects := []client.Object{sts, zk.MakeClientService(z), zk.MakeHeadlessService(z), zk.MakeAdminServerService(z)} + for _, obj := range objects { + if err := controllerutil.SetControllerReference(z, obj, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, obj, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + } +} + +// assertEnvtestSettled checks that further reconciles do not change any persisted object. +func assertEnvtestSettled(t *testing.T, ctx context.Context, r *ZookeeperClusterReconciler, cl client.Client, z *api.ZookeeperCluster, passes int) { + t.Helper() + settled := envtestResourceVersions(t, ctx, cl, z) + for range passes { + runReconcile(t, ctx, r, z) + } + if got := envtestResourceVersions(t, ctx, cl, z); !reflect.DeepEqual(got, settled) { + t.Fatalf("settled reconciliation changed persisted objects:\nbefore %v\nafter %v", settled, got) + } +} + +// envtestResourceVersions returns the resourceVersion of every object the reconciler writes. +func envtestResourceVersions(t *testing.T, ctx context.Context, cl client.Client, z *api.ZookeeperCluster) map[string]string { + t.Helper() + objects := map[string]client.Object{ + "cluster": &api.ZookeeperCluster{ObjectMeta: metav1.ObjectMeta{Name: z.Name}}, + "statefulset": &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: z.Name}}, + "configmap": &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: z.ConfigMapName()}}, + "pdb": &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: z.Name}}, + "client": &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: z.GetClientServiceName()}}, + "headless": &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: z.Name + "-headless"}}, + "admin": &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: z.GetAdminServerServiceName()}}, + } + versions := map[string]string{} + for name, obj := range objects { + if err := cl.Get(ctx, client.ObjectKey{Name: obj.GetName(), Namespace: z.Namespace}, obj); err != nil { + t.Fatalf("get %s: %v", name, err) + } + versions[name] = obj.GetResourceVersion() + } + return versions +} + func TestEnvtestStatusSubresource(t *testing.T) { ctx, c, cluster := newEnvtestCluster(t) cluster.Status.ReadyReplicas = 1 diff --git a/controllers/legacy_local_runner_envtest_test.go b/controllers/legacy_local_runner_envtest_test.go new file mode 100644 index 000000000..5ef36db34 --- /dev/null +++ b/controllers/legacy_local_runner_envtest_test.go @@ -0,0 +1,47 @@ +//go:build envtest + +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "maps" + "slices" + "testing" + + "github.com/pravega/zookeeper-operator/pkg/zk" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/csaupgrade" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" +) + +func TestEnvtestPreservesUnrelatedMainManager(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + sts := zk.MakeStatefulSet(z) + sts.Labels = maps.Clone(sts.Labels) + sts.Labels["owner-rv"] = z.ResourceVersion + if err := controllerutil.SetControllerReference(z, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, sts, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + const key = "example.com/unrelated" + sts.Annotations = map[string]string{key: "keep"} + if err := cl.Update(ctx, sts, client.FieldOwner("main")); err != nil { + t.Fatal(err) + } + if err := convergenceReconciler(cl).reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + owners := csaupgrade.FindFieldsOwners(sts.ManagedFields, metav1.ManagedFieldsOperationUpdate, fieldpath.NewSet(fieldpath.MakePathOrDie("metadata", "annotations", key))) + if sts.Annotations[key] != "keep" || !slices.ContainsFunc(owners, func(owner metav1.ManagedFieldsEntry) bool { return owner.Manager == "main" }) { + t.Fatal("an unrelated main manager must retain its annotation and ownership") + } +} diff --git a/controllers/server_side_apply.go b/controllers/server_side_apply.go new file mode 100644 index 000000000..62ac25334 --- /dev/null +++ b/controllers/server_side_apply.go @@ -0,0 +1,160 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "slices" + + "github.com/pravega/zookeeper-operator/api/v1beta1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/csaupgrade" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" +) + +// applyFieldManager owns the StatefulSet and Service fields that the operator renders. +const applyFieldManager = "zookeeper-operator-apply" + +// StatefulSets and Services use create-only requests so cache misses cannot overwrite live objects. +const createFieldManager = "zookeeper-operator-create" + +// Fields owned by Create and Update managers move to applyFieldManager so that an apply +// can remove fields the operator no longer renders. +var legacyFieldManagers = sets.New("zookeeper-operator", createFieldManager) + +func legacyManagersFor(current client.Object) sets.Set[string] { + // make run-local wrote as "main". Only migrate that generic manager when + // it owns the legacy operator's StatefulSet bookkeeping label. + if _, ok := current.(*appsv1.StatefulSet); ok && current.GetLabels()["owner-rv"] != "" { + marker := fieldpath.NewSet(fieldpath.MakePathOrDie("metadata", "labels", "owner-rv")) + for _, owner := range csaupgrade.FindFieldsOwners(current.GetManagedFields(), metav1.ManagedFieldsOperationUpdate, marker) { + if owner.Manager == "main" && owner.Subresource == "" { + return legacyFieldManagers.Union(sets.New("main")) + } + } + } + return legacyFieldManagers +} + +// applyStatefulSet updates an existing StatefulSet without changing its immutable fields. +func (r *ZookeeperClusterReconciler) applyStatefulSet(ctx context.Context, current, desired *appsv1.StatefulSet) error { + intent := desired.DeepCopy() + intent.Spec.Selector = current.Spec.Selector + intent.Spec.ServiceName = current.Spec.ServiceName + intent.Spec.PodManagementPolicy = current.Spec.PodManagementPolicy + intent.Spec.VolumeClaimTemplates = current.Spec.VolumeClaimTemplates + operatorFields, externalFields := statefulSetFields(current) + defaultRenderedPullPolicies(&intent.Spec.Template.Spec, externalFields) + prepared := current.DeepCopy() + retired := removeSharedRetiredContainers(&prepared.Spec.Template.Spec, &desired.Spec.Template.Spec, operatorFields, externalFields) + portDrift := removeStaleContainerPorts(&prepared.Spec.Template.Spec, &desired.Spec.Template.Spec) + orderedLists := normalizeContainerLists(&intent.Spec.Template.Spec) + sourceDrift := clearIncompatibleSources(&prepared.Spec.Template.Spec, &intent.Spec.Template.Spec) + securityDrift := clearInheritedSecurityOverrides(&prepared.Spec.Template.Spec, &intent.Spec.Template.Spec) + securityDrift = clearConflictingSeccompAnnotations(&prepared.Spec.Template, &intent.Spec.Template) || securityDrift + retiringVolumes := retiringVolumesForMerge(&prepared.Spec.Template.Spec, &intent.Spec.Template.Spec, operatorFields, externalFields) + if orderedLists || portDrift || sourceDrift || retired || securityDrift || len(retiringVolumes) != 0 { + return r.applyCompatibleStatefulSet(ctx, current, prepared, intent, desired, retiringVolumes) + } + return r.apply(ctx, current, intent) +} + +// applyService updates an existing Service only if this cluster controls it. +func (r *ZookeeperClusterReconciler) applyService(ctx context.Context, cluster *v1beta1.ZookeeperCluster, current, desired *corev1.Service) error { + if err := requireClusterOwner(cluster, current, "service"); err != nil { + return err + } + intent := desired.DeepCopy() + prepared := current.DeepCopy() + external := intent.Spec.Type == corev1.ServiceTypeLoadBalancer + if external && (current.Spec.Type == corev1.ServiceTypeLoadBalancer || current.Spec.Type == corev1.ServiceTypeNodePort) { + // Carry the allocation by name even when the numeric port key changes. + // Keep it in later applies so ownership migration cannot prune it. + for i := range intent.Spec.Ports { + port := &intent.Spec.Ports[i] + if j := slices.IndexFunc(current.Spec.Ports, func(live corev1.ServicePort) bool { return live.Name == port.Name }); port.NodePort == 0 && j >= 0 { + port.NodePort = current.Spec.Ports[j].NodePort + } + } + } + prepared.Spec.Ports = slices.DeleteFunc(prepared.Spec.Ports, func(live corev1.ServicePort) bool { + if slices.ContainsFunc(intent.Spec.Ports, func(wanted corev1.ServicePort) bool { + return servicePortKey(live) == servicePortKey(wanted) && live.Name == wanted.Name + }) { + return false + } + return external || (live.Name != "" && slices.ContainsFunc(intent.Spec.Ports, func(wanted corev1.ServicePort) bool { return live.Name == wanted.Name })) + }) + const sourceRanges = "service.beta.kubernetes.io/load-balancer-source-ranges" + incompatibleRanges := false + if !external { + _, hasAnnotation := prepared.Annotations[sourceRanges] + incompatibleRanges = hasAnnotation || len(prepared.Spec.LoadBalancerSourceRanges) != 0 + delete(prepared.Annotations, sourceRanges) + delete(intent.Annotations, sourceRanges) + prepared.Spec.LoadBalancerSourceRanges = nil + } + if len(current.Spec.Ports) == len(prepared.Spec.Ports) && !incompatibleRanges { + return r.apply(ctx, current, intent) + } + // Publish the type, ports, and annotations together or not at all. + next := &corev1.Service{} + if err := r.mergeWorkloadLocally(current, prepared, intent, next); err != nil { + return err + } + next.Status = current.Status + if err := r.patchMergedWorkload(ctx, current, next); err != nil { + return err + } + *current = *next + return nil +} + +func servicePortKey(port corev1.ServicePort) portIdentity { return portKey(port.Port, port.Protocol) } + +func requireClusterOwner(cluster *v1beta1.ZookeeperCluster, obj client.Object, kind string) error { + if !metav1.IsControlledBy(obj, cluster) { + return fmt.Errorf("%s %s already exists and is not controlled by ZookeeperCluster %s", kind, client.ObjectKeyFromObject(obj), client.ObjectKeyFromObject(cluster)) + } + return nil +} + +// apply updates the observed object; its UID prevents apply from recreating a deleted object. +func (r *ZookeeperClusterReconciler) apply(ctx context.Context, current, intent client.Object) error { + if current.GetResourceVersion() == "" { + return fmt.Errorf("cannot apply %T %s without a resource version", current, client.ObjectKeyFromObject(current)) + } + migration, err := csaupgrade.UpgradeManagedFieldsPatch(current, legacyManagersFor(current), applyFieldManager) + if err != nil { + return err + } + if migration != nil { + if err := r.Client.Patch(ctx, current, client.RawPatch(types.JSONPatchType, migration)); err != nil { + return err + } + } + intent.SetResourceVersion(current.GetResourceVersion()) + intent.SetUID(current.GetUID()) + data, err := json.Marshal(intent) + if err != nil { + return err + } + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return err + } + delete(body, "status") + if data, err = json.Marshal(body); err != nil { + return err + } + return r.Client.Patch(ctx, current, client.RawPatch(types.ApplyPatchType, data), + client.FieldOwner(applyFieldManager), client.ForceOwnership) +} diff --git a/controllers/server_side_apply_test.go b/controllers/server_side_apply_test.go new file mode 100644 index 000000000..b119d8bcd --- /dev/null +++ b/controllers/server_side_apply_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "testing" + + "github.com/pravega/zookeeper-operator/pkg/zk" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestApplyPreservesIntegerValues(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + ctx := t.Context() + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + const gracePeriod = 9007199254740993 + z.Spec.Pod.TerminationGracePeriodSeconds = gracePeriod + applies := 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Patch: func(_ context.Context, _ client.WithWatch, obj client.Object, patch client.Patch, _ ...client.PatchOption) error { + if patch.Type() != types.ApplyPatchType { + t.Fatalf("expected apply, got %v", patch.Type()) + } + applies++ + data, err := patch.Data(obj) + if err != nil { + return err + } + intent := &appsv1.StatefulSet{} + if err := json.Unmarshal(data, intent); err != nil { + return err + } + if got := intent.Spec.Template.Spec.TerminationGracePeriodSeconds; got == nil || *got != gracePeriod { + t.Errorf("apply did not preserve the rendered grace period: %s", data) + } + return nil + }, + }) + if err := r.applyStatefulSet(ctx, sts, zk.MakeStatefulSet(z)); err != nil { + t.Fatal(err) + } + if applies != 1 { + t.Fatalf("expected one apply payload, got %d", applies) + } +} diff --git a/controllers/statefulset_compatibility.go b/controllers/statefulset_compatibility.go new file mode 100644 index 000000000..7542fbb28 --- /dev/null +++ b/controllers/statefulset_compatibility.go @@ -0,0 +1,216 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "slices" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/managedfields" + applyconfigs "k8s.io/client-go/applyconfigurations" + "k8s.io/client-go/util/csaupgrade" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type portIdentity struct { + number int32 + protocol corev1.Protocol +} + +func portKey(number int32, protocol corev1.Protocol) portIdentity { + if protocol == "" { + protocol = corev1.ProtocolTCP + } + return portIdentity{number, protocol} +} + +func containerPortKey(port corev1.ContainerPort) portIdentity { + return portKey(port.ContainerPort, port.Protocol) +} + +func removeStaleContainerPorts(live, desired *corev1.PodSpec) bool { + changed := false + forRenderedContainers(live, desired, func(current, wanted *corev1.Container) { + current.Ports = slices.DeleteFunc(current.Ports, func(port corev1.ContainerPort) bool { + stale := port.Name != "" && slices.ContainsFunc(wanted.Ports, func(next corev1.ContainerPort) bool { + return next.Name == port.Name && containerPortKey(next) != containerPortKey(port) + }) + changed = changed || stale + return stale + }) + }) + return changed +} + +func normalizeContainerLists(spec *corev1.PodSpec) bool { + changed := false + for _, containers := range [][]corev1.Container{spec.Containers, spec.InitContainers} { + for i := range containers { + c := &containers[i] + ports, env := len(c.Ports), len(c.Env) + c.Ports = uniqueBy(c.Ports, containerPortKey) + c.Env = uniqueBy(c.Env, envKey) + changed = changed || len(c.Ports) != ports || len(c.Env) != env + } + } + return changed +} + +func envKey(env corev1.EnvVar) string { return env.Name } + +func uniqueBy[T any, K comparable](values []T, key func(T) K) []T { + seen := make(map[K]bool, len(values)) + return slices.DeleteFunc(values, func(value T) bool { + id := key(value) + previous := seen[id] + seen[id] = true + return previous + }) +} + +// Apply cannot express duplicate list keys. Merge an SSA-compatible intent locally, +// then restore the exact ordered lists and let the API server validate one guarded patch. +func (r *ZookeeperClusterReconciler) applyCompatibleStatefulSet(ctx context.Context, current, prepared, intent, desired *appsv1.StatefulSet, retiringVolumes []corev1.Volume) error { + next := &appsv1.StatefulSet{} + if err := r.mergeWorkloadLocally(current, prepared, intent, next); err != nil { + return err + } + if resolveRetiringVolumes(prepared, intent, next, retiringVolumes) { + // Keep complete sources while referenced; don't persist orphaned fragments. + next = &appsv1.StatefulSet{} + if err := r.mergeWorkloadLocally(current, prepared, intent, next); err != nil { + return err + } + } + restoreContainerLists(current.Spec.Template.Spec.Containers, next.Spec.Template.Spec.Containers, desired.Spec.Template.Spec.Containers, intent.Spec.Template.Spec.Containers) + restoreContainerLists(current.Spec.Template.Spec.InitContainers, next.Spec.Template.Spec.InitContainers, desired.Spec.Template.Spec.InitContainers, intent.Spec.Template.Spec.InitContainers) + next.Status = current.Status + if err := r.patchMergedWorkload(ctx, current, next); err != nil { + return err + } + *current = *next + return nil +} + +func (r *ZookeeperClusterReconciler) mergeWorkloadLocally(current, prepared, intent, next client.Object) error { + gvk := intent.GetObjectKind().GroupVersionKind() + prepared.GetObjectKind().SetGroupVersionKind(gvk) + if err := csaupgrade.UpgradeManagedFields(prepared, legacyManagersFor(current), applyFieldManager); err != nil { + return err + } + manager, err := managedfields.NewDefaultFieldManager(applyconfigs.NewTypeConverter(r.Scheme), r.Scheme, r.Scheme, r.Scheme, gvk, gvk.GroupVersion(), "", nil) + if err != nil { + return err + } + payload, err := runtime.DefaultUnstructuredConverter.ToUnstructured(intent) + if err != nil { + return err + } + delete(payload, "status") + merged, err := manager.Apply(prepared, &unstructured.Unstructured{Object: payload}, applyFieldManager, true) + if err != nil { + return err + } + data, err := json.Marshal(merged) + if err != nil { + return err + } + return json.Unmarshal(data, next) +} + +func (r *ZookeeperClusterReconciler) patchMergedWorkload(ctx context.Context, current, next client.Object) error { + next.GetObjectKind().SetGroupVersionKind(current.GetObjectKind().GroupVersionKind()) + next.SetResourceVersion(current.GetResourceVersion()) + if equality.Semantic.DeepEqual(current, next) { + return nil + } + // Changes made by this non-apply PATCH must migrate back to the apply manager. + return r.Client.Patch(ctx, next, client.MergeFromWithOptions(current, client.MergeFromWithOptimisticLock{}), client.FieldOwner(createFieldManager)) +} + +func restoreContainerLists(live, merged, desired, normalized []corev1.Container) { + for i, container := range desired { + restorePorts := len(container.Ports) != len(normalized[i].Ports) + restoreEnv := len(container.Env) != len(normalized[i].Env) + if !restorePorts && !restoreEnv { + continue + } + for j := range merged { + if merged[j].Name != container.Name { + continue + } + var previous corev1.Container + if at := slices.IndexFunc(live, func(c corev1.Container) bool { return c.Name == container.Name }); at >= 0 { + previous = live[at] + } + if restorePorts { + merged[j].Ports = restoreList(previous.Ports, merged[j].Ports, container.Ports, containerPortKey) + } + if restoreEnv { + merged[j].Env = restoreList(previous.Env, merged[j].Env, container.Env, envKey) + } + break + } + } +} + +func restoreList[T any, K comparable](live, merged, desired []T, key func(T) K) []T { + positions := make(map[K][]int, len(desired)) + for i, value := range desired { + id := key(value) + positions[id] = append(positions[id], i) + } + resolved := slices.Clone(desired) + extra := make(map[K][]int) + for i, value := range merged { + id := key(value) + switch len(positions[id]) { + case 0: + extra[id] = append(extra[id], i) + case 1: + resolved[positions[id][0]] = value + } + } + // Anchor independent entries before the next rendered occurrence in the + // live list. Preserve their order even when the rendered order changes. + gaps := make(map[int][]T) + used := make([]bool, len(merged)) + anchor := len(resolved) + for i := len(live) - 1; i >= 0; i-- { + id := key(live[i]) + if indexes := positions[id]; len(indexes) != 0 { + anchor = min(anchor, indexes[len(indexes)-1]) + if len(indexes) > 1 { + positions[id] = indexes[:len(indexes)-1] + } + } else if indexes := extra[id]; len(indexes) != 0 { + index := indexes[len(indexes)-1] + gaps[anchor] = append(gaps[anchor], merged[index]) + used[index] = true + extra[id] = indexes[:len(indexes)-1] + } + } + result := make([]T, 0, len(resolved)) + for i, value := range resolved { + gap := gaps[i] + slices.Reverse(gap) + result = append(result, gap...) + result = append(result, value) + } + tail := gaps[len(resolved)] + slices.Reverse(tail) + result = append(result, tail...) + for i, value := range merged { + if !used[i] && len(positions[key(value)]) == 0 { + result = append(result, value) + } + } + return result +} diff --git a/controllers/statefulset_repairs.go b/controllers/statefulset_repairs.go new file mode 100644 index 000000000..489ab2444 --- /dev/null +++ b/controllers/statefulset_repairs.go @@ -0,0 +1,343 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "reflect" + "slices" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" +) + +func clearInheritedSecurityOverrides(live, intent *corev1.PodSpec) bool { + pod := intent.SecurityContext + if pod == nil { + pod = &corev1.PodSecurityContext{} + } + changed := false + forRenderedContainers(live, intent, func(current, desired *corev1.Container) { + container := current.SecurityContext + if container == nil { + return + } + requested := desired.SecurityContext + if requested == nil { + requested = &corev1.SecurityContext{} + } + changed = clearOverride(&container.RunAsUser, pod.RunAsUser, requested.RunAsUser) || changed + changed = clearOverride(&container.RunAsGroup, pod.RunAsGroup, requested.RunAsGroup) || changed + changed = clearOverride(&container.RunAsNonRoot, pod.RunAsNonRoot, requested.RunAsNonRoot) || changed + changed = clearOverride(&container.SELinuxOptions, pod.SELinuxOptions, requested.SELinuxOptions) || changed + changed = clearOverride(&container.WindowsOptions, pod.WindowsOptions, requested.WindowsOptions) || changed + changed = clearOverride(&container.SeccompProfile, pod.SeccompProfile, requested.SeccompProfile) || changed + changed = clearOverride(&container.AppArmorProfile, pod.AppArmorProfile, requested.AppArmorProfile) || changed + if nonRoot := preferExplicit(requested.RunAsNonRoot, pod.RunAsNonRoot); nonRoot != nil && *nonRoot && requested.RunAsUser == nil && container.RunAsUser != nil && *container.RunAsUser == 0 { + container.RunAsUser = nil + changed = true + } + seccomp := preferExplicit(requested.SeccompProfile, pod.SeccompProfile) + apparmor := preferExplicit(requested.AppArmorProfile, pod.AppArmorProfile) + confinement := seccomp != nil && seccomp.Type != corev1.SeccompProfileTypeUnconfined || apparmor != nil && apparmor.Type != corev1.AppArmorProfileTypeUnconfined + if confinement && requested.Privileged == nil && container.Privileged != nil && *container.Privileged { + container.Privileged = nil + changed = true + } + }) + return changed +} + +func preferExplicit[T any](explicit, inherited *T) *T { + if explicit != nil { + return explicit + } + return inherited +} + +func clearOverride[T any](live **T, inherited, explicit *T) bool { + if *live == nil || inherited == nil || explicit != nil || equality.Semantic.DeepEqual(*live, inherited) { + return false + } + *live = nil + return true +} + +func clearConflictingSeccompAnnotations(live, intent *corev1.PodTemplateSpec) bool { + if len(live.Annotations) == 0 { + return false + } + changed := false + clear := func(key string, profile *corev1.SeccompProfile) { + annotation, exists := live.Annotations[key] + if _, explicit := intent.Annotations[key]; exists && !explicit && profile != nil && !matchesSeccompProfile(annotation, profile) { + delete(live.Annotations, key) + changed = true + } + } + var podProfile *corev1.SeccompProfile + if intent.Spec.SecurityContext != nil { + podProfile = intent.Spec.SecurityContext.SeccompProfile + } + clear(corev1.SeccompPodAnnotationKey, podProfile) + // A legacy container annotation can override the inherited pod profile. + for _, list := range [][]corev1.Container{intent.Spec.Containers, intent.Spec.InitContainers} { + for _, container := range list { + profile := podProfile + if container.SecurityContext != nil { + profile = preferExplicit(container.SecurityContext.SeccompProfile, podProfile) + } + clear(corev1.SeccompContainerAnnotationKeyPrefix+container.Name, profile) + } + } + return changed +} + +func matchesSeccompProfile(annotation string, profile *corev1.SeccompProfile) bool { + switch profile.Type { + case corev1.SeccompProfileTypeRuntimeDefault: + return annotation == corev1.SeccompProfileRuntimeDefault || annotation == corev1.DeprecatedSeccompProfileDockerDefault + case corev1.SeccompProfileTypeUnconfined: + return annotation == corev1.SeccompProfileNameUnconfined + case corev1.SeccompProfileTypeLocalhost: + return profile.LocalhostProfile != nil && annotation == corev1.SeccompLocalhostProfileNamePrefix+*profile.LocalhostProfile + default: + return false + } +} + +func statefulSetFields(current *appsv1.StatefulSet) (operator, external *fieldpath.Set) { + operator, external = fieldpath.NewSet(), fieldpath.NewSet() + legacy := legacyManagersFor(current) + for _, entry := range current.ManagedFields { + if entry.Subresource != "" || entry.FieldsV1 == nil || entry.APIVersion != "apps/v1" { + continue + } + fields := &fieldpath.Set{} + if fields.FromJSON(entry.FieldsV1.GetRawReader()) != nil { + continue + } + if entry.Manager == applyFieldManager && entry.Operation == metav1.ManagedFieldsOperationApply || legacy.Has(entry.Manager) && entry.Operation == metav1.ManagedFieldsOperationUpdate { + operator = operator.Union(fields) + } else { + external = external.Union(fields) + } + } + return operator, external +} + +func containerPath(list, name string) fieldpath.Path { + return fieldpath.MakePathOrDie("spec", "template", "spec", list, fieldpath.KeyByFields("name", name)) +} + +func referencedVolumes(spec *corev1.PodSpec) sets.Set[string] { + names := sets.New[string]() + for _, list := range [][]corev1.Container{spec.Containers, spec.InitContainers} { + for _, container := range list { + for _, mount := range container.VolumeMounts { + names.Insert(mount.Name) + } + for _, device := range container.VolumeDevices { + names.Insert(device.Name) + } + } + } + return names +} + +func retiringVolumesForMerge(live, desired *corev1.PodSpec, operator, external *fieldpath.Set) []corev1.Volume { + requested := sets.New[string]() + for _, volume := range desired.Volumes { + requested.Insert(volume.Name) + } + referenced := referencedVolumes(live) + var retiring []corev1.Volume + for _, volume := range live.Volumes { + if requested.Has(volume.Name) { + continue + } + path := fieldpath.MakePathOrDie("spec", "template", "spec", "volumes", fieldpath.KeyByFields("name", volume.Name)) + if ownsFieldOrChild(operator, path) && (referenced.Has(volume.Name) || ownsFieldOrChild(external, path)) { + retiring = append(retiring, volume) + } + } + return retiring +} + +func ownsFieldOrChild(fields *fieldpath.Set, path fieldpath.Path) bool { + if fields.Has(path) { + return true + } + for _, element := range path { + fields = fields.WithPrefix(element) + } + return !fields.Empty() +} + +func resolveRetiringVolumes(prepared, intent, merged *appsv1.StatefulSet, retiring []corev1.Volume) bool { + if len(retiring) == 0 { + return false + } + referenced := referencedVolumes(&merged.Spec.Template.Spec) + for _, claim := range merged.Spec.VolumeClaimTemplates { + referenced.Delete(claim.Name) + } + changed := false + volumes := merged.Spec.Template.Spec.Volumes + for _, volume := range retiring { + i := slices.IndexFunc(volumes, func(v corev1.Volume) bool { return v.Name == volume.Name }) + if i >= 0 && equality.Semantic.DeepEqual(volumes[i].VolumeSource, volume.VolumeSource) { + continue + } + if referenced.Has(volume.Name) { + intent.Spec.Template.Spec.Volumes = append(intent.Spec.Template.Spec.Volumes, volume) + changed = true + } else if i >= 0 { + // A co-owned fragment is not a complete independent volume definition. + live := &prepared.Spec.Template.Spec.Volumes + *live = slices.DeleteFunc(*live, func(v corev1.Volume) bool { return v.Name == volume.Name }) + changed = true + } + } + return changed +} + +func defaultRenderedPullPolicies(desired *corev1.PodSpec, external *fieldpath.Set) { + for _, list := range []struct { + name string + containers []corev1.Container + }{{"containers", desired.Containers}, {"initContainers", desired.InitContainers}} { + for i := range list.containers { + container := &list.containers[i] + path := append(containerPath(list.name, container.Name), fieldpath.MakePathOrDie("imagePullPolicy")...) + if container.ImagePullPolicy != "" || external.Has(path) { + continue + } + image, _, digest := strings.Cut(container.Image, "@") + tagStart := strings.LastIndexByte(image, ':') + hasTag := tagStart > strings.LastIndexByte(image, '/') + container.ImagePullPolicy = corev1.PullIfNotPresent + if hasTag && image[tagStart+1:] == "latest" || !hasTag && !digest { + container.ImagePullPolicy = corev1.PullAlways + } + } + } +} + +func removeSharedRetiredContainers(live, desired *corev1.PodSpec, operator, external *fieldpath.Set) bool { + changed := false + for _, list := range []struct { + name string + live *[]corev1.Container + desired []corev1.Container + }{{"containers", &live.Containers, desired.Containers}, {"initContainers", &live.InitContainers, desired.InitContainers}} { + *list.live = slices.DeleteFunc(*list.live, func(container corev1.Container) bool { + if slices.ContainsFunc(list.desired, func(wanted corev1.Container) bool { return wanted.Name == container.Name }) { + return false + } + path := containerPath(list.name, container.Name) + if !operator.Has(path) && !operator.Has(append(path, fieldpath.MakePathOrDie("image")...)) { + return false + } + shared := ownsFieldOrChild(external, path) + changed = changed || shared + return shared + }) + } + return changed +} + +func clearIncompatibleSources(live, intent *corev1.PodSpec) bool { + changed := false + for i := range live.Volumes { + volume := &live.Volumes[i] + for _, wanted := range intent.Volumes { + if volume.Name == wanted.Name && volumeSourceKind(volume.VolumeSource) != volumeSourceKind(wanted.VolumeSource) { + volume.VolumeSource = corev1.VolumeSource{} + changed = true + break + } + } + } + forRenderedContainers(live, intent, func(current, desired *corev1.Container) { + changed = clearIncompatibleHandlers(current, desired) || changed + for i := range current.Env { + env := ¤t.Env[i] + j := slices.IndexFunc(desired.Env, func(wanted corev1.EnvVar) bool { return wanted.Name == env.Name }) + if j < 0 { + continue + } + wanted := desired.Env[j] + // An omitted literal on a rendered name means empty, not unowned. + if (wanted.ValueFrom != nil || wanted.Value == "") && env.Value != "" { + env.Value = "" + changed = true + } + if env.ValueFrom != nil && (wanted.ValueFrom == nil || sourceKind(*env.ValueFrom) != sourceKind(*wanted.ValueFrom)) { + env.ValueFrom = nil + changed = true + } + } + }) + return changed +} + +func volumeSourceKind(source corev1.VolumeSource) int { + // Kubernetes defaults a missing volume source to EmptyDir. + if source == (corev1.VolumeSource{}) { + source.EmptyDir = &corev1.EmptyDirVolumeSource{} + } + return sourceKind(source) +} + +func clearIncompatibleHandlers(live, desired *corev1.Container) bool { + changed := false + for _, pair := range [][2]*corev1.Probe{{live.ReadinessProbe, desired.ReadinessProbe}, {live.LivenessProbe, desired.LivenessProbe}, {live.StartupProbe, desired.StartupProbe}} { + if pair[0] != nil && pair[1] != nil { + changed = clearIncompatibleHandler(&pair[0].ProbeHandler, &pair[1].ProbeHandler) || changed + } + } + if live.Lifecycle != nil && desired.Lifecycle != nil { + changed = clearIncompatibleHandler(live.Lifecycle.PostStart, desired.Lifecycle.PostStart) || changed + changed = clearIncompatibleHandler(live.Lifecycle.PreStop, desired.Lifecycle.PreStop) || changed + } + return changed +} + +func clearIncompatibleHandler[T any](live, desired *T) bool { + if live == nil || desired == nil || sourceKind(*live) == sourceKind(*desired) { + return false + } + var zero T + *live = zero + return true +} + +// Kubernetes models volume/env sources and probe/lifecycle handlers as unions. +func sourceKind(source any) int { + fields := reflect.ValueOf(source) + for i := range fields.NumField() { + if !fields.Field(i).IsZero() { + return i + } + } + return -1 +} + +func forRenderedContainers(live, desired *corev1.PodSpec, apply func(live, desired *corev1.Container)) { + for _, pair := range [][2][]corev1.Container{{live.Containers, desired.Containers}, {live.InitContainers, desired.InitContainers}} { + for i := range pair[0] { + for j := range pair[1] { + if pair[0][i].Name == pair[1][j].Name { + apply(&pair[0][i], &pair[1][j]) + break + } + } + } + } +} diff --git a/controllers/statefulset_repairs_test.go b/controllers/statefulset_repairs_test.go new file mode 100644 index 000000000..70498d9e4 --- /dev/null +++ b/controllers/statefulset_repairs_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func TestClearInheritedSecurityOverrides(t *testing.T) { + root, requestedUID, sidecarUID := int64(0), int64(1000), int64(2000) + yes, no := true, false + intent := &corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: &requestedUID, RunAsNonRoot: &yes, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{{Name: "zookeeper"}, {Name: "sidecar", SecurityContext: &corev1.SecurityContext{RunAsUser: &sidecarUID}}}, + InitContainers: []corev1.Container{{Name: "init"}}, + } + live := intent.DeepCopy() + live.Containers[0].SecurityContext = &corev1.SecurityContext{Privileged: &yes, ReadOnlyRootFilesystem: &yes} + live.Containers[1].SecurityContext.RunAsNonRoot = &no + live.InitContainers[0].SecurityContext = &corev1.SecurityContext{RunAsUser: &root} + live.Containers = append(live.Containers, corev1.Container{Name: "unmanaged", SecurityContext: &corev1.SecurityContext{RunAsUser: &sidecarUID}}) + if !clearInheritedSecurityOverrides(live, intent) { + t.Fatal("expected inherited settings to be reconciled") + } + zoo, sidecar, init := live.Containers[0].SecurityContext, live.Containers[1].SecurityContext, live.InitContainers[0].SecurityContext + if zoo.Privileged != nil || zoo.ReadOnlyRootFilesystem == nil || sidecar.RunAsNonRoot != nil || sidecar.RunAsUser == nil || *sidecar.RunAsUser != sidecarUID || init.RunAsUser != nil || live.Containers[2].SecurityContext.RunAsUser == nil { + t.Fatal("repair did not enforce inheritance while keeping explicit and unrelated settings") + } +} + +func TestClearInheritedRunAsNonRootOverrides(t *testing.T) { + for _, requested := range []bool{false, true} { + liveFlag, root := !requested, int64(0) + intent := &corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &requested}, + Containers: []corev1.Container{{Name: "zookeeper"}}, + } + live := intent.DeepCopy() + live.Containers[0].SecurityContext = &corev1.SecurityContext{RunAsNonRoot: &liveFlag, RunAsUser: &root} + if !clearInheritedSecurityOverrides(live, intent) { + t.Fatalf("expected conflicting runAsNonRoot to be repaired for %v", requested) + } + sc := live.Containers[0].SecurityContext + if sc.RunAsNonRoot != nil || (sc.RunAsUser == nil) != requested { + t.Fatalf("unexpected overrides for requested runAsNonRoot=%v: %+v", requested, sc) + } + } +} diff --git a/controllers/zookeepercluster_controller.go b/controllers/zookeepercluster_controller.go index 139a7624a..dbb4e4974 100644 --- a/controllers/zookeepercluster_controller.go +++ b/controllers/zookeepercluster_controller.go @@ -12,6 +12,7 @@ package controllers import ( "context" "fmt" + "sort" "strconv" "time" @@ -33,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -103,6 +105,7 @@ func (r *ZookeeperClusterReconciler) Reconcile(ctx context.Context, request ctrl if err := instance.Spec.ValidateLogging(); err != nil { return reconcile.Result{}, err } + original := instance.DeepCopy() changed := instance.WithDefaults() if instance.GetTriggerRollingRestart() { ctrl.LoggerFrom(ctx).Info("Restarting zookeeper cluster") @@ -116,7 +119,7 @@ func (r *ZookeeperClusterReconciler) Reconcile(ctx context.Context, request ctrl } if changed { ctrl.LoggerFrom(ctx).Info("Setting default settings for zookeeper-cluster") - if err := r.Client.Update(ctx, instance); err != nil { + if err := r.Client.Patch(ctx, instance, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return reconcile.Result{}, err } return reconcile.Result{Requeue: true}, nil @@ -146,41 +149,6 @@ func getRollingRestartAnnotation() (string, string) { return "restartTime", time.Now().Format(time.RFC850) } -// compareResourceVersion compare resoure versions for the supplied ZookeeperCluster and StatefulSet -// resources -// Returns: -// 0 if versions are equal -// -1 if ZookeeperCluster version is less than StatefulSet version -// 1 if ZookeeperCluster version is greater than StatefulSet version -func compareResourceVersion(zk *zookeeperv1beta1.ZookeeperCluster, sts *appsv1.StatefulSet) int { - - zkResourceVersion, zkErr := strconv.Atoi(zk.ResourceVersion) - stsVersion, stsVersionFound := sts.Labels["owner-rv"] - - if !stsVersionFound { - if zkErr != nil { - log.Info("Fail to parse ZookeeperCluster version. Cannot decide zookeeper StatefulSet version") - return 0 - } - return 1 - } - stsResourceVersion, err := strconv.Atoi(stsVersion) - if err != nil { - if zkErr != nil { - log.Info("Fail to parse ZookeeperCluster version. Cannot decide zookeeper StatefulSet version") - return 0 - } - log.Info("Fail to convert StatefulSet version %s to integer; setting it to ZookeeperCluster version", stsVersion) - return 1 - } - if zkResourceVersion < stsResourceVersion { - return -1 - } else if zkResourceVersion > stsResourceVersion { - return 1 - } - return 0 -} - func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, instance *zookeeperv1beta1.ZookeeperCluster) (err error) { // we cannot upgrade if cluster is in UpgradeFailed @@ -195,8 +163,7 @@ func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, i Namespace: sts.Namespace, }, foundSts) if err == nil { - err = r.Client.Update(ctx, foundSts) - if err != nil { + if err := requireClusterOwner(instance, foundSts, "statefulset"); err != nil { return err } if foundSts.Status.Replicas == foundSts.Status.ReadyReplicas && foundSts.Status.CurrentRevision == foundSts.Status.UpdateRevision { @@ -232,9 +199,10 @@ func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, i return err } else { if !equality.Semantic.DeepEqual(foundServiceAccount.ImagePullSecrets, serviceAccount.ImagePullSecrets) { + original := foundServiceAccount.DeepCopy() foundServiceAccount.ImagePullSecrets = serviceAccount.ImagePullSecrets ctrl.LoggerFrom(ctx).Info("Updating ServiceAccount", "ServiceAccount.Namespace", serviceAccount.Namespace, "ServiceAccount.Name", serviceAccount.Name) - err = r.Client.Update(ctx, foundServiceAccount) + err = r.Client.Patch(ctx, foundServiceAccount, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})) if err != nil { return err } @@ -255,23 +223,16 @@ func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, i "StatefulSet.Namespace", sts.Namespace, "StatefulSet.Name", sts.Name) zk.EnsureLogbackProjection(instance, nil, sts) - // label the RV of the zookeeperCluster when creating the sts - sts.Labels["owner-rv"] = instance.ResourceVersion - err = r.Client.Create(ctx, sts) - if err != nil { - return err - } - return nil + return r.Client.Create(ctx, sts, client.FieldOwner(createFieldManager)) } else if err != nil { return err } else { - // check whether zookeeperCluster is updated before updating the sts - cmp := compareResourceVersion(instance, foundSts) - if cmp < 0 { - return fmt.Errorf("Staleness: cr.ResourceVersion %s is smaller than labeledRV %s", instance.ResourceVersion, foundSts.Labels["owner-rv"]) - } else if cmp > 0 { - // Zookeeper StatefulSet version inherits ZookeeperCluster resource version - foundSts.Labels["owner-rv"] = instance.ResourceVersion + if err := requireClusterOwner(instance, foundSts, "statefulset"); err != nil { + return err + } + // Read the parent after the child. Its resource version cannot fence stale parent intent. + if err := r.requireCurrentClusterSpec(ctx, instance); err != nil { + return err } zk.EnsureLogbackProjection(instance, foundSts, sts) foundSTSSize := *foundSts.Spec.Replicas @@ -293,6 +254,12 @@ func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, i if err != nil { return fmt.Errorf("error doing exists check for znode %s: %w", path, err) } + if err := r.requireCurrentClusterSpec(zkCtx, instance); err != nil { + return err + } + if err := r.requireCurrentStatefulSet(zkCtx, foundSts); err != nil { + return err + } data := "CLUSTER_SIZE=" + strconv.Itoa(int(newSTSSize)) ctrl.LoggerFrom(ctx).Info("Updating Cluster Size.", "New Data:", data, "Version", version) @@ -312,10 +279,7 @@ func (r *ZookeeperClusterReconciler) updateStatefulSet(ctx context.Context, inst ctrl.LoggerFrom(ctx).Info("Updating StatefulSet", "StatefulSet.Namespace", foundSts.Namespace, "StatefulSet.Name", foundSts.Name) - zk.SyncStatefulSet(foundSts, sts) - - err = r.Client.Update(ctx, foundSts) - if err != nil { + if err := r.applyStatefulSet(ctx, foundSts, sts); err != nil { return err } instance.Status.Replicas = foundSts.Status.Replicas @@ -366,30 +330,20 @@ func (r *ZookeeperClusterReconciler) upgradeStatefulSet(ctx context.Context, ins err = checkSyncTimeout(instance, zookeeperv1beta1.UpdatingZookeeperReason, foundSts.Status.UpdatedReplicas, 10*time.Minute) if err != nil { instance.Status.SetErrorConditionTrue("UpgradeFailed", err.Error()) - return r.Client.Status().Update(ctx, instance) + return r.patchClusterStatus(ctx, instance) } else { return nil } } } } - return r.Client.Status().Update(ctx, instance) + return r.patchClusterStatus(ctx, instance) } func (r *ZookeeperClusterReconciler) clearUpgradeStatus(ctx context.Context, z *zookeeperv1beta1.ZookeeperCluster) (err error) { z.Status.SetUpgradingConditionFalse() z.Status.TargetVersion = "" - // need to deep copy the status struct, otherwise it will be overwritten - // when updating the CR below - status := z.Status.DeepCopy() - - err = r.Client.Update(ctx, z) - if err != nil { - return err - } - - z.Status = *status - return nil + return r.patchClusterStatus(ctx, z) } func checkSyncTimeout(z *zookeeperv1beta1.ZookeeperCluster, reason string, updatedReplicas int32, t time.Duration) error { @@ -423,19 +377,14 @@ func (r *ZookeeperClusterReconciler) reconcileClientService(ctx context.Context, ctrl.LoggerFrom(ctx).Info("Creating new client service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err = r.Client.Create(ctx, svc) - if err != nil { - return err - } - return nil + return r.Client.Create(ctx, svc, client.FieldOwner(createFieldManager)) } else if err != nil { return err } else { ctrl.LoggerFrom(ctx).Info("Updating existing client service", "Service.Namespace", foundSvc.Namespace, "Service.Name", foundSvc.Name) - zk.SyncService(foundSvc, svc) - err = r.Client.Update(ctx, foundSvc) + err = r.applyService(ctx, instance, foundSvc, svc) if err != nil { return err } @@ -470,19 +419,14 @@ func (r *ZookeeperClusterReconciler) reconcileHeadlessService(ctx context.Contex ctrl.LoggerFrom(ctx).Info("Creating new headless service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err = r.Client.Create(ctx, svc) - if err != nil { - return err - } - return nil + return r.Client.Create(ctx, svc, client.FieldOwner(createFieldManager)) } else if err != nil { return err } else { ctrl.LoggerFrom(ctx).Info("Updating existing headless service", "Service.Namespace", foundSvc.Namespace, "Service.Name", foundSvc.Name) - zk.SyncService(foundSvc, svc) - err = r.Client.Update(ctx, foundSvc) + err = r.applyService(ctx, instance, foundSvc, svc) if err != nil { return err } @@ -504,19 +448,14 @@ func (r *ZookeeperClusterReconciler) reconcileAdminServerService(ctx context.Con ctrl.LoggerFrom(ctx).Info("Creating admin server service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err = r.Client.Create(ctx, svc) - if err != nil { - return err - } - return nil + return r.Client.Create(ctx, svc, client.FieldOwner(createFieldManager)) } else if err != nil { return err } else { ctrl.LoggerFrom(ctx).Info("Updating existing admin server service", "Service.Namespace", foundSvc.Namespace, "Service.Name", foundSvc.Name) - zk.SyncService(foundSvc, svc) - err = r.Client.Update(ctx, foundSvc) + err = r.applyService(ctx, instance, foundSvc, svc) if err != nil { return err } @@ -553,7 +492,7 @@ func (r *ZookeeperClusterReconciler) reconcilePodDisruptionBudget(ctx context.Co if equality.Semantic.DeepEqual(original.Spec, foundPdb.Spec) { return nil } - return r.Client.Patch(ctx, foundPdb, client.MergeFrom(original)) + return r.Client.Patch(ctx, foundPdb, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})) } func (r *ZookeeperClusterReconciler) reconcileConfigMap(ctx context.Context, instance *zookeeperv1beta1.ZookeeperCluster) (err error) { @@ -585,8 +524,9 @@ func (r *ZookeeperClusterReconciler) reconcileConfigMap(ctx context.Context, ins ctrl.LoggerFrom(ctx).Info("Updating existing config-map", "ConfigMap.Namespace", foundCm.Namespace, "ConfigMap.Name", foundCm.Name) + original := foundCm.DeepCopy() zk.SyncConfigMap(foundCm, cm) - err = r.Client.Update(ctx, foundCm) + err = r.Client.Patch(ctx, foundCm, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})) if err != nil { return err } @@ -626,6 +566,8 @@ func (r *ZookeeperClusterReconciler) reconcileClusterStatus(ctx context.Context, unreadyMembers = append(unreadyMembers, p.Name) } } + sort.Strings(readyMembers) + sort.Strings(unreadyMembers) instance.Status.Members.Ready = readyMembers instance.Status.Members.Unready = unreadyMembers @@ -660,7 +602,66 @@ func (r *ZookeeperClusterReconciler) reconcileClusterStatus(ctx context.Context, if instance.Status.CurrentVersion == "" && instance.Status.IsClusterInReadyState() { instance.Status.CurrentVersion = instance.Spec.Image.Tag } - return r.Client.Status().Update(ctx, instance) + return r.patchClusterStatus(ctx, instance) +} + +func (r *ZookeeperClusterReconciler) patchClusterStatus(ctx context.Context, instance *zookeeperv1beta1.ZookeeperCluster) error { + current := &zookeeperv1beta1.ZookeeperCluster{} + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, client.ObjectKeyFromObject(instance), current); err != nil { + return err + } + if current.ResourceVersion != instance.ResourceVersion || current.UID != instance.UID { + return staleCluster(instance) + } + if equality.Semantic.DeepEqual(current.Status, instance.Status) { + return nil + } + desired := current.DeepCopy() + desired.Status = instance.Status + if err := r.Client.Status().Patch(ctx, desired, client.MergeFromWithOptions(current, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + instance.ResourceVersion = desired.ResourceVersion + instance.Status = desired.Status + return nil +} + +func staleCluster(instance *zookeeperv1beta1.ZookeeperCluster) error { + return errors.NewConflict(schema.GroupResource{Group: zookeeperv1beta1.GroupVersion.Group, Resource: "zookeeperclusters"}, instance.Name, fmt.Errorf("reconcile snapshot is stale")) +} + +func (r *ZookeeperClusterReconciler) requireCurrentClusterSpec(ctx context.Context, instance *zookeeperv1beta1.ZookeeperCluster) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + current := &zookeeperv1beta1.ZookeeperCluster{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(instance), current); err != nil { + return err + } + if current.UID != instance.UID || current.Generation != instance.Generation || !current.DeletionTimestamp.IsZero() { + return staleCluster(instance) + } + return nil +} + +func (r *ZookeeperClusterReconciler) requireCurrentStatefulSet(ctx context.Context, observed *appsv1.StatefulSet) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + current := &appsv1.StatefulSet{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(observed), current); err != nil { + return err + } + if current.UID != observed.UID || current.ResourceVersion != observed.ResourceVersion || !current.DeletionTimestamp.IsZero() { + return errors.NewConflict(appsv1.Resource("statefulsets"), observed.Name, fmt.Errorf("resize snapshot is stale")) + } + return nil } // YAMLExporterReconciler returns a fake Reconciler which is being used for generating YAML files @@ -755,8 +756,9 @@ func (r *ZookeeperClusterReconciler) reconcileFinalizers(ctx context.Context, in return nil } if !utils.ContainsString(instance.ObjectMeta.Finalizers, utils.ZkFinalizer) && !config.DisableFinalizer { + original := instance.DeepCopy() instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, utils.ZkFinalizer) - if err = r.Client.Update(ctx, instance); err != nil { + if err = r.Client.Patch(ctx, instance, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return err } } @@ -768,8 +770,9 @@ func (r *ZookeeperClusterReconciler) reconcileFinalizers(ctx context.Context, in return err } } + original := instance.DeepCopy() instance.Finalizers = utils.RemoveString(instance.Finalizers, utils.ZkFinalizer) - if err = r.Client.Update(ctx, instance); err != nil { + if err = r.Client.Patch(ctx, instance, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return err } } diff --git a/controllers/zookeepercluster_controller_test.go b/controllers/zookeepercluster_controller_test.go index 871622504..b8bc6507f 100644 --- a/controllers/zookeepercluster_controller_test.go +++ b/controllers/zookeepercluster_controller_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -62,7 +63,7 @@ func TestReconcileEventPredicate(t *testing.T) { stsSpec.Generation++ metadata := sts.DeepCopy() metadata.ResourceVersion = "2" - metadata.Labels = map[string]string{"owner-rv": "2"} + metadata.Labels = map[string]string{"external": "value"} cluster := &v1beta1.ZookeeperCluster{ObjectMeta: metav1.ObjectMeta{Generation: 1}} clusterSpec := cluster.DeepCopy() clusterSpec.Generation++ @@ -367,12 +368,12 @@ func TestFinalizerDeletionSkipsNormalReconciliation(t *testing.T) { Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { return errors.New("terminating clusters must not create resources") }, - Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { cluster, ok := obj.(*v1beta1.ZookeeperCluster) if !ok || !reflect.DeepEqual(cluster.Spec, originalSpec) || utils.ContainsString(cluster.Finalizers, utils.ZkFinalizer) { return errors.New("termination must only remove the operator finalizer") } - return c.Update(ctx, obj, opts...) + return c.Patch(ctx, obj, patch, opts...) }, }) result, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(z)}) @@ -576,11 +577,15 @@ func TestCanceledReconcileStopsClientWork(t *testing.T) { } other := z.DeepCopy() other.Name = "other" + other.UID = "other-metadata-test" other.ResourceVersion = "" if err := r.Client.Create(context.Background(), other); err != nil { t.Fatal(err) } otherSts := zk.MakeStatefulSet(other) + if err := controllerutil.SetControllerReference(other, otherSts, r.Scheme); err != nil { + t.Fatal(err) + } replicas := int32(3) otherSts.Spec.Replicas = &replicas if err := r.Client.Create(context.Background(), otherSts); err != nil { @@ -872,7 +877,7 @@ func TestScaleMetadataStatefulSetConflictRetries(t *testing.T) { z.Spec.Replicas = 5 cl := r.Client r.Client = interceptor.NewClient(cl.(client.WithWatch), interceptor.Funcs{ - Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error { + Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error { return apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, sts.Name, errors.New("stale writer")) }, }) @@ -904,7 +909,10 @@ func newScaleMetadataTestReconciler(t *testing.T, replicas int32) (*ZookeeperClu z.WithDefaults() z.Spec.Replicas = replicas sts := zk.MakeStatefulSet(z) - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(z, sts).WithStatusSubresource(z, sts).Build() + if err := controllerutil.SetControllerReference(z, sts, s); err != nil { + t.Fatal(err) + } + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(z, sts).WithStatusSubresource(z, sts).WithReturnManagedFields().Build() return &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return &MockZookeeperClient{} }, Log: log}, z, sts } @@ -953,6 +961,11 @@ var _ = Describe("ZookeeperCluster Controller", func() { mockZkClient = new(MockZookeeperClient) r *ZookeeperClusterReconciler ) + makeOwnedStatefulSet := func(z *v1beta1.ZookeeperCluster) *appsv1.StatefulSet { + sts := zk.MakeStatefulSet(z) + Ω(controllerutil.SetControllerReference(z, sts, s)).To(Succeed()) + return sts + } Context("Reconcile", func() { var ( @@ -972,6 +985,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: Name, Namespace: Namespace, + UID: "test-zookeeper-uid", }, } s.AddKnownTypes(v1beta1.GroupVersion, z) @@ -985,7 +999,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { z.Status.Init() z.Spec.Pod.Annotations = map[string]string{"custom": "keep"} cl = fake.NewClientBuilder().WithScheme(s). - WithRuntimeObjects(z, zk.MakeConfigMap(z), zk.MakeStatefulSet(z)). + WithRuntimeObjects(z, zk.MakeConfigMap(z), makeOwnedStatefulSet(z)). WithStatusSubresource(z).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} }) @@ -1055,11 +1069,11 @@ var _ = Describe("ZookeeperCluster Controller", func() { z.Spec.Logging.Level = "WARN" Ω(cl.Update(context.TODO(), z)).To(Succeed()) r.Client = interceptor.NewClient(cl, interceptor.Funcs{ - Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*corev1.ConfigMap); ok { return errors.New("config update failed") } - return c.Update(ctx, obj, opts...) + return c.Patch(ctx, obj, patch, opts...) }, }) _, err := r.Reconcile(context.TODO(), req) @@ -1252,7 +1266,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { z.Spec.Pod.ServiceAccountName = "zookeeper" z.Status.Init() next := z.DeepCopy() - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) next.Spec.Replicas = 6 cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} @@ -1281,7 +1295,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { z.WithDefaults() z.Status.Init() next := z.DeepCopy() - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) @@ -1354,7 +1368,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next.Spec.Image.Tag = "0.2.7" next.Status.CurrentVersion = "0.2.6" next.Status.SetPodsReadyConditionTrue() - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() st = &appsv1.StatefulSet{} err = cl.Get(context.TODO(), req.NamespacedName, st) @@ -1414,7 +1428,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next.Status.CurrentVersion = "0.2.6" next.Status.TargetVersion = "0.2.7" next.Status.SetUpgradingConditionTrue(" ", " ") - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() st = &appsv1.StatefulSet{} err = cl.Get(context.TODO(), req.NamespacedName, st) @@ -1461,7 +1475,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next := z.DeepCopy() next.Status.SetUpgradingConditionTrue(" ", "1") next.Status.TargetVersion = "0.2.7" - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() st = &appsv1.StatefulSet{} err = cl.Get(context.TODO(), req.NamespacedName, st) @@ -1507,8 +1521,8 @@ var _ = Describe("ZookeeperCluster Controller", func() { next.Status.ReadyReplicas = 3 next.Spec.Replicas = 3 next.Spec.Image.Tag = "0.2.7" - st := zk.MakeStatefulSet(z) - cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).Build() + st := makeOwnedStatefulSet(z) + cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() st = &appsv1.StatefulSet{} err = cl.Get(context.TODO(), req.NamespacedName, st) // changing the Revision value to simulate the upgrade scenario @@ -1551,7 +1565,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next.Status.CurrentVersion = "0.2.6" next.Status.TargetVersion = "" next.Status.IsClusterInUpgradingState() - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) @@ -1670,6 +1684,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next := z.DeepCopy() next.Spec.Ports[0].ContainerPort = 2182 svc := zk.MakeClientService(z) + Ω(controllerutil.SetControllerReference(next, svc, s)).To(Succeed()) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) @@ -1730,64 +1745,18 @@ var _ = Describe("ZookeeperCluster Controller", func() { }) }) - Context("Checking resource version", func() { - var ( - sts *appsv1.StatefulSet - ) - - BeforeEach(func() { - z.WithDefaults() - z.ResourceVersion = "100" - sts = &appsv1.StatefulSet{} - sts.Labels = make(map[string]string) - }) - - It("should return 1 as 100 > 99", func() { - sts.Labels["owner-rv"] = "99" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(1)) - }) - - It("should return 0 as 100 == 100", func() { - sts.Labels["owner-rv"] = "100" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(0)) - }) - - It("should return -1 as 100 < 101", func() { - sts.Labels["owner-rv"] = "101" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(-1)) - }) - - It("should return 1 as 100 > invalid numeric version 'xoxo'", func() { - sts.Labels["owner-rv"] = "xoxo" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(1)) - }) - - It("should return 0 as invalid zk version z0z0 == invalid numeric version 'x0x0'", func() { - sts.Labels["owner-rv"] = "x0x0" - z.ResourceVersion = "z0z0" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(0)) - }) - }) - Context("trigger rolling restart", func() { var ( cl client.Client err error foundZk = &v1beta1.ZookeeperCluster{} next *v1beta1.ZookeeperCluster - svc *corev1.Service ) BeforeEach(func() { z.WithDefaults() next = z.DeepCopy() next.Spec.TriggerRollingRestart = true - svc = zk.MakeClientService(z) - cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build() + cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) err = cl.Get(context.TODO(), req.NamespacedName, foundZk) @@ -1806,8 +1775,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { Ω(restartTimeExists).To(Equal(true)) next.Spec.TriggerRollingRestart = false - svc = zk.MakeClientService(z) - cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build() + cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) @@ -1827,7 +1795,6 @@ var _ = Describe("ZookeeperCluster Controller", func() { // update the crd instance next.Spec.TriggerRollingRestart = false - svc = zk.MakeClientService(z) r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) err = cl.Get(context.TODO(), req.NamespacedName, foundZk) @@ -1844,8 +1811,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { // update the crd instance to trigger rolling restart next.Spec.TriggerRollingRestart = true - svc = zk.MakeClientService(z) - cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build() + cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) err = cl.Get(context.TODO(), req.NamespacedName, foundZk) diff --git a/doc/operator-upgrade.md b/doc/operator-upgrade.md index b4a640f68..ea553faee 100644 --- a/doc/operator-upgrade.md +++ b/doc/operator-upgrade.md @@ -1,4 +1,47 @@ # Upgrading Operator + +## Before upgrading to a release that applies StatefulSets and Services + +- Check the controller owner reference on each existing ZooKeeper StatefulSet and + client, headless, and admin Service. Its UID must match the current + `ZookeeperCluster` UID. The operator does not automatically adopt restored, + manually created, or orphaned resources. If references do not match, pause the + upgrade. Identify the resources and their storage before attempting recovery; + do not delete the StatefulSet or PVCs to clear the error. Explicit orphan + recovery is tracked in [#122](https://github.com/adobe/zookeeper-operator/issues/122). +- Check any custom selectors on the existing Services. The operator restores + their selector to `app: `, even when the owner UID matches. + If clients need custom routing, create a separately managed Service with a + different name and switch the clients to it before the upgrade. +- If you manage your own operator RBAC (for example `rbac.create: false`), + compare it with the release's bundled role. In addition to existing permissions, + grant `patch` on `statefulsets` (`apps`), `services`, `configmaps`, and + `serviceaccounts` (core API group), `poddisruptionbudgets` (`policy`), and + `zookeeperclusters` and `zookeeperclusters/status` (`zookeeper.pravega.io`). + The bundled roles already include these permissions. A Ready operator Pod + does not prove that the operator can write these resources. + +### Headless DNS on clusters created with `make run-local` + +Older local binaries recorded Service writes under the generic `main` field +manager. The upgrade does not transfer Service fields from `main`, because that +name can also represent an unrelated writer. If you clear `spec.domainName`, a +legacy `external-dns.alpha.kubernetes.io/hostname` annotation may remain on the +headless Service. + +If the Service belongs to the current cluster UID, neither `spec.domainName` nor +`spec.headlessService.annotations` requests that hostname, and your old cluster +configuration confirms that the operator generated the live annotation, remove +just that key from the headless Service. Do not remove it if another controller +or owner manages it intentionally. + +```sh +kubectl -n annotate service -headless \ + external-dns.alpha.kubernetes.io/hostname- +``` + +## Upgrade the operator + zookeeperoperator can be upgraded to a version **[VERSION]** via helm using the following command ``` diff --git a/go.mod b/go.mod index 774d8fb4f..d57bc7f40 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( k8s.io/apimachinery v0.37.2 k8s.io/client-go v0.37.2 sigs.k8s.io/controller-runtime v0.25.2 + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 ) require ( @@ -81,6 +82,5 @@ require ( k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/pkg/zk/generators.go b/pkg/zk/generators.go index e9cf34c53..a148e00ae 100644 --- a/pkg/zk/generators.go +++ b/pkg/zk/generators.go @@ -14,6 +14,7 @@ import ( "crypto/sha256" "fmt" "reflect" + "sort" "strconv" "strings" @@ -333,8 +334,13 @@ func makeZkConfigString(z *v1beta1.ZookeeperCluster) string { ports := z.ZookeeperPorts() var zkConfig = "" - for key, value := range z.Spec.Conf.AdditionalConfig { - zkConfig = zkConfig + fmt.Sprintf("%s=%s\n", key, value) + keys := make([]string, 0, len(z.Spec.Conf.AdditionalConfig)) + for key := range z.Spec.Conf.AdditionalConfig { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + zkConfig = zkConfig + fmt.Sprintf("%s=%s\n", key, z.Spec.Conf.AdditionalConfig[key]) } return zkConfig + "4lw.commands.whitelist=cons, envi, conf, crst, srvr, stat, mntr, ruok\n" + "dataDir=/data\n" + @@ -469,6 +475,7 @@ func makeService(name string, ports []v1.ServicePort, clusterIP bool, external b Annotations: annotationMap, }, Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, Ports: ports, Selector: map[string]string{"app": z.GetName()}, }, diff --git a/pkg/zk/synchronizers.go b/pkg/zk/synchronizers.go index 77a37f89f..11cbf3a9a 100644 --- a/pkg/zk/synchronizers.go +++ b/pkg/zk/synchronizers.go @@ -11,24 +11,9 @@ package zk import ( - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ) -// SyncStatefulSet synchronizes any updates to the stateful-set -func SyncStatefulSet(curr *appsv1.StatefulSet, next *appsv1.StatefulSet) { - curr.Spec.Replicas = next.Spec.Replicas - curr.Spec.Template = next.Spec.Template - curr.Spec.UpdateStrategy = next.Spec.UpdateStrategy -} - -// SyncService synchronizes a service with an updated spec and validates it -func SyncService(curr *v1.Service, next *v1.Service) { - curr.Spec.Ports = next.Spec.Ports - curr.Spec.Type = next.Spec.Type - curr.SetAnnotations(next.GetAnnotations()) -} - // SyncConfigMap synchronizes a configmap with an updated spec and validates it func SyncConfigMap(curr *v1.ConfigMap, next *v1.ConfigMap) { curr.Data = next.Data diff --git a/pkg/zk/synchronizers_test.go b/pkg/zk/synchronizers_test.go index 9e909b8f6..d00517920 100644 --- a/pkg/zk/synchronizers_test.go +++ b/pkg/zk/synchronizers_test.go @@ -13,9 +13,6 @@ package zk_test import ( "github.com/pravega/zookeeper-operator/api/v1beta1" "github.com/pravega/zookeeper-operator/pkg/zk" - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/onsi/ginkgo" @@ -24,62 +21,6 @@ import ( var _ = Describe("Synchronizers", func() { - Context("with a valid update specs", func() { - var sts1 *appsv1.StatefulSet - - BeforeEach(func() { - z := &v1beta1.ZookeeperCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "example", - Namespace: "default", - }, - } - z.WithDefaults() - z.Spec.Pod.Resources.Limits = v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("20Gi"), - } - sts1 = zk.MakeStatefulSet(z) - sts2 := zk.MakeStatefulSet(z) - reps := int32(4) - sts2.Spec.Replicas = &reps - sts2.Spec.Template.Spec.Containers[0].Image = "repo/newimage:latest" - zk.SyncStatefulSet(sts1, sts2) - }) - - It("should have the updated fields", func() { - Ω(*sts1.Spec.Replicas).To(BeEquivalentTo(4)) - Ω(sts1.Spec.Template.Spec.Containers[0].Image). - To(Equal("repo/newimage:latest")) - }) - }) - - Context("with a valid update of Service port", func() { - var port int32 - var value string - - BeforeEach(func() { - z := &v1beta1.ZookeeperCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "example", - Namespace: "default", - }, - } - z.WithDefaults() - svc1 := zk.MakeClientService(z) - svc2 := svc1.DeepCopy() - svc2.Spec.Ports[0].Port = int32(4000) - svc2.Spec.Type = "temp" - zk.SyncService(svc1, svc2) - port = svc1.Spec.Ports[0].Port - value = string(svc1.Spec.Type) - }) - - It("should have the updated fields for service", func() { - Ω(port).To(BeEquivalentTo(4000)) - Ω(value).To(BeEquivalentTo("temp")) - }) - }) - Context("with a valid update of config map", func() { var value string BeforeEach(func() { diff --git a/pkg/zk/zookeeper_client_integration_test.go b/pkg/zk/zookeeper_client_integration_test.go index a90e16786..d16388a36 100644 --- a/pkg/zk/zookeeper_client_integration_test.go +++ b/pkg/zk/zookeeper_client_integration_test.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) func TestMetadataCreateWaitsForServingZooKeeper(t *testing.T) { @@ -228,17 +229,20 @@ func TestMetadataCreateRetryPreservesExistingData(t *testing.T) { cluster.WithDefaults() cluster.Spec.Replicas = 1 sts := operatorzk.MakeStatefulSet(cluster) + if err := controllerutil.SetControllerReference(cluster, sts, s); err != nil { + t.Fatal(err) + } sts.Status.Replicas = 1 sts.Status.ReadyReplicas = 1 cl := fake.NewClientBuilder().WithScheme(s).WithObjects(cluster, sts).WithStatusSubresource(cluster, sts).Build() conflict := true faultClient := interceptor.NewClient(cl, interceptor.Funcs{ - SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + SubResourcePatch: func(ctx context.Context, c client.Client, subresource string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { if subresource == "status" && conflict { conflict = false return apierrors.NewConflict(schema.GroupResource{Group: v1beta1.GroupVersion.Group, Resource: "zookeeperclusters"}, obj.GetName(), errors.New("stale writer")) } - return c.SubResource(subresource).Update(ctx, obj, opts...) + return c.SubResource(subresource).Patch(ctx, obj, patch, opts...) }, }) r := &controllers.ZookeeperClusterReconciler{