From 11da34281ad6341e2df468f70f1a9822c04b7c43 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Fri, 20 Nov 2026 15:48:30 +0100 Subject: [PATCH 1/2] fix(controller): converge ZooKeeper resources without write churn Use deterministic rendering and optimistic patches to avoid persisting unchanged resources. Persist upgrade status and reject stale resize snapshots before writing ZooKeeper metadata. Reconcile StatefulSets and Services with guarded apply and compatibility writes. Preserve immutable settings and unrelated ownership, keep complete sources for mounted volumes, and update Service exposure rules atomically without replacing allocated NodePorts. Resolve conflicting legacy seccomp annotations when applying explicit profiles. Document ownership and RBAC prerequisites and cover convergence, migration, and races in unit tests and an isolated shared envtest control plane. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- controllers/compatibility_envtest_test.go | 690 +++++++++++++++ controllers/convergence_test.go | 588 +++++++++++++ controllers/envtest_test.go | 806 +++++++++++++++++- controllers/server_side_apply.go | 145 ++++ controllers/server_side_apply_test.go | 53 ++ controllers/statefulset_compatibility.go | 216 +++++ controllers/statefulset_repairs.go | 343 ++++++++ controllers/statefulset_repairs_test.go | 54 ++ controllers/zookeepercluster_controller.go | 195 ++--- .../zookeepercluster_controller_test.go | 102 +-- doc/operator-upgrade.md | 24 + go.mod | 2 +- pkg/zk/generators.go | 11 +- pkg/zk/synchronizers.go | 15 - pkg/zk/synchronizers_test.go | 59 -- pkg/zk/zookeeper_client_integration_test.go | 8 +- 16 files changed, 3055 insertions(+), 256 deletions(-) create mode 100644 controllers/compatibility_envtest_test.go create mode 100644 controllers/convergence_test.go create mode 100644 controllers/server_side_apply.go create mode 100644 controllers/server_side_apply_test.go create mode 100644 controllers/statefulset_compatibility.go create mode 100644 controllers/statefulset_repairs.go create mode 100644 controllers/statefulset_repairs_test.go diff --git a/controllers/compatibility_envtest_test.go b/controllers/compatibility_envtest_test.go new file mode 100644 index 000000000..ea4adb324 --- /dev/null +++ b/controllers/compatibility_envtest_test.go @@ -0,0 +1,690 @@ +//go:build envtest + +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "slices" + "testing" + + "github.com/pravega/zookeeper-operator/pkg/zk" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func TestEnvtestMutableApplyPreservesDifferentImmutableFields(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + sts := zk.MakeStatefulSet(z).DeepCopy() + zk.EnsureLogbackProjection(z, nil, sts) + sts.Spec.ServiceName = "existing-headless" + sts.Spec.Selector.MatchLabels["kind"] = "ZookeeperMember" + sts.Spec.PodManagementPolicy = appsv1.ParallelPodManagement + storage := resource.MustParse("32Gi") + sts.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[corev1.ResourceStorage] = storage + if err := controllerutil.SetControllerReference(z, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, sts, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + z.Spec.Image.Tag = "updated" + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + if err := convergenceReconciler(cl).reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + spec := sts.Spec + if spec.ServiceName != "existing-headless" || spec.PodManagementPolicy != appsv1.ParallelPodManagement || spec.Selector.MatchLabels["kind"] != "ZookeeperMember" || !spec.VolumeClaimTemplates[0].Spec.Resources.Requests.Storage().Equal(storage) { + t.Fatal("StatefulSet immutable fields were not preserved") + } + if spec.Template.Spec.Containers[0].Image != z.Spec.Image.ToString() { + t.Fatal("mutable template change did not take effect") + } +} + +func TestEnvtestPreservesNamedContainerPortAliases(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Ports = append(z.Spec.Ports, corev1.ContainerPort{Name: "client-alt", ContainerPort: 2181}) + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + hasAlias := func(name string) bool { + return slices.ContainsFunc(sts.Spec.Template.Spec.Containers[0].Ports, func(port corev1.ContainerPort) bool { + return port.Name == name && port.ContainerPort == 2181 && port.Protocol == corev1.ProtocolTCP + }) + } + if len(sts.Spec.Template.Spec.Containers[0].Ports) != len(z.Spec.Ports) || !hasAlias("client") || !hasAlias("client-alt") { + t.Fatal("an accepted port alias was dropped") + } + z.Spec.Ports = z.Spec.Ports[:len(z.Spec.Ports)-1] + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + if len(sts.Spec.Template.Spec.Containers[0].Ports) != len(z.Spec.Ports) || !hasAlias("client") || hasAlias("client-alt") { + t.Fatal("expected only the canonical client port after retiring the alias") + } +} + +func TestEnvtestRepairsNamedPortKeyDrift(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + svc := &corev1.Service{} + svcKey := client.ObjectKey{Namespace: z.Namespace, Name: z.GetClientServiceName()} + if err := cl.Get(ctx, svcKey, svc); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Ports[0].ContainerPort = 2199 + sts.Spec.Template.Spec.Containers[0].Ports = append(sts.Spec.Template.Spec.Containers[0].Ports, corev1.ContainerPort{Name: "extra", ContainerPort: 9090}) + svc.Spec.Ports[0].Port = 2199 + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Name: "extra", Port: 9090}) + for _, obj := range []client.Object{sts, svc} { + if err := cl.Update(ctx, obj, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, svcKey, svc); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + ports := sts.Spec.Template.Spec.Containers[0].Ports + for name, port := range map[string]int32{"client": 2181, "extra": 9090} { + if !slices.ContainsFunc(ports, func(p corev1.ContainerPort) bool { return p.Name == name && p.ContainerPort == port }) { + t.Fatalf("expected %s/%d in container ports: %+v", name, port, ports) + } + } + for name, port := range map[string]int32{"tcp-client": 2181, "extra": 9090} { + if !slices.ContainsFunc(svc.Spec.Ports, func(p corev1.ServicePort) bool { return p.Name == name && p.Port == port }) { + t.Fatalf("expected %s/%d in Service ports: %+v", name, port, svc.Spec.Ports) + } + } + previous := sts.DeepCopy() + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + if sts.Generation != previous.Generation { + t.Fatalf("repair was followed by another rollout: ports before=%+v after=%+v", previous.Spec.Template.Spec.Containers[0].Ports, sts.Spec.Template.Spec.Containers[0].Ports) + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestReplacesMutuallyExclusiveSources(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Pod.Env = []corev1.EnvVar{{Name: "SETTING", Value: "desired"}} + z.Spec.Volumes = []corev1.Volume{{Name: "input", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + env := sts.Spec.Template.Spec.Containers[0].Env + env[0] = corev1.EnvVar{Name: "ENVOY_SIDECAR_STATUS", Value: "different"} + env[1] = corev1.EnvVar{Name: "SETTING", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}} + sts.Spec.Template.Spec.Volumes[0].VolumeSource = corev1.VolumeSource{ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "another"}}} + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + env = sts.Spec.Template.Spec.Containers[0].Env + vol := sts.Spec.Template.Spec.Volumes[0] + if env[0].Value != "" || env[0].ValueFrom == nil || env[1].Value != "desired" || env[1].ValueFrom != nil || vol.EmptyDir == nil || vol.ConfigMap != nil { + t.Fatal("operator did not replace the incompatible environment and volume sources") + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestRetainsVolumeUntilIndependentMountIsRemoved(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Volumes = []corev1.Volume{{Name: "shared", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{}}}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers = append(sts.Spec.Template.Spec.Containers, corev1.Container{ + Name: "independent", Image: "busybox:1", + VolumeMounts: []corev1.VolumeMount{{Name: "shared", MountPath: "/shared"}}, + }) + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + z.Spec.Volumes = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + for _, mounted := range []bool{true, false} { + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + pod := &sts.Spec.Template.Spec + i := slices.IndexFunc(pod.Containers, func(c corev1.Container) bool { return c.Name == "independent" }) + if i < 0 || slices.ContainsFunc(pod.Volumes, func(v corev1.Volume) bool { return v.Name == "shared" }) != mounted || (len(pod.Containers[i].VolumeMounts) != 0) != mounted { + t.Fatalf("expected the volume to survive only while the independent container mounts it: mounted=%v", mounted) + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) + if mounted { + pod.Containers[i].VolumeMounts = nil + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + } + } +} + +func TestEnvtestRetainsCompleteSourceOfSharedVolume(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Volumes = []corev1.Volume{{Name: "shared", VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{LocalObjectReference: corev1.LocalObjectReference{Name: "shared-config"}}, + }}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + // An independent applier owns the mount and defaultMode, but not the ConfigMap name. + intent := map[string]any{ + "apiVersion": "apps/v1", "kind": "StatefulSet", + "metadata": map[string]string{"name": z.Name, "namespace": z.Namespace}, + "spec": map[string]any{"template": map[string]any{"spec": map[string]any{ + "containers": []corev1.Container{{Name: "independent", Image: "busybox:1", VolumeMounts: []corev1.VolumeMount{{Name: "shared", MountPath: "/shared"}}}}, + "volumes": []map[string]any{{"name": "shared", "configMap": map[string]int{"defaultMode": 0o640}}}, + }}}, + } + payload, err := json.Marshal(intent) + if err != nil { + t.Fatal(err) + } + if err := cl.Patch(ctx, sts, client.RawPatch(types.ApplyPatchType, payload), client.FieldOwner("independent"), client.ForceOwnership); err != nil { + t.Fatal(err) + } + z.Spec.Volumes = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + pod := &sts.Spec.Template.Spec + if !slices.ContainsFunc(pod.Volumes, func(v corev1.Volume) bool { + return v.Name == "shared" && v.ConfigMap != nil && v.ConfigMap.Name == "shared-config" && v.ConfigMap.DefaultMode != nil && *v.ConfigMap.DefaultMode == 0o640 + }) { + t.Fatalf("shared volume lost its ConfigMap name or independently owned mode: %+v", pod.Volumes) + } + i := slices.IndexFunc(pod.Containers, func(c corev1.Container) bool { return c.Name == "independent" }) + if i < 0 || !slices.ContainsFunc(pod.Containers[i].VolumeMounts, func(m corev1.VolumeMount) bool { return m.Name == "shared" }) { + t.Fatal("independent mount was not retained") + } + pod.Containers[i].VolumeMounts = nil + if err := cl.Update(ctx, sts, client.FieldOwner("independent")); err != nil { + t.Fatal(err) + } + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + if slices.ContainsFunc(sts.Spec.Template.Spec.Volumes, func(v corev1.Volume) bool { return v.Name == "shared" }) { + t.Fatal("partially shared volume was not pruned after its last mount disappeared") + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestRestoresDuplicateEnvWithoutMovingIndependentVariables(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Pod.Env = []corev1.EnvVar{ + {Name: "DATA_DIR", Value: "$(EXTERNAL_BASE)/data"}, + {Name: "WRAPPED", Value: "first"}, + {Name: "WRAPPED", Value: "$(WRAPPED)/second"}, + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + c := &sts.Spec.Template.Spec.Containers[0] + c.Env = slices.Insert(c.Env, 1, corev1.EnvVar{Name: "EXTERNAL_BASE", Value: "/srv"}) + c.Env = append(c.Env, corev1.EnvVar{Name: "EXTERNAL_TRAILING", Value: "$(WRAPPED)"}) + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + var names []string + for _, env := range sts.Spec.Template.Spec.Containers[0].Env { + names = append(names, env.Name) + } + want := []string{"ENVOY_SIDECAR_STATUS", "EXTERNAL_BASE", "DATA_DIR", "WRAPPED", "WRAPPED", "EXTERNAL_TRAILING"} + if !slices.Equal(names, want) { + t.Fatalf("expected the injected variables to keep their positions: got %v, want %v", names, want) + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestClearsRenderedEmptyEnvironmentLiteral(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Pod.Env = []corev1.EnvVar{{Name: "SETTING"}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + c := &sts.Spec.Template.Spec.Containers[0] + c.Env[1].Value = "out-of-date" + c.Env = append(c.Env, corev1.EnvVar{Name: "UNRELATED", Value: "keep"}) + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + env := sts.Spec.Template.Spec.Containers[0].Env + if !slices.ContainsFunc(env, func(e corev1.EnvVar) bool { return e.Name == "SETTING" && e.Value == "" && e.ValueFrom == nil }) || !slices.ContainsFunc(env, func(e corev1.EnvVar) bool { return e.Name == "UNRELATED" && e.Value == "keep" }) { + t.Fatalf("expected the rendered empty value and preserved unrelated value: %+v", env) + } +} + +func TestEnvtestRepairsRenderedProbeAndLifecycleActions(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + container := &sts.Spec.Template.Spec.Containers[0] + container.ReadinessProbe.ProbeHandler = corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/health", Port: intstr.FromInt32(8080)}} + container.Lifecycle.PreStop = &corev1.LifecycleHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/stop", Port: intstr.FromInt32(8080)}} + container.Lifecycle.PostStart = &corev1.LifecycleHandler{Exec: &corev1.ExecAction{Command: []string{"true"}}} + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + container = &sts.Spec.Template.Spec.Containers[0] + if container.ReadinessProbe.Exec == nil || container.ReadinessProbe.HTTPGet != nil || container.Lifecycle.PreStop.Exec == nil || container.Lifecycle.PreStop.HTTPGet != nil || container.Lifecycle.PostStart == nil { + t.Fatal("requested actions did not replace incompatible variants while preserving the unrelated hook") + } +} + +func TestEnvtestRetiresSharedContainerAsAUnit(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Containers = []corev1.Container{{Name: "retired", Image: "busybox:1"}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + intent := map[string]any{ + "apiVersion": "apps/v1", "kind": "StatefulSet", + "metadata": map[string]string{"name": z.Name, "namespace": z.Namespace}, + "spec": map[string]any{"template": map[string]any{"spec": map[string]any{"containers": []corev1.Container{ + {Name: "retired", Args: []string{"extra"}}, + {Name: "independent", Image: "busybox:1"}, + }}}}, + } + data, err := json.Marshal(intent) + if err != nil { + t.Fatal(err) + } + if err := cl.Patch(ctx, sts, client.RawPatch(types.ApplyPatchType, data), client.FieldOwner("injector")); err != nil { + t.Fatal(err) + } + z.Spec.Containers = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + containers := sts.Spec.Template.Spec.Containers + if len(containers) != 2 || !slices.ContainsFunc(containers, func(c corev1.Container) bool { return c.Name == "independent" }) || slices.ContainsFunc(containers, func(c corev1.Container) bool { return c.Name == "retired" }) { + t.Fatalf("expected the retired container removed and the independent one retained: %+v", containers) + } +} + +func TestEnvtestPullPolicyFollowsImageButPreservesExplicitOwners(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Containers = []corev1.Container{ + {Name: "defaulted", Image: "busybox:1"}, + {Name: "external-policy", Image: "busybox:1"}, + {Name: "explicit-policy", Image: "busybox:1", ImagePullPolicy: corev1.PullIfNotPresent}, + } + z.Spec.InitContainers = []corev1.Container{{Name: "defaulted-init", Image: "busybox:1"}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[1].ImagePullPolicy = corev1.PullNever + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + for i := range z.Spec.Containers { + z.Spec.Containers[i].Image = "busybox:latest" + } + z.Spec.InitContainers[0].Image = "busybox" + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + for i, want := range []corev1.PullPolicy{corev1.PullAlways, corev1.PullNever, corev1.PullIfNotPresent} { + if got := sts.Spec.Template.Spec.Containers[i].ImagePullPolicy; got != want { + t.Errorf("container %d pull policy = %s, want %s", i, got, want) + } + } + if sts.Spec.Template.Spec.InitContainers[0].ImagePullPolicy != corev1.PullAlways { + t.Error("an untagged init image should default to Always") + } +} + +func TestEnvtestAdminServiceTransitionsAreAtomic(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKey{Namespace: z.Namespace, Name: z.GetAdminServerServiceName()} + svc := &corev1.Service{} + if err := cl.Get(ctx, key, svc); err != nil { + t.Fatal(err) + } + svc.Spec.Ports = append(svc.Spec.Ports, corev1.ServicePort{Name: "internal-extra", Port: 9090}) + svc.Annotations = map[string]string{"external": "preserve"} + if err := cl.Update(ctx, svc, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + z.Spec.AdminServerService.External = true + const frontend = "networking.gke.io/load-balancer-type" + const ranges = "service.beta.kubernetes.io/load-balancer-source-ranges" + z.Spec.AdminServerService.Annotations = map[string]string{frontend: "Internal"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + for _, external := range []bool{true, false} { + writes := 0 + r.Client = observeServicePatches(cl, key, func(live *corev1.Service) { + writes++ + wantType := corev1.ServiceTypeClusterIP + if external { + wantType = corev1.ServiceTypeLoadBalancer + } + if live.Spec.Type != wantType || len(live.Spec.Ports) != 1 || live.Spec.Ports[0].Name != "tcp-admin-server" || live.Annotations["external"] != "preserve" { + t.Fatalf("unexpected persisted admin Service state: type=%s ports=%+v", live.Spec.Type, live.Spec.Ports) + } + if external && live.Annotations[frontend] != "Internal" || !external && (live.Annotations[ranges] != "" || len(live.Spec.LoadBalancerSourceRanges) != 0) { + t.Fatal("Service type and exposure annotations must transition together") + } + }) + if err := r.reconcileAdminServerService(ctx, z); err != nil { + t.Fatal(err) + } + if writes != 1 { + t.Fatalf("transition issued %d Service patches, want 1", writes) + } + if external { + if err := cl.Get(ctx, key, svc); err != nil { + t.Fatal(err) + } + svc.Annotations[ranges] = "192.0.2.0/24" + svc.Spec.LoadBalancerSourceRanges = []string{"198.51.100.0/24"} + if err := cl.Update(ctx, svc, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + z.Spec.AdminServerService.External = false + z.Spec.AdminServerService.Annotations = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + } + } +} + +func TestEnvtestAdminPortChangeRetainsManuallyAllocatedNodePort(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.AdminServerService.External = true + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKey{Namespace: z.Namespace, Name: z.GetAdminServerServiceName()} + svc := &corev1.Service{} + if err := cl.Get(ctx, key, svc); err != nil { + t.Fatal(err) + } + nodePort := svc.Spec.Ports[0].NodePort + if nodePort == 0 { + t.Fatal("expected the test Service to have an allocated NodePort") + } + allocate := false + svc.Spec.AllocateLoadBalancerNodePorts = &allocate + svc.Spec.LoadBalancerSourceRanges = []string{"192.0.2.0/24"} + if err := cl.Update(ctx, svc, client.FieldOwner("network-config")); err != nil { + t.Fatal(err) + } + const newPort = 8081 + for i := range z.Spec.Ports { + if z.Spec.Ports[i].Name == "admin-server" { + z.Spec.Ports[i].ContainerPort = newPort + } + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + for i := range 3 { + if err := r.reconcileAdminServerService(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, key, svc); err != nil { + t.Fatal(err) + } + if ports := svc.Spec.Ports; len(ports) != 1 || ports[0].Port != newPort || ports[0].NodePort != nodePort { + t.Fatalf("reconcile %d: expected port %d to retain NodePort %d, got %+v", i+1, newPort, nodePort, ports) + } + if svc.Spec.AllocateLoadBalancerNodePorts == nil || *svc.Spec.AllocateLoadBalancerNodePorts || !slices.Equal(svc.Spec.LoadBalancerSourceRanges, []string{"192.0.2.0/24"}) { + t.Fatal("external access settings changed during the port update") + } + } +} + +func TestEnvtestHeadlessServicePortSwapIsAtomic(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKeyFromObject(zk.MakeHeadlessService(z)) + z.Spec.Ports[0].ContainerPort, z.Spec.Ports[1].ContainerPort = z.Spec.Ports[1].ContainerPort, z.Spec.Ports[0].ContainerPort + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + desired := zk.MakeHeadlessService(z) + writes := 0 + r.Client = observeServicePatches(cl, key, func(live *corev1.Service) { + writes++ + for _, wanted := range desired.Spec.Ports { + if !slices.ContainsFunc(live.Spec.Ports, func(p corev1.ServicePort) bool { return p.Name == wanted.Name && p.Port == wanted.Port }) { + t.Fatalf("persisted headless Service lacks %s/%d", wanted.Name, wanted.Port) + } + } + }) + if err := r.reconcileHeadlessService(ctx, z); err != nil { + t.Fatal(err) + } + if writes != 1 { + t.Fatalf("port swap issued %d Service patches, want 1", writes) + } +} + +func observeServicePatches(cl client.WithWatch, key client.ObjectKey, observe func(*corev1.Service)) client.WithWatch { + return interceptor.NewClient(cl, interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := c.Patch(ctx, obj, patch, opts...); err != nil { + return err + } + if svc, ok := obj.(*corev1.Service); ok && client.ObjectKeyFromObject(svc) == key { + observe(svc) + } + return nil + }, + }) +} + +func TestEnvtestEnforcesRequestedZooKeeperIdentity(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKeyFromObject(z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + oldUser, requestedUser, readOnly := int64(2000), int64(1000), true + sts.Spec.Template.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{RunAsUser: &oldUser, ReadOnlyRootFilesystem: &readOnly} + if err := cl.Update(ctx, sts, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + z.Spec.Pod.SecurityContext = &corev1.PodSecurityContext{RunAsUser: &requestedUser} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + pod, container := sts.Spec.Template.Spec.SecurityContext, sts.Spec.Template.Spec.Containers[0].SecurityContext + if pod == nil || pod.RunAsUser == nil || *pod.RunAsUser != requestedUser || container == nil || container.RunAsUser != nil || container.ReadOnlyRootFilesystem == nil || !*container.ReadOnlyRootFilesystem { + t.Fatal("operator must enforce the requested identity without dropping other security settings") + } +} + +func TestEnvtestSeccompProfileReplacesConflictingLegacyAnnotation(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKeyFromObject(z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Annotations[corev1.SeccompPodAnnotationKey] = "localhost/previous.json" + sts.Spec.Template.Annotations[corev1.SeccompContainerAnnotationKeyPrefix+"zookeeper"] = "localhost/previous.json" + sts.Spec.Template.Annotations["example.com/unrelated"] = "keep" + if err := cl.Update(ctx, sts, client.FieldOwner("profile-config")); err != nil { + t.Fatal(err) + } + z.Spec.Pod.SecurityContext = &corev1.PodSecurityContext{ + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + template := sts.Spec.Template + templateGeneration := sts.Generation + if template.Spec.SecurityContext == nil || template.Spec.SecurityContext.SeccompProfile == nil || template.Spec.SecurityContext.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Fatal("requested seccomp profile was not applied") + } + for _, key := range []string{corev1.SeccompPodAnnotationKey, corev1.SeccompContainerAnnotationKeyPrefix + "zookeeper"} { + if value, ok := template.Annotations[key]; ok && value != corev1.SeccompProfileRuntimeDefault && value != corev1.DeprecatedSeccompProfileDockerDefault { + t.Fatalf("conflicting legacy annotation %s retained: %s", key, value) + } + } + if template.Annotations["example.com/unrelated"] != "keep" { + t.Fatal("unrelated annotation was removed") + } + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + if sts.Generation != templateGeneration { + t.Fatal("completing the ownership migration triggered a second template change") + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} diff --git a/controllers/convergence_test.go b/controllers/convergence_test.go new file mode 100644 index 000000000..f9eed4173 --- /dev/null +++ b/controllers/convergence_test.go @@ -0,0 +1,588 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "testing" + + api "github.com/pravega/zookeeper-operator/api/v1beta1" + "github.com/pravega/zookeeper-operator/pkg/zk" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + policyv1 "k8s.io/api/policy/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +// workloadWrites are the StatefulSet and Service applies that every reconcile issues. +// The API server does not persist an apply that changes nothing; envtest checks this. +var workloadWrites = []string{"apply *v1.StatefulSet/", "apply *v1.Service/"} + +func TestReconcileSettledClusterSkipsStatusAndConfigWrites(t *testing.T) { + r, z, _ := newScaleMetadataTestReconciler(t, 1) + cl := r.Client.(client.WithWatch) + ctx := t.Context() + z.Spec.Replicas = 4 + z.Spec.Pod.ServiceAccountName = "example-account" + z.Spec.Conf.AdditionalConfig = map[string]string{"custom.a": "one", "custom.b": "two", "custom.c": "three"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r = convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + sts.Status.Replicas, sts.Status.ReadyReplicas = z.Spec.Replicas, 2 + sts.Status.CurrentRevision, sts.Status.UpdateRevision = "settled", "settled" + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + for i, name := range []string{"member-a", "member-b", "member-c", "member-d"} { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: z.Namespace, Labels: map[string]string{"app": z.Name}}, + Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "zookeeper", Image: "example:1"}}}, + } + if err := cl.Create(ctx, pod); err != nil { + t.Fatal(err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "zookeeper", Ready: i < 2}} + if err := cl.Status().Update(ctx, pod); err != nil { + t.Fatal(err) + } + } + settleCluster(t, ctx, r, z) + if !slices.Equal(z.Status.Members.Ready, []string{"member-a", "member-b"}) || !slices.Equal(z.Status.Members.Unready, []string{"member-c", "member-d"}) { + t.Fatalf("expected both ready and unready members in canonical order: %+v", z.Status.Members) + } + var writes []string + reverse := false + r.Client = observeMutations(interceptor.NewClient(cl, interceptor.Funcs{ + List: func(ctx context.Context, c client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := c.List(ctx, list, opts...); err != nil { + return err + } + if pods, ok := list.(*corev1.PodList); ok && len(pods.Items) == 4 { + reverse = !reverse + if reverse { + slices.Reverse(pods.Items) + } + } + return nil + }, + }), &writes) + for range 5 { + runReconcile(t, ctx, r, z) + } + if unexpected := writesExcept(writes, workloadWrites...); len(unexpected) != 0 { + t.Fatalf("settled cluster must not rewrite status, config or metadata: %v", unexpected) + } +} + +func TestReconcileConflictRetries(t *testing.T) { + r, z, _ := newScaleMetadataTestReconciler(t, 1) + exerciseReconcileConflictRetries(t, t.Context(), r.Client.(client.WithWatch), z, "StatefulSet", "Service", "PodDisruptionBudget", "status") +} + +func TestUnchangedStatusRejectsStaleCluster(t *testing.T) { + r, z, _ := newScaleMetadataTestReconciler(t, 1) + key := client.ObjectKeyFromObject(z) + if err := r.Client.Get(t.Context(), key, z); err != nil { + t.Fatal(err) + } + newer := z.DeepCopy() + newer.Spec.AdminServerService.External = true + if err := r.Client.Update(t.Context(), newer); err != nil { + t.Fatal(err) + } + var writes []string + r.Client = observeMutations(r.Client.(client.WithWatch), &writes) + if err := r.patchClusterStatus(t.Context(), z); !apierrors.IsConflict(err) || len(writes) != 0 { + t.Fatalf("stale snapshot with unchanged status must conflict without writing: err=%v writes=%v", err, writes) + } +} + +func TestReconcilePreservesUnownedContainerFields(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + settleCluster(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Args = []string{"external-argument"} + if err := r.Client.Update(t.Context(), sts); err != nil { + t.Fatal(err) + } + z.Spec.Pod.NodeSelector = map[string]string{"pool": "new"} + if err := r.Client.Update(t.Context(), z); err != nil { + t.Fatal(err) + } + runReconcile(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if sts.Spec.Template.Spec.NodeSelector["pool"] != "new" { + t.Fatal("desired node selector was not applied") + } + if args := sts.Spec.Template.Spec.Containers[0].Args; len(args) != 1 || args[0] != "external-argument" { + t.Fatal("Pod-template apply replaced unowned container arguments") + } +} + +func TestReconcileAppliesOnlyRenderedFields(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + settleCluster(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Args = []string{"external-argument"} + if err := r.Client.Update(t.Context(), sts); err != nil { + t.Fatal(err) + } + z.Spec.Image.Tag = "next" + if err := r.Client.Update(t.Context(), z); err != nil { + t.Fatal(err) + } + applied := false + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok { + if patch.Type() != types.ApplyPatchType { + t.Errorf("StatefulSet must be written with server-side apply, got %s", patch.Type()) + } + data, err := patch.Data(obj) + if err != nil { + return err + } + applied = true + var body map[string]any + if err := json.Unmarshal(data, &body); err != nil { + return err + } + if _, ok := body["status"]; ok || strings.Contains(string(data), `"args"`) { + t.Errorf("apply included status or unowned container arguments: %s", data) + } + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + runReconcile(t, t.Context(), r, z) + if !applied { + t.Fatal("image change did not apply the StatefulSet") + } +} + +func TestReconcileRepairsOwnedFieldDrift(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + settleCluster(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Image = "external:drift" + if err := r.Client.Update(t.Context(), sts); err != nil { + t.Fatal(err) + } + runReconcile(t, t.Context(), r, z) + if err := r.Client.Get(t.Context(), client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if sts.Spec.Template.Spec.Containers[0].Image != z.Spec.Image.ToString() { + t.Fatal("owned-field drift was not corrected") + } +} + +func TestReconcileRemovesOnlyPreviouslyAppliedMetadata(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + cl := r.Client.(client.WithWatch) + ctx := t.Context() + z.Spec.Pod.Annotations = map[string]string{"requested": "old"} + z.Spec.Pod.Labels["custom"] = "old" + z.Spec.ClientService.Annotations = map[string]string{"requested": "old"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + settleCluster(t, ctx, r, z) + svc := &corev1.Service{} + if err := cl.Get(ctx, client.ObjectKey{Name: z.GetClientServiceName(), Namespace: z.Namespace}, svc); err != nil { + t.Fatal(err) + } + svc.Annotations["external"] = "keep" + if err := cl.Update(ctx, svc); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Annotations["external"] = "keep" + sts.Spec.Template.Labels["external"] = "keep" + if err := cl.Update(ctx, sts); err != nil { + t.Fatal(err) + } + z.Spec.Pod.Annotations = nil + delete(z.Spec.Pod.Labels, "custom") + z.Spec.ClientService.Annotations = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(svc), svc); err != nil { + t.Fatal(err) + } + if _, exists := sts.Spec.Template.Annotations["requested"]; exists { + t.Error("removed CR annotation remains on Pod template") + } + if _, exists := sts.Spec.Template.Labels["custom"]; exists { + t.Error("removed CR label remains on Pod template") + } + if _, exists := svc.Annotations["requested"]; exists { + t.Error("removed CR annotation remains on Service") + } + if sts.Spec.Template.Annotations["external"] != "keep" || + sts.Spec.Template.Labels["external"] != "keep" || svc.Annotations["external"] != "keep" { + t.Fatal("removing applied keys erased unrelated metadata") + } +} + +// exerciseReconcileConflictRetries injects a concurrent write before the reconciler's write to each kind +// and checks that the stale write conflicts and the retry keeps both the concurrent and the desired change. +func exerciseReconcileConflictRetries(t *testing.T, ctx context.Context, cl client.WithWatch, z *api.ZookeeperCluster, kinds ...string) { + t.Helper() + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + for _, kind := range kinds { + t.Run(kind, func(t *testing.T) { + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + switch kind { + case "StatefulSet": + z.Spec.Pod.NodeSelector = map[string]string{"pool": "new"} + case "Service": + z.Spec.ClientService.Annotations = map[string]string{"requested": "new"} + case "PodDisruptionBudget": + z.Spec.MaxUnavailableReplicas = 2 + case "status": + z.Status.InternalClientEndpoint = "stale" + if err := cl.Status().Update(ctx, z); err != nil { + t.Fatal(err) + } + } + if kind != "status" { + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + } + injected := false + concurrent := func(ctx context.Context, c client.Client, obj client.Object, status bool) error { + matches := false + switch obj.(type) { + case *appsv1.StatefulSet: + matches = kind == "StatefulSet" + case *corev1.Service: + matches = kind == "Service" + case *policyv1.PodDisruptionBudget: + matches = kind == "PodDisruptionBudget" + case *api.ZookeeperCluster: + matches = kind == "status" && status + } + if !matches || injected { + return nil + } + injected = true + fresh := obj.DeepCopyObject().(client.Object) + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), fresh); err != nil { + return err + } + annotations := fresh.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations["external"] = "concurrent" + fresh.SetAnnotations(annotations) + return c.Update(ctx, fresh) + } + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if err := concurrent(ctx, c, obj, false); err != nil { + return err + } + return c.Update(ctx, obj, opts...) + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if err := concurrent(ctx, c, obj, false); err != nil { + return err + } + return c.Patch(ctx, obj, patch, opts...) + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if err := concurrent(ctx, c, obj, true); err != nil { + return err + } + return c.SubResource(sub).Update(ctx, obj, opts...) + }, + SubResourcePatch: func(ctx context.Context, c client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + if err := concurrent(ctx, c, obj, true); err != nil { + return err + } + return c.SubResource(sub).Patch(ctx, obj, patch, opts...) + }, + }) + _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(z)}) + if !injected || !apierrors.IsConflict(err) { + t.Fatalf("stale %s write must conflict: injected=%v err=%v", kind, injected, err) + } + r.Client = cl + runReconcile(t, ctx, r, z) + var obj client.Object + key := client.ObjectKeyFromObject(z) + switch kind { + case "StatefulSet": + obj = &appsv1.StatefulSet{} + case "Service": + obj = &corev1.Service{} + key.Name = z.GetClientServiceName() + case "PodDisruptionBudget": + obj = &policyv1.PodDisruptionBudget{} + default: + obj = &api.ZookeeperCluster{} + } + if err := cl.Get(ctx, key, obj); err != nil { + t.Fatal(err) + } + if obj.GetAnnotations()["external"] != "concurrent" { + t.Fatal("retry overwrote concurrent metadata") + } + switch got := obj.(type) { + case *appsv1.StatefulSet: + if got.Spec.Template.Spec.NodeSelector["pool"] != "new" { + t.Fatal("retry lost desired node selector") + } + case *corev1.Service: + if got.Annotations["requested"] != "new" { + t.Fatal("retry lost desired Service annotation") + } + case *policyv1.PodDisruptionBudget: + if got.Spec.MaxUnavailable.IntValue() != 2 { + t.Fatal("retry lost desired disruption budget") + } + case *api.ZookeeperCluster: + if got.Status.InternalClientEndpoint == "stale" { + t.Fatal("retry did not persist observed endpoint") + } + } + }) + } +} + +func TestLoggingAndUpgradeStatusRemainCorrect(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + ctx := t.Context() + cl := r.Client.(client.WithWatch) + r.APIReader = cl + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Status.Replicas, sts.Status.ReadyReplicas = 1, 1 + sts.Status.CurrentRevision, sts.Status.UpdateRevision = "old", "logging" + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + initialImage := z.Spec.Image.Tag + z.Status.SetPodsReadyConditionTrue() + z.Status.CurrentVersion = initialImage + if err := cl.Status().Update(ctx, z); err != nil { + t.Fatal(err) + } + for _, logging := range []api.LoggingConfig{{Level: "WARN", Format: "json"}, {Level: "INFO", Format: "text"}} { + z.Spec.Logging = logging + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + config := sts.Spec.Template.Annotations["zookeeper.pravega.io/logback-config"] + if !strings.Contains(config, ``) { + t.Fatalf("logging change did not reach rollout annotation: %q", config) + } + if logging.Level == "INFO" && sts.Spec.Template.Annotations["zookeeper.pravega.io/logback-hash"] != "" { + t.Fatal("logging rollback retained the non-default hash") + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + if z.Status.IsClusterInUpgradingState() || z.Status.CurrentVersion != initialImage || z.Status.TargetVersion != "" { + t.Fatalf("logging-only rollout changed image upgrade state: %+v", z.Status) + } + } + z.Spec.Image.Tag = "next" + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + sts.Status.CurrentRevision, sts.Status.UpdateRevision = "old", "new" + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + // A later failure must not discard the intentional intermediate upgrade status. + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Service); ok { + return errors.New("later Service read failed") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + if _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(z)}); err == nil { + t.Fatal("expected later read failure") + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + if !z.Status.IsClusterInUpgradingState() || z.Status.TargetVersion != "next" || z.Status.IsClusterInReadyState() { + t.Fatalf("intermediate upgrade transition was lost: %+v", z.Status) + } + r.Client = cl + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Status.CurrentRevision, sts.Status.UpdateRevision = "new", "new" + sts.Status.Replicas, sts.Status.ReadyReplicas = 1, 1 + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + if z.Status.IsClusterInUpgradingState() || z.Status.CurrentVersion != "next" || z.Status.TargetVersion != "" { + t.Fatalf("upgrade completion was lost: %+v", z.Status) + } + var writes []string + r.Client = observeMutations(cl, &writes) + runReconcile(t, ctx, r, z) + if unexpected := writesExcept(writes, workloadWrites...); len(unexpected) != 0 { + t.Fatalf("completed upgrade must settle: %v", unexpected) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + z.Status.CurrentVersion, z.Status.TargetVersion = "old", "next" + z.Status.SetUpgradingConditionTrue(api.UpdatingZookeeperReason, "0") + z.Status.SetErrorConditionTrue("UpgradeFailed", "deadline exceeded") + z.Status.SetPodsReadyConditionFalse() + if err := cl.Status().Update(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + sts.Status.ReadyReplicas = 0 + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + writes = nil + runReconcile(t, ctx, r, z) + if unexpected := writesExcept(writes, workloadWrites...); len(unexpected) != 0 { + t.Fatalf("unchanged failed upgrade must not write: %v", unexpected) + } + sts.Status.ReadyReplicas = 1 + if err := cl.Status().Update(ctx, sts); err != nil { + t.Fatal(err) + } + r.Client = cl + settleCluster(t, ctx, r, z) + if z.Status.IsClusterInUpgradeFailedState() || z.Status.IsClusterInUpgradingState() || + z.Status.CurrentVersion != "next" || z.Status.TargetVersion != "" { + t.Fatalf("failed upgrade recovery was lost: %+v", z.Status) + } +} + +func convergenceReconciler(cl client.WithWatch) *ZookeeperClusterReconciler { + return &ZookeeperClusterReconciler{ + Client: cl, APIReader: cl, Scheme: cl.Scheme(), Log: log, + NewZkClient: func() zk.ZookeeperClient { return &MockZookeeperClient{} }, + } +} + +func runReconcile(t *testing.T, ctx context.Context, r *ZookeeperClusterReconciler, z *api.ZookeeperCluster) { + t.Helper() + if _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(z)}); err != nil { + t.Fatal(err) + } +} + +func settleCluster(t *testing.T, ctx context.Context, r *ZookeeperClusterReconciler, z *api.ZookeeperCluster) { + t.Helper() + for range 4 { + runReconcile(t, ctx, r, z) + } + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } +} + +func writesExcept(writes []string, ignored ...string) []string { + var remaining []string + for _, write := range writes { + skip := false + for _, prefix := range ignored { + skip = skip || strings.HasPrefix(write, prefix) + } + if !skip { + remaining = append(remaining, write) + } + } + return remaining +} + +func observeMutations(cl client.WithWatch, writes *[]string) client.WithWatch { + record := func(verb string, obj client.Object) { + *writes = append(*writes, fmt.Sprintf("%s %T/%s", verb, obj, obj.GetName())) + } + return interceptor.NewClient(cl, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + record("create", obj) + return c.Create(ctx, obj, opts...) + }, + Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + record("update", obj) + return c.Update(ctx, obj, opts...) + }, + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + verb := "patch" + if patch.Type() == types.ApplyPatchType { + verb = "apply" + } + record(verb, obj) + return c.Patch(ctx, obj, patch, opts...) + }, + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + record("delete", obj) + return c.Delete(ctx, obj, opts...) + }, + SubResourceUpdate: func(ctx context.Context, c client.Client, sub string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + record(sub+" update", obj) + return c.SubResource(sub).Update(ctx, obj, opts...) + }, + SubResourcePatch: func(ctx context.Context, c client.Client, sub string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { + record(sub+" patch", obj) + return c.SubResource(sub).Patch(ctx, obj, patch, opts...) + }, + }) +} diff --git a/controllers/envtest_test.go b/controllers/envtest_test.go index ac4a99af7..1c132876c 100644 --- a/controllers/envtest_test.go +++ b/controllers/envtest_test.go @@ -7,46 +7,80 @@ package controllers import ( "context" + "errors" + "fmt" "os" "path/filepath" + "reflect" + "slices" + "strings" + "sync" "testing" "time" api "github.com/pravega/zookeeper-operator/api/v1beta1" + "github.com/pravega/zookeeper-operator/pkg/zk" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/envtest" "sigs.k8s.io/yaml" ) +var defaultEnvtest = envtest.Environment{ + UseExistingCluster: new(bool), + CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, + ErrorIfCRDPathMissing: true, +} + +var startDefaultEnvtest = sync.OnceValues(defaultEnvtest.Start) + +func TestMain(m *testing.M) { + code := m.Run() + if defaultEnvtest.Config != nil { + if err := defaultEnvtest.Stop(); err != nil { + fmt.Fprintf(os.Stderr, "stop shared envtest API server: %v\n", err) + code = 1 + } + } + os.Exit(code) +} + func newEnvtestCluster(t *testing.T, crds ...*apiextensionsv1.CustomResourceDefinition) (context.Context, client.WithWatch, *api.ZookeeperCluster) { t.Helper() - environment := &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")}, - ErrorIfCRDPathMissing: true, - } - if len(crds) > 0 { - environment.CRDDirectoryPaths = nil - environment.CRDs = crds + var cfg *rest.Config + var err error + if len(crds) == 0 { + cfg, err = startDefaultEnvtest() + } else { + environment := &envtest.Environment{UseExistingCluster: new(bool), CRDs: crds} + cfg, err = environment.Start() + if err == nil { + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Errorf("stop isolated API server: %v", err) + } + }) + } } - cfg, err := environment.Start() if err != nil { t.Fatalf("start isolated API server (run make test-envtest to provision assets): %v", err) } - t.Cleanup(func() { - if err := environment.Stop(); err != nil { - t.Errorf("stop isolated API server: %v", err) - } - }) scheme := runtime.NewScheme() if err := corev1.AddToScheme(scheme); err != nil { t.Fatal(err) @@ -93,6 +127,25 @@ func newEnvtestCluster(t *testing.T, crds ...*apiextensionsv1.CustomResourceDefi t.Error(err) } } + for _, list := range []client.ObjectList{ + &appsv1.StatefulSetList{}, &corev1.ServiceList{}, &policyv1.PodDisruptionBudgetList{}, &corev1.PodList{}, + &corev1.ConfigMapList{}, &corev1.ServiceAccountList{}, &corev1.PersistentVolumeClaimList{}, + } { + if err := c.List(ctx, list, client.InNamespace(namespace.Name)); err != nil { + t.Error(err) + continue + } + items, err := meta.ExtractList(list) + if err != nil { + t.Error(err) + continue + } + for _, item := range items { + if err := client.IgnoreNotFound(c.Delete(ctx, item.(client.Object), client.GracePeriodSeconds(0))); err != nil { + t.Error(err) + } + } + } if err := c.Delete(ctx, namespace); err != nil { t.Error(err) return @@ -351,6 +404,733 @@ func TestEnvtestLegacyStorageObjectRemainsUpdatable(t *testing.T) { } } +func TestEnvtestReconcileSettles(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Replicas = 2 + z.Spec.Conf.AdditionalConfig = map[string]string{"custom.a": "one", "custom.b": "two", "custom.c": "three"} + z.Spec.Pod.Annotations = map[string]string{"custom": "set"} + z.Spec.ClientService.Annotations = map[string]string{"custom": "set"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 5) +} + +func TestEnvtestReconcileConflictRetries(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + exerciseReconcileConflictRetries(t, ctx, cl, z, "StatefulSet", "Service", "PodDisruptionBudget", "status") +} + +func TestEnvtestStatefulSetCacheMissDoesNotBypassScaleMetadata(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + key := client.ObjectKeyFromObject(z) + readStatefulSet := func() *appsv1.StatefulSet { + t.Helper() + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + return sts + } + before := readStatefulSet() + metadataErr := errors.New("metadata unavailable") + zkClient := &scaleMetadataClient{updateErr: metadataErr} + metadataAttempts := 0 + r.NewZkClient = func() zk.ZookeeperClient { + metadataAttempts++ + return zkClient + } + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, name client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok && name == key { + return apierrors.NewNotFound(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, name.Name) + } + return c.Get(ctx, name, obj, opts...) + }, + }) + if err := r.reconcileStatefulSet(ctx, z); !apierrors.IsAlreadyExists(err) { + t.Fatalf("expected the create to fail with AlreadyExists, got %v", err) + } + if got := readStatefulSet(); got.ResourceVersion != before.ResourceVersion || metadataAttempts != 0 { + t.Fatalf("cache miss changed the existing StatefulSet or attempted to update ZooKeeper: replicas=%d, metadata attempts=%d", *got.Spec.Replicas, metadataAttempts) + } + + r.Client = cl + if err := r.reconcileStatefulSet(ctx, z); !errors.Is(err, metadataErr) { + t.Fatalf("retry should attempt to update ZooKeeper before scaling: %v", err) + } + if got := readStatefulSet(); got.ResourceVersion != before.ResourceVersion { + t.Fatalf("StatefulSet changed despite the failed ZooKeeper metadata update: replicas=%d", *got.Spec.Replicas) + } + zkClient.updateErr = nil + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if got := readStatefulSet(); *got.Spec.Replicas != z.Spec.Replicas || metadataAttempts != 2 { + t.Fatalf("successful metadata retry did not scale as expected: replicas=%d, metadata attempts=%d", *got.Spec.Replicas, metadataAttempts) + } +} + +func TestEnvtestStatefulSetRejectsStaleParentSpec(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + fresh := z.DeepCopy() + fresh.Spec.Replicas = 1 + if err := cl.Update(ctx, fresh); err != nil { + t.Fatal(err) + } + var writes []string + r.Client = observeMutations(cl, &writes) + r.NewZkClient = func() zk.ZookeeperClient { + t.Fatal("stale parent must not start a resize") + return nil + } + if err := r.reconcileStatefulSet(ctx, z); !apierrors.IsConflict(err) || len(writes) != 0 { + t.Fatalf("stale parent must conflict without writing: err=%v writes=%v", err, writes) + } +} + +func TestEnvtestResizeRechecksParentWithinZooKeeperBudget(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + var sessionContext context.Context + r.NewZkClient = func() zk.ZookeeperClient { + return &resizeMetadataProbe{exists: func(zkCtx context.Context) error { + sessionContext = zkCtx + newer := z.DeepCopy() + newer.Spec.Replicas = 3 + return cl.Update(zkCtx, newer) + }, update: func() { t.Error("stale intent must not update ZooKeeper metadata") }} + } + r.APIReader = interceptor.NewClient(cl, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*api.ZookeeperCluster); ok && sessionContext != nil && ctx != sessionContext { + t.Error("post-NodeExists freshness check did not use the bounded ZooKeeper context") + } + return c.Get(ctx, key, obj, opts...) + }, + }) + var writes []string + r.Client = observeMutations(cl, &writes) + if err := r.reconcileStatefulSet(ctx, z); sessionContext == nil || !apierrors.IsConflict(err) || len(writes) != 0 { + t.Fatalf("stale resize must conflict after NodeExists without writing: err=%v writes=%v", err, writes) + } +} + +func TestEnvtestResizeRejectsChangedStatefulSetBeforeMetadataWrite(t *testing.T) { + for _, change := range []string{"deleted", "terminating", "replaced", "updated"} { + t.Run(change, func(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + sts := &appsv1.StatefulSet{} + key := client.ObjectKeyFromObject(z) + if err := cl.Get(ctx, key, sts); err != nil { + t.Fatal(err) + } + if change == "terminating" { + sts.Finalizers = []string{"example.com/hold"} + if err := cl.Update(ctx, sts); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := cl.Get(ctx, key, sts); err != nil { + t.Error(err) + return + } + sts.Finalizers = nil + if err := cl.Update(ctx, sts); err != nil { + t.Error(err) + } + }) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + changed, metadataWritten := false, false + r.NewZkClient = func() zk.ZookeeperClient { + return &resizeMetadataProbe{exists: func(ctx context.Context) error { + if change == "updated" { + sts.Annotations = map[string]string{"example.com/changed": "true"} + if err := cl.Update(ctx, sts); err != nil { + return err + } + } else if err := cl.Delete(ctx, sts); err != nil { + return err + } + if change == "replaced" { + replacement := zk.MakeStatefulSet(z) + if err := controllerutil.SetControllerReference(z, replacement, cl.Scheme()); err != nil { + return err + } + if err := cl.Create(ctx, replacement); err != nil { + return err + } + } + changed = true + return nil + }, update: func() { metadataWritten = true }} + } + err := r.reconcileStatefulSet(ctx, z) + if !changed || metadataWritten || !apierrors.IsConflict(err) && !apierrors.IsNotFound(err) { + t.Fatalf("changed StatefulSet must stop resize before writing metadata: changed=%v metadataWritten=%v err=%v", changed, metadataWritten, err) + } + }) + } +} + +type resizeMetadataProbe struct { + MockZookeeperClient + exists func(context.Context) error + update func() +} + +func (c *resizeMetadataProbe) NodeExists(ctx context.Context, _ string) (int32, error) { + return 0, c.exists(ctx) +} + +func (c *resizeMetadataProbe) UpdateNode(context.Context, string, string, int32) error { + c.update() + return nil +} + +func TestEnvtestApplyDoesNotRecreateDeletedStatefulSet(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + key := client.ObjectKeyFromObject(z) + deleted := false + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok && patch.Type() == types.ApplyPatchType && !deleted { + if err := c.Delete(ctx, obj); err != nil { + return err + } + deleted = true + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + if err := r.reconcileStatefulSet(ctx, z); err == nil || !deleted { + t.Fatalf("apply must not create after deletion: deleted=%v err=%v", deleted, err) + } + if err := cl.Get(ctx, key, &appsv1.StatefulSet{}); !apierrors.IsNotFound(err) { + t.Fatalf("StatefulSet should stay deleted until the next reconcile: %v", err) + } + r.Client = cl + if err := r.reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, key, &appsv1.StatefulSet{}); err != nil { + t.Fatalf("create path did not recreate the StatefulSet: %v", err) + } +} + +func TestEnvtestReconcileRejectsStatefulSetOwnershipConflict(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + sts := zk.MakeStatefulSet(z) + if err := cl.Create(ctx, sts, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + z.Spec.Replicas = 2 + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + var metadataAttempts int + r.NewZkClient = func() zk.ZookeeperClient { + metadataAttempts++ + return &MockZookeeperClient{} + } + for _, scenario := range []string{"no controller", "previous cluster UID", "failed upgrade"} { + t.Run(scenario, func(t *testing.T) { + switch scenario { + case "previous cluster UID": + previous := z.DeepCopy() + previous.UID = "previous-cluster" + if err := controllerutil.SetControllerReference(previous, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Update(ctx, sts); err != nil { + t.Fatal(err) + } + case "failed upgrade": + z.Status.SetErrorConditionTrue("UpgradeFailed", "upgrade timed out") + if err := cl.Status().Update(ctx, z); err != nil { + t.Fatal(err) + } + } + if err := r.reconcileStatefulSet(ctx, z); err == nil || !strings.Contains(err.Error(), "is not controlled by ZookeeperCluster") { + t.Fatalf("expected an ownership error, got %v", err) + } + got := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), got); err != nil { + t.Fatal(err) + } + if got.ResourceVersion != sts.ResourceVersion || metadataAttempts != 0 { + t.Fatalf("ownership conflict caused a StatefulSet write or ZooKeeper metadata update: metadata attempts=%d", metadataAttempts) + } + if scenario == "failed upgrade" && !z.Status.IsClusterInUpgradeFailedState() { + t.Fatal("an unrelated StatefulSet cleared the failed upgrade state") + } + }) + } +} + +func TestEnvtestReconcileRejectsServiceNameCollision(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + for _, test := range []struct { + name string + svc *corev1.Service + reconcile reconcileFun + }{ + {"unowned client Service", zk.MakeClientService(z), r.reconcileClientService}, + {"headless Service from previous cluster UID", zk.MakeHeadlessService(z), r.reconcileHeadlessService}, + {"concurrently created admin Service", zk.MakeAdminServerService(z), r.reconcileAdminServerService}, + } { + t.Run(test.name, func(t *testing.T) { + svc := test.svc + svc.Spec.Selector["app"] = "another-app" + key := client.ObjectKeyFromObject(svc) + if svc.Spec.ClusterIP == corev1.ClusterIPNone { + previous := z.DeepCopy() + previous.UID = "previous-cluster" + if err := controllerutil.SetControllerReference(previous, svc, cl.Scheme()); err != nil { + t.Fatal(err) + } + } + concurrent := svc.Name == z.GetAdminServerServiceName() + if concurrent { + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, name client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if name != key { + return c.Get(ctx, name, obj, opts...) + } + if err := c.Create(ctx, svc, client.FieldOwner("zookeeper-operator")); err != nil { + return err + } + return apierrors.NewNotFound(schema.GroupResource{Resource: "services"}, name.Name) + }, + }) + defer func() { r.Client = cl }() + } else if err := cl.Create(ctx, svc, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + err := test.reconcile(ctx, z) + if concurrent { + if !apierrors.IsAlreadyExists(err) || svc.ResourceVersion == "" { + t.Fatalf("concurrent create must return AlreadyExists, got %v", err) + } + } else if err == nil || !strings.Contains(err.Error(), "is not controlled by ZookeeperCluster") { + t.Fatalf("expected an ownership error, got %v", err) + } + got := &corev1.Service{} + if err := cl.Get(ctx, key, got); err != nil { + t.Fatal(err) + } + if got.ResourceVersion != svc.ResourceVersion { + t.Fatalf("Service changed despite the name collision: selector %v, owners %v", got.Spec.Selector, got.OwnerReferences) + } + }) + } +} + +func TestEnvtestReconcileZookeeperEnvOverride(t *testing.T) { + for _, mode := range []string{"new", "legacy"} { + t.Run(mode, func(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + optional := true + env := []corev1.EnvVar{ + {Name: "ORIGINAL_STATUS", Value: "$(ENVOY_SIDECAR_STATUS)"}, + {Name: "BASE", Value: "/srv"}, + {Name: "BASE", Value: "$(BASE)/data"}, + {Name: "ENVOY_SIDECAR_STATUS", ValueFrom: &corev1.EnvVarSource{ConfigMapKeyRef: &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "optional-config"}, Key: "status", Optional: &optional, + }}}, + } + if mode == "legacy" { + sts := zk.MakeStatefulSet(z) + container := &sts.Spec.Template.Spec.Containers[0] + // The previous operator appended user variables to the built-in variable. + container.Env = append(container.Env, env...) + if err := controllerutil.SetControllerReference(z, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, sts, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + } + z.Spec.Pod.Env = env + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + got := sts.Spec.Template.Spec.Containers[0].Env + if len(got) != len(env)+1 || got[0].Name != "ENVOY_SIDECAR_STATUS" || got[0].ValueFrom == nil || got[0].ValueFrom.FieldRef == nil || !reflect.DeepEqual(got[1:], env) { + t.Fatalf("expected the generated fallback and the exact ordered env overrides: got %+v", got) + } + fallback := got[0] + z.Spec.Pod.Env = nil + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + if got := sts.Spec.Template.Spec.Containers[0].Env; !reflect.DeepEqual(got, []corev1.EnvVar{fallback}) { + t.Fatalf("removing duplicate overrides should leave only the generated fallback: got %+v", got) + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + }) + } +} + +func TestEnvtestReconcileRestoresInternalAdminService(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + admin := &corev1.Service{} + key := client.ObjectKey{Namespace: z.Namespace, Name: z.GetAdminServerServiceName()} + if err := cl.Get(ctx, key, admin); err != nil { + t.Fatal(err) + } + admin.Spec.Type = corev1.ServiceTypeLoadBalancer + admin.Spec.LoadBalancerSourceRanges = []string{"192.0.2.0/24"} + if err := cl.Update(ctx, admin, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + if err := cl.Get(ctx, key, admin); err != nil { + t.Fatal(err) + } + if admin.Spec.Type != corev1.ServiceTypeClusterIP || admin.Spec.Ports[0].NodePort != 0 || len(admin.Spec.LoadBalancerSourceRanges) != 0 { + t.Fatalf("expected an internal admin Service without a NodePort or load balancer source ranges, got %+v", admin.Spec) + } +} + +func TestEnvtestReconcileDefaultsConverge(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Containers = []corev1.Container{{ + Name: "sidecar", Image: "busybox:1", + Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1.1m"), + }}, + StartupProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{ + GRPC: &corev1.GRPCAction{Port: 9090}, + }}, + ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{Port: intstr.FromInt32(8080)}, + }}, + }} + z.Spec.Volumes = []corev1.Volume{{ + Name: "secret", VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{SecretName: "example"}, + }, + }, { + Name: "empty", + }} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + patches := map[types.PatchType]int{} + r.Client = interceptor.NewClient(cl, interceptor.Funcs{ + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + if _, ok := obj.(*appsv1.StatefulSet); ok { + patches[patch.Type()]++ + } + return c.Patch(ctx, obj, patch, opts...) + }, + }) + const passes = 3 + assertEnvtestSettled(t, ctx, r, cl, z, passes) + if compatibility, apply := patches[types.MergePatchType], patches[types.ApplyPatchType]; compatibility != 0 || apply != passes { + t.Fatalf("settled reconciles sent %d StatefulSet compatibility patches and %d applies, want 0 and %d", compatibility, apply, passes) + } + r.Client = cl + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + if sts.Spec.Template.Spec.DNSPolicy != corev1.DNSClusterFirst || + sts.Spec.Template.Spec.Containers[1].TerminationMessagePath != "/dev/termination-log" { + t.Fatal("test requires real API-defaulted Pod-template fields") + } + if !slices.ContainsFunc(sts.Spec.Template.Spec.Volumes, func(v corev1.Volume) bool { + return v.Name == "empty" && v.EmptyDir != nil + }) { + t.Fatal("test requires the API to default the sourceless volume to EmptyDir") + } + svc := &corev1.Service{} + svcKey := types.NamespacedName{Name: z.GetClientServiceName(), Namespace: z.Namespace} + if err := cl.Get(ctx, svcKey, svc); err != nil { + t.Fatal(err) + } + if svc.Spec.Ports[0].Protocol != corev1.ProtocolTCP || svc.Spec.Ports[0].TargetPort.IntVal != 2181 { + t.Fatal("test requires real API-defaulted Service ports") + } + for _, external := range []bool{true, false} { + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + z.Spec.AdminServerService.External = external + z.Spec.DomainName = "" + if external { + z.Spec.DomainName = "example.test" + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + admin := &corev1.Service{} + if err := cl.Get(ctx, client.ObjectKey{Name: z.GetAdminServerServiceName(), Namespace: z.Namespace}, admin); err != nil { + t.Fatal(err) + } + if external { + if admin.Spec.Type != corev1.ServiceTypeLoadBalancer || admin.Spec.Ports[0].NodePort == 0 { + t.Fatal("external admin service did not get an allocated NodePort") + } + } else if admin.Spec.Type != corev1.ServiceTypeClusterIP || admin.Spec.Ports[0].NodePort != 0 { + t.Fatal("removing the external service did not clear its NodePort") + } + headless := &corev1.Service{} + if err := cl.Get(ctx, client.ObjectKey{Name: z.Name + "-headless", Namespace: z.Namespace}, headless); err != nil { + t.Fatal(err) + } + if _, hasDNS := headless.Annotations["external-dns.alpha.kubernetes.io/hostname"]; hasDNS != external { + t.Fatal("external-DNS annotation change or removal did not converge") + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) + } +} + +func TestEnvtestReconcileAdoptsLegacyResources(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + createLegacyResources(t, ctx, cl, z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + sts.Spec.Template.Spec.Containers[0].Args = []string{"external-argument"} + if err := cl.Update(ctx, sts, client.FieldOwner("external-injector")); err != nil { + t.Fatal(err) + } + clientSvc := &corev1.Service{} + clientKey := client.ObjectKey{Name: z.GetClientServiceName(), Namespace: z.Namespace} + if err := cl.Get(ctx, clientKey, clientSvc); err != nil { + t.Fatal(err) + } + clientSvc.Annotations = map[string]string{"external-dns.alpha.kubernetes.io/hostname": "zk.user.example"} + clientSvc.Spec.Ports = append(clientSvc.Spec.Ports, corev1.ServicePort{Name: "extra", Port: 8081}) + if err := cl.Update(ctx, clientSvc, client.FieldOwner("user")); err != nil { + t.Fatal(err) + } + generation, clusterIP := sts.Generation, clientSvc.Spec.ClusterIP + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if sts.Generation != generation { + t.Fatalf("adopting a legacy StatefulSet changed its spec: generation %d -> %d", generation, sts.Generation) + } + if _, ok := sts.Labels["owner-rv"]; ok { + t.Error("legacy owner-rv label was not removed") + } + if args := sts.Spec.Template.Spec.Containers[0].Args; len(args) != 1 || args[0] != "external-argument" { + t.Error("adoption removed a field owned by another manager") + } + for _, entry := range sts.ManagedFields { + if legacyFieldManagers.Has(entry.Manager) { + t.Errorf("legacy field manager %q was not migrated", entry.Manager) + } + } + if err := cl.Get(ctx, clientKey, clientSvc); err != nil { + t.Fatal(err) + } + if clientSvc.Spec.ClusterIP != clusterIP { + t.Error("adoption changed the allocated cluster IP") + } + if clientSvc.Annotations["external-dns.alpha.kubernetes.io/hostname"] != "zk.user.example" { + t.Error("adoption removed a user-owned external-DNS annotation") + } + if !slices.ContainsFunc(clientSvc.Spec.Ports, func(port corev1.ServicePort) bool { + return port.Name == "extra" && port.Port == 8081 + }) { + t.Error("adoption removed a user-owned Service port") + } + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +func TestEnvtestReconcileLegacyOwnedFieldRemoval(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Pod.NodeSelector = map[string]string{"pool": "old"} + z.Spec.Pod.Env = []corev1.EnvVar{{Name: "OWNED", Value: "remove"}} + z.Spec.Pod.Annotations = map[string]string{"custom": "remove"} + z.Spec.Containers = []corev1.Container{ + {Name: "sidecar", Image: "busybox:1"}, + {Name: "retired", Image: "busybox:1"}, + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + createLegacyResources(t, ctx, cl, z) + sts := &appsv1.StatefulSet{} + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), sts); err != nil { + t.Fatal(err) + } + for i := range sts.Spec.Template.Spec.Containers { + container := &sts.Spec.Template.Spec.Containers[i] + if container.Name == "sidecar" { + container.Args = []string{"external-argument"} + } + if container.Name == "zookeeper" { + container.Env = append(container.Env, corev1.EnvVar{Name: "EXTERNAL", Value: "keep"}) + } + } + if err := cl.Update(ctx, sts, client.FieldOwner("external-injector")); err != nil { + t.Fatal(err) + } + // The CR changes before the new operator first reconciles the legacy StatefulSet. + z.Spec.Pod.NodeSelector = nil + z.Spec.Pod.Env = nil + z.Spec.Pod.Annotations = nil + z.Spec.Containers = []corev1.Container{{Name: "sidecar", Image: "busybox:2"}} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + r := convergenceReconciler(cl) + settleCluster(t, ctx, r, z) + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + if len(sts.Spec.Template.Spec.NodeSelector) != 0 || len(sts.Spec.Template.Spec.Containers) != 2 { + t.Fatal("fields the legacy operator rendered were not removed") + } + if _, ok := sts.Spec.Template.Annotations["custom"]; ok { + t.Error("Pod annotation the legacy operator rendered was not removed") + } + externalEnv := false + for _, container := range sts.Spec.Template.Spec.Containers { + if container.Name == "sidecar" && + (container.Image != "busybox:2" || len(container.Args) != 1 || container.Args[0] != "external-argument") { + t.Error("sidecar image change erased an unowned container field") + } + for _, env := range container.Env { + if env.Name == "OWNED" { + t.Error("environment entry the legacy operator rendered was not removed") + } + externalEnv = externalEnv || env.Name == "EXTERNAL" + } + } + if !externalEnv { + t.Error("removing an owned environment entry erased an unowned entry") + } + for _, logging := range []api.LoggingConfig{{Level: "WARN", Format: "json"}, {Level: "INFO", Format: "text"}} { + if err := cl.Get(ctx, client.ObjectKeyFromObject(z), z); err != nil { + t.Fatal(err) + } + z.Spec.Logging = logging + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + runReconcile(t, ctx, r, z) + } + settleCluster(t, ctx, r, z) + assertEnvtestSettled(t, ctx, r, cl, z, 3) +} + +// createLegacyResources creates the StatefulSet and Services as earlier operator versions did: +// with Create requests from the legacy field manager and the owner-rv label. +func createLegacyResources(t *testing.T, ctx context.Context, cl client.Client, z *api.ZookeeperCluster) { + t.Helper() + sts := zk.MakeStatefulSet(z) + zk.EnsureLogbackProjection(z, nil, sts) + sts.Labels = map[string]string{"owner-rv": z.ResourceVersion} + for key, value := range z.Spec.Labels { + sts.Labels[key] = value + } + objects := []client.Object{sts, zk.MakeClientService(z), zk.MakeHeadlessService(z), zk.MakeAdminServerService(z)} + for _, obj := range objects { + if err := controllerutil.SetControllerReference(z, obj, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, obj, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + } +} + +// assertEnvtestSettled checks that further reconciles do not change any persisted object. +func assertEnvtestSettled(t *testing.T, ctx context.Context, r *ZookeeperClusterReconciler, cl client.Client, z *api.ZookeeperCluster, passes int) { + t.Helper() + settled := envtestResourceVersions(t, ctx, cl, z) + for range passes { + runReconcile(t, ctx, r, z) + } + if got := envtestResourceVersions(t, ctx, cl, z); !reflect.DeepEqual(got, settled) { + t.Fatalf("settled reconciliation changed persisted objects:\nbefore %v\nafter %v", settled, got) + } +} + +// envtestResourceVersions returns the resourceVersion of every object the reconciler writes. +func envtestResourceVersions(t *testing.T, ctx context.Context, cl client.Client, z *api.ZookeeperCluster) map[string]string { + t.Helper() + objects := map[string]client.Object{ + "cluster": &api.ZookeeperCluster{ObjectMeta: metav1.ObjectMeta{Name: z.Name}}, + "statefulset": &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: z.Name}}, + "configmap": &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: z.ConfigMapName()}}, + "pdb": &policyv1.PodDisruptionBudget{ObjectMeta: metav1.ObjectMeta{Name: z.Name}}, + "client": &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: z.GetClientServiceName()}}, + "headless": &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: z.Name + "-headless"}}, + "admin": &corev1.Service{ObjectMeta: metav1.ObjectMeta{Name: z.GetAdminServerServiceName()}}, + } + versions := map[string]string{} + for name, obj := range objects { + if err := cl.Get(ctx, client.ObjectKey{Name: obj.GetName(), Namespace: z.Namespace}, obj); err != nil { + t.Fatalf("get %s: %v", name, err) + } + versions[name] = obj.GetResourceVersion() + } + return versions +} + func TestEnvtestStatusSubresource(t *testing.T) { ctx, c, cluster := newEnvtestCluster(t) cluster.Status.ReadyReplicas = 1 diff --git a/controllers/server_side_apply.go b/controllers/server_side_apply.go new file mode 100644 index 000000000..9916bf44b --- /dev/null +++ b/controllers/server_side_apply.go @@ -0,0 +1,145 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + "slices" + + "github.com/pravega/zookeeper-operator/api/v1beta1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/csaupgrade" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// applyFieldManager owns the StatefulSet and Service fields that the operator renders. +const applyFieldManager = "zookeeper-operator-apply" + +// StatefulSets and Services use create-only requests so cache misses cannot overwrite live objects. +const createFieldManager = "zookeeper-operator-create" + +// Fields owned by Create and Update managers move to applyFieldManager so that an apply +// can remove fields the operator no longer renders. +var legacyFieldManagers = sets.New("zookeeper-operator", createFieldManager) + +// applyStatefulSet updates an existing StatefulSet without changing its immutable fields. +func (r *ZookeeperClusterReconciler) applyStatefulSet(ctx context.Context, current, desired *appsv1.StatefulSet) error { + intent := desired.DeepCopy() + intent.Spec.Selector = current.Spec.Selector + intent.Spec.ServiceName = current.Spec.ServiceName + intent.Spec.PodManagementPolicy = current.Spec.PodManagementPolicy + intent.Spec.VolumeClaimTemplates = current.Spec.VolumeClaimTemplates + operatorFields, externalFields := statefulSetFields(current) + defaultRenderedPullPolicies(&intent.Spec.Template.Spec, externalFields) + prepared := current.DeepCopy() + retired := removeSharedRetiredContainers(&prepared.Spec.Template.Spec, &desired.Spec.Template.Spec, operatorFields, externalFields) + portDrift := removeStaleContainerPorts(&prepared.Spec.Template.Spec, &desired.Spec.Template.Spec) + orderedLists := normalizeContainerLists(&intent.Spec.Template.Spec) + sourceDrift := clearIncompatibleSources(&prepared.Spec.Template.Spec, &intent.Spec.Template.Spec) + securityDrift := clearInheritedSecurityOverrides(&prepared.Spec.Template.Spec, &intent.Spec.Template.Spec) + securityDrift = clearConflictingSeccompAnnotations(&prepared.Spec.Template, &intent.Spec.Template) || securityDrift + retiringVolumes := retiringVolumesForMerge(&prepared.Spec.Template.Spec, &intent.Spec.Template.Spec, operatorFields, externalFields) + if orderedLists || portDrift || sourceDrift || retired || securityDrift || len(retiringVolumes) != 0 { + return r.applyCompatibleStatefulSet(ctx, current, prepared, intent, desired, retiringVolumes) + } + return r.apply(ctx, current, intent) +} + +// applyService updates an existing Service only if this cluster controls it. +func (r *ZookeeperClusterReconciler) applyService(ctx context.Context, cluster *v1beta1.ZookeeperCluster, current, desired *corev1.Service) error { + if err := requireClusterOwner(cluster, current, "service"); err != nil { + return err + } + intent := desired.DeepCopy() + prepared := current.DeepCopy() + external := intent.Spec.Type == corev1.ServiceTypeLoadBalancer + if external && (current.Spec.Type == corev1.ServiceTypeLoadBalancer || current.Spec.Type == corev1.ServiceTypeNodePort) { + // Carry the allocation by name even when the numeric port key changes. + // Keep it in later applies so ownership migration cannot prune it. + for i := range intent.Spec.Ports { + port := &intent.Spec.Ports[i] + if j := slices.IndexFunc(current.Spec.Ports, func(live corev1.ServicePort) bool { return live.Name == port.Name }); port.NodePort == 0 && j >= 0 { + port.NodePort = current.Spec.Ports[j].NodePort + } + } + } + prepared.Spec.Ports = slices.DeleteFunc(prepared.Spec.Ports, func(live corev1.ServicePort) bool { + if slices.ContainsFunc(intent.Spec.Ports, func(wanted corev1.ServicePort) bool { + return servicePortKey(live) == servicePortKey(wanted) && live.Name == wanted.Name + }) { + return false + } + return external || (live.Name != "" && slices.ContainsFunc(intent.Spec.Ports, func(wanted corev1.ServicePort) bool { return live.Name == wanted.Name })) + }) + const sourceRanges = "service.beta.kubernetes.io/load-balancer-source-ranges" + incompatibleRanges := false + if !external { + _, hasAnnotation := prepared.Annotations[sourceRanges] + incompatibleRanges = hasAnnotation || len(prepared.Spec.LoadBalancerSourceRanges) != 0 + delete(prepared.Annotations, sourceRanges) + delete(intent.Annotations, sourceRanges) + prepared.Spec.LoadBalancerSourceRanges = nil + } + if len(current.Spec.Ports) == len(prepared.Spec.Ports) && !incompatibleRanges { + return r.apply(ctx, current, intent) + } + // Publish the type, ports, and annotations together or not at all. + next := &corev1.Service{} + if err := r.mergeWorkloadLocally(prepared, intent, next); err != nil { + return err + } + next.Status = current.Status + if err := r.patchMergedWorkload(ctx, current, next); err != nil { + return err + } + *current = *next + return nil +} + +func servicePortKey(port corev1.ServicePort) portIdentity { return portKey(port.Port, port.Protocol) } + +func requireClusterOwner(cluster *v1beta1.ZookeeperCluster, obj client.Object, kind string) error { + if !metav1.IsControlledBy(obj, cluster) { + return fmt.Errorf("%s %s already exists and is not controlled by ZookeeperCluster %s", kind, client.ObjectKeyFromObject(obj), client.ObjectKeyFromObject(cluster)) + } + return nil +} + +// apply updates the observed object; its UID prevents apply from recreating a deleted object. +func (r *ZookeeperClusterReconciler) apply(ctx context.Context, current, intent client.Object) error { + if current.GetResourceVersion() == "" { + return fmt.Errorf("cannot apply %T %s without a resource version", current, client.ObjectKeyFromObject(current)) + } + migration, err := csaupgrade.UpgradeManagedFieldsPatch(current, legacyFieldManagers, applyFieldManager) + if err != nil { + return err + } + if migration != nil { + if err := r.Client.Patch(ctx, current, client.RawPatch(types.JSONPatchType, migration)); err != nil { + return err + } + } + intent.SetResourceVersion(current.GetResourceVersion()) + intent.SetUID(current.GetUID()) + data, err := json.Marshal(intent) + if err != nil { + return err + } + var body map[string]json.RawMessage + if err := json.Unmarshal(data, &body); err != nil { + return err + } + delete(body, "status") + if data, err = json.Marshal(body); err != nil { + return err + } + return r.Client.Patch(ctx, current, client.RawPatch(types.ApplyPatchType, data), + client.FieldOwner(applyFieldManager), client.ForceOwnership) +} diff --git a/controllers/server_side_apply_test.go b/controllers/server_side_apply_test.go new file mode 100644 index 000000000..b119d8bcd --- /dev/null +++ b/controllers/server_side_apply_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "testing" + + "github.com/pravega/zookeeper-operator/pkg/zk" + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestApplyPreservesIntegerValues(t *testing.T) { + r, z, sts := newScaleMetadataTestReconciler(t, 1) + ctx := t.Context() + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + const gracePeriod = 9007199254740993 + z.Spec.Pod.TerminationGracePeriodSeconds = gracePeriod + applies := 0 + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{ + Patch: func(_ context.Context, _ client.WithWatch, obj client.Object, patch client.Patch, _ ...client.PatchOption) error { + if patch.Type() != types.ApplyPatchType { + t.Fatalf("expected apply, got %v", patch.Type()) + } + applies++ + data, err := patch.Data(obj) + if err != nil { + return err + } + intent := &appsv1.StatefulSet{} + if err := json.Unmarshal(data, intent); err != nil { + return err + } + if got := intent.Spec.Template.Spec.TerminationGracePeriodSeconds; got == nil || *got != gracePeriod { + t.Errorf("apply did not preserve the rendered grace period: %s", data) + } + return nil + }, + }) + if err := r.applyStatefulSet(ctx, sts, zk.MakeStatefulSet(z)); err != nil { + t.Fatal(err) + } + if applies != 1 { + t.Fatalf("expected one apply payload, got %d", applies) + } +} diff --git a/controllers/statefulset_compatibility.go b/controllers/statefulset_compatibility.go new file mode 100644 index 000000000..170103284 --- /dev/null +++ b/controllers/statefulset_compatibility.go @@ -0,0 +1,216 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "context" + "encoding/json" + "slices" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/managedfields" + applyconfigs "k8s.io/client-go/applyconfigurations" + "k8s.io/client-go/util/csaupgrade" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type portIdentity struct { + number int32 + protocol corev1.Protocol +} + +func portKey(number int32, protocol corev1.Protocol) portIdentity { + if protocol == "" { + protocol = corev1.ProtocolTCP + } + return portIdentity{number, protocol} +} + +func containerPortKey(port corev1.ContainerPort) portIdentity { + return portKey(port.ContainerPort, port.Protocol) +} + +func removeStaleContainerPorts(live, desired *corev1.PodSpec) bool { + changed := false + forRenderedContainers(live, desired, func(current, wanted *corev1.Container) { + current.Ports = slices.DeleteFunc(current.Ports, func(port corev1.ContainerPort) bool { + stale := port.Name != "" && slices.ContainsFunc(wanted.Ports, func(next corev1.ContainerPort) bool { + return next.Name == port.Name && containerPortKey(next) != containerPortKey(port) + }) + changed = changed || stale + return stale + }) + }) + return changed +} + +func normalizeContainerLists(spec *corev1.PodSpec) bool { + changed := false + for _, containers := range [][]corev1.Container{spec.Containers, spec.InitContainers} { + for i := range containers { + c := &containers[i] + ports, env := len(c.Ports), len(c.Env) + c.Ports = uniqueBy(c.Ports, containerPortKey) + c.Env = uniqueBy(c.Env, envKey) + changed = changed || len(c.Ports) != ports || len(c.Env) != env + } + } + return changed +} + +func envKey(env corev1.EnvVar) string { return env.Name } + +func uniqueBy[T any, K comparable](values []T, key func(T) K) []T { + seen := make(map[K]bool, len(values)) + return slices.DeleteFunc(values, func(value T) bool { + id := key(value) + previous := seen[id] + seen[id] = true + return previous + }) +} + +// Apply cannot express duplicate list keys. Merge an SSA-compatible intent locally, +// then restore the exact ordered lists and let the API server validate one guarded patch. +func (r *ZookeeperClusterReconciler) applyCompatibleStatefulSet(ctx context.Context, current, prepared, intent, desired *appsv1.StatefulSet, retiringVolumes []corev1.Volume) error { + next := &appsv1.StatefulSet{} + if err := r.mergeWorkloadLocally(prepared, intent, next); err != nil { + return err + } + if resolveRetiringVolumes(prepared, intent, next, retiringVolumes) { + // Keep complete sources while referenced; don't persist orphaned fragments. + next = &appsv1.StatefulSet{} + if err := r.mergeWorkloadLocally(prepared, intent, next); err != nil { + return err + } + } + restoreContainerLists(current.Spec.Template.Spec.Containers, next.Spec.Template.Spec.Containers, desired.Spec.Template.Spec.Containers, intent.Spec.Template.Spec.Containers) + restoreContainerLists(current.Spec.Template.Spec.InitContainers, next.Spec.Template.Spec.InitContainers, desired.Spec.Template.Spec.InitContainers, intent.Spec.Template.Spec.InitContainers) + next.Status = current.Status + if err := r.patchMergedWorkload(ctx, current, next); err != nil { + return err + } + *current = *next + return nil +} + +func (r *ZookeeperClusterReconciler) mergeWorkloadLocally(prepared, intent, next client.Object) error { + gvk := intent.GetObjectKind().GroupVersionKind() + prepared.GetObjectKind().SetGroupVersionKind(gvk) + if err := csaupgrade.UpgradeManagedFields(prepared, legacyFieldManagers, applyFieldManager); err != nil { + return err + } + manager, err := managedfields.NewDefaultFieldManager(applyconfigs.NewTypeConverter(r.Scheme), r.Scheme, r.Scheme, r.Scheme, gvk, gvk.GroupVersion(), "", nil) + if err != nil { + return err + } + payload, err := runtime.DefaultUnstructuredConverter.ToUnstructured(intent) + if err != nil { + return err + } + delete(payload, "status") + merged, err := manager.Apply(prepared, &unstructured.Unstructured{Object: payload}, applyFieldManager, true) + if err != nil { + return err + } + data, err := json.Marshal(merged) + if err != nil { + return err + } + return json.Unmarshal(data, next) +} + +func (r *ZookeeperClusterReconciler) patchMergedWorkload(ctx context.Context, current, next client.Object) error { + next.GetObjectKind().SetGroupVersionKind(current.GetObjectKind().GroupVersionKind()) + next.SetResourceVersion(current.GetResourceVersion()) + if equality.Semantic.DeepEqual(current, next) { + return nil + } + // Changes made by this non-apply PATCH must migrate back to the apply manager. + return r.Client.Patch(ctx, next, client.MergeFromWithOptions(current, client.MergeFromWithOptimisticLock{}), client.FieldOwner(createFieldManager)) +} + +func restoreContainerLists(live, merged, desired, normalized []corev1.Container) { + for i, container := range desired { + restorePorts := len(container.Ports) != len(normalized[i].Ports) + restoreEnv := len(container.Env) != len(normalized[i].Env) + if !restorePorts && !restoreEnv { + continue + } + for j := range merged { + if merged[j].Name != container.Name { + continue + } + var previous corev1.Container + if at := slices.IndexFunc(live, func(c corev1.Container) bool { return c.Name == container.Name }); at >= 0 { + previous = live[at] + } + if restorePorts { + merged[j].Ports = restoreList(previous.Ports, merged[j].Ports, container.Ports, containerPortKey) + } + if restoreEnv { + merged[j].Env = restoreList(previous.Env, merged[j].Env, container.Env, envKey) + } + break + } + } +} + +func restoreList[T any, K comparable](live, merged, desired []T, key func(T) K) []T { + positions := make(map[K][]int, len(desired)) + for i, value := range desired { + id := key(value) + positions[id] = append(positions[id], i) + } + resolved := slices.Clone(desired) + extra := make(map[K][]int) + for i, value := range merged { + id := key(value) + switch len(positions[id]) { + case 0: + extra[id] = append(extra[id], i) + case 1: + resolved[positions[id][0]] = value + } + } + // Anchor independent entries before the next rendered occurrence in the + // live list. Preserve their order even when the rendered order changes. + gaps := make(map[int][]T) + used := make([]bool, len(merged)) + anchor := len(resolved) + for i := len(live) - 1; i >= 0; i-- { + id := key(live[i]) + if indexes := positions[id]; len(indexes) != 0 { + anchor = min(anchor, indexes[len(indexes)-1]) + if len(indexes) > 1 { + positions[id] = indexes[:len(indexes)-1] + } + } else if indexes := extra[id]; len(indexes) != 0 { + index := indexes[len(indexes)-1] + gaps[anchor] = append(gaps[anchor], merged[index]) + used[index] = true + extra[id] = indexes[:len(indexes)-1] + } + } + result := make([]T, 0, len(resolved)) + for i, value := range resolved { + gap := gaps[i] + slices.Reverse(gap) + result = append(result, gap...) + result = append(result, value) + } + tail := gaps[len(resolved)] + slices.Reverse(tail) + result = append(result, tail...) + for i, value := range merged { + if !used[i] && len(positions[key(value)]) == 0 { + result = append(result, value) + } + } + return result +} diff --git a/controllers/statefulset_repairs.go b/controllers/statefulset_repairs.go new file mode 100644 index 000000000..f69954dcc --- /dev/null +++ b/controllers/statefulset_repairs.go @@ -0,0 +1,343 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "reflect" + "slices" + "strings" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" +) + +func clearInheritedSecurityOverrides(live, intent *corev1.PodSpec) bool { + pod := intent.SecurityContext + if pod == nil { + pod = &corev1.PodSecurityContext{} + } + changed := false + forRenderedContainers(live, intent, func(current, desired *corev1.Container) { + container := current.SecurityContext + if container == nil { + return + } + requested := desired.SecurityContext + if requested == nil { + requested = &corev1.SecurityContext{} + } + changed = clearOverride(&container.RunAsUser, pod.RunAsUser, requested.RunAsUser) || changed + changed = clearOverride(&container.RunAsGroup, pod.RunAsGroup, requested.RunAsGroup) || changed + changed = clearOverride(&container.RunAsNonRoot, pod.RunAsNonRoot, requested.RunAsNonRoot) || changed + changed = clearOverride(&container.SELinuxOptions, pod.SELinuxOptions, requested.SELinuxOptions) || changed + changed = clearOverride(&container.WindowsOptions, pod.WindowsOptions, requested.WindowsOptions) || changed + changed = clearOverride(&container.SeccompProfile, pod.SeccompProfile, requested.SeccompProfile) || changed + changed = clearOverride(&container.AppArmorProfile, pod.AppArmorProfile, requested.AppArmorProfile) || changed + if nonRoot := preferExplicit(requested.RunAsNonRoot, pod.RunAsNonRoot); nonRoot != nil && *nonRoot && requested.RunAsUser == nil && container.RunAsUser != nil && *container.RunAsUser == 0 { + container.RunAsUser = nil + changed = true + } + seccomp := preferExplicit(requested.SeccompProfile, pod.SeccompProfile) + apparmor := preferExplicit(requested.AppArmorProfile, pod.AppArmorProfile) + confinement := seccomp != nil && seccomp.Type != corev1.SeccompProfileTypeUnconfined || apparmor != nil && apparmor.Type != corev1.AppArmorProfileTypeUnconfined + if confinement && requested.Privileged == nil && container.Privileged != nil && *container.Privileged { + container.Privileged = nil + changed = true + } + }) + return changed +} + +func preferExplicit[T any](explicit, inherited *T) *T { + if explicit != nil { + return explicit + } + return inherited +} + +func clearOverride[T any](live **T, inherited, explicit *T) bool { + if *live == nil || inherited == nil || explicit != nil || equality.Semantic.DeepEqual(*live, inherited) { + return false + } + *live = nil + return true +} + +func clearConflictingSeccompAnnotations(live, intent *corev1.PodTemplateSpec) bool { + if len(live.Annotations) == 0 { + return false + } + changed := false + clear := func(key string, profile *corev1.SeccompProfile) { + annotation, exists := live.Annotations[key] + if _, explicit := intent.Annotations[key]; exists && !explicit && profile != nil && !matchesSeccompProfile(annotation, profile) { + delete(live.Annotations, key) + changed = true + } + } + var podProfile *corev1.SeccompProfile + if intent.Spec.SecurityContext != nil { + podProfile = intent.Spec.SecurityContext.SeccompProfile + } + clear(corev1.SeccompPodAnnotationKey, podProfile) + // A legacy container annotation can override the inherited pod profile. + for _, list := range [][]corev1.Container{intent.Spec.Containers, intent.Spec.InitContainers} { + for _, container := range list { + profile := podProfile + if container.SecurityContext != nil { + profile = preferExplicit(container.SecurityContext.SeccompProfile, podProfile) + } + clear(corev1.SeccompContainerAnnotationKeyPrefix+container.Name, profile) + } + } + return changed +} + +func matchesSeccompProfile(annotation string, profile *corev1.SeccompProfile) bool { + switch profile.Type { + case corev1.SeccompProfileTypeRuntimeDefault: + return annotation == corev1.SeccompProfileRuntimeDefault || annotation == corev1.DeprecatedSeccompProfileDockerDefault + case corev1.SeccompProfileTypeUnconfined: + return annotation == corev1.SeccompProfileNameUnconfined + case corev1.SeccompProfileTypeLocalhost: + return profile.LocalhostProfile != nil && annotation == corev1.SeccompLocalhostProfileNamePrefix+*profile.LocalhostProfile + default: + return false + } +} + +func statefulSetFields(current *appsv1.StatefulSet) (operator, external *fieldpath.Set) { + operator, external = fieldpath.NewSet(), fieldpath.NewSet() + legacy := legacyFieldManagers + for _, entry := range current.ManagedFields { + if entry.Subresource != "" || entry.FieldsV1 == nil || entry.APIVersion != "apps/v1" { + continue + } + fields := &fieldpath.Set{} + if fields.FromJSON(entry.FieldsV1.GetRawReader()) != nil { + continue + } + if entry.Manager == applyFieldManager && entry.Operation == metav1.ManagedFieldsOperationApply || legacy.Has(entry.Manager) && entry.Operation == metav1.ManagedFieldsOperationUpdate { + operator = operator.Union(fields) + } else { + external = external.Union(fields) + } + } + return operator, external +} + +func containerPath(list, name string) fieldpath.Path { + return fieldpath.MakePathOrDie("spec", "template", "spec", list, fieldpath.KeyByFields("name", name)) +} + +func referencedVolumes(spec *corev1.PodSpec) sets.Set[string] { + names := sets.New[string]() + for _, list := range [][]corev1.Container{spec.Containers, spec.InitContainers} { + for _, container := range list { + for _, mount := range container.VolumeMounts { + names.Insert(mount.Name) + } + for _, device := range container.VolumeDevices { + names.Insert(device.Name) + } + } + } + return names +} + +func retiringVolumesForMerge(live, desired *corev1.PodSpec, operator, external *fieldpath.Set) []corev1.Volume { + requested := sets.New[string]() + for _, volume := range desired.Volumes { + requested.Insert(volume.Name) + } + referenced := referencedVolumes(live) + var retiring []corev1.Volume + for _, volume := range live.Volumes { + if requested.Has(volume.Name) { + continue + } + path := fieldpath.MakePathOrDie("spec", "template", "spec", "volumes", fieldpath.KeyByFields("name", volume.Name)) + if ownsFieldOrChild(operator, path) && (referenced.Has(volume.Name) || ownsFieldOrChild(external, path)) { + retiring = append(retiring, volume) + } + } + return retiring +} + +func ownsFieldOrChild(fields *fieldpath.Set, path fieldpath.Path) bool { + if fields.Has(path) { + return true + } + for _, element := range path { + fields = fields.WithPrefix(element) + } + return !fields.Empty() +} + +func resolveRetiringVolumes(prepared, intent, merged *appsv1.StatefulSet, retiring []corev1.Volume) bool { + if len(retiring) == 0 { + return false + } + referenced := referencedVolumes(&merged.Spec.Template.Spec) + for _, claim := range merged.Spec.VolumeClaimTemplates { + referenced.Delete(claim.Name) + } + changed := false + volumes := merged.Spec.Template.Spec.Volumes + for _, volume := range retiring { + i := slices.IndexFunc(volumes, func(v corev1.Volume) bool { return v.Name == volume.Name }) + if i >= 0 && equality.Semantic.DeepEqual(volumes[i].VolumeSource, volume.VolumeSource) { + continue + } + if referenced.Has(volume.Name) { + intent.Spec.Template.Spec.Volumes = append(intent.Spec.Template.Spec.Volumes, volume) + changed = true + } else if i >= 0 { + // A co-owned fragment is not a complete independent volume definition. + live := &prepared.Spec.Template.Spec.Volumes + *live = slices.DeleteFunc(*live, func(v corev1.Volume) bool { return v.Name == volume.Name }) + changed = true + } + } + return changed +} + +func defaultRenderedPullPolicies(desired *corev1.PodSpec, external *fieldpath.Set) { + for _, list := range []struct { + name string + containers []corev1.Container + }{{"containers", desired.Containers}, {"initContainers", desired.InitContainers}} { + for i := range list.containers { + container := &list.containers[i] + path := append(containerPath(list.name, container.Name), fieldpath.MakePathOrDie("imagePullPolicy")...) + if container.ImagePullPolicy != "" || external.Has(path) { + continue + } + image, _, digest := strings.Cut(container.Image, "@") + tagStart := strings.LastIndexByte(image, ':') + hasTag := tagStart > strings.LastIndexByte(image, '/') + container.ImagePullPolicy = corev1.PullIfNotPresent + if hasTag && image[tagStart+1:] == "latest" || !hasTag && !digest { + container.ImagePullPolicy = corev1.PullAlways + } + } + } +} + +func removeSharedRetiredContainers(live, desired *corev1.PodSpec, operator, external *fieldpath.Set) bool { + changed := false + for _, list := range []struct { + name string + live *[]corev1.Container + desired []corev1.Container + }{{"containers", &live.Containers, desired.Containers}, {"initContainers", &live.InitContainers, desired.InitContainers}} { + *list.live = slices.DeleteFunc(*list.live, func(container corev1.Container) bool { + if slices.ContainsFunc(list.desired, func(wanted corev1.Container) bool { return wanted.Name == container.Name }) { + return false + } + path := containerPath(list.name, container.Name) + if !operator.Has(path) && !operator.Has(append(path, fieldpath.MakePathOrDie("image")...)) { + return false + } + shared := ownsFieldOrChild(external, path) + changed = changed || shared + return shared + }) + } + return changed +} + +func clearIncompatibleSources(live, intent *corev1.PodSpec) bool { + changed := false + for i := range live.Volumes { + volume := &live.Volumes[i] + for _, wanted := range intent.Volumes { + if volume.Name == wanted.Name && volumeSourceKind(volume.VolumeSource) != volumeSourceKind(wanted.VolumeSource) { + volume.VolumeSource = corev1.VolumeSource{} + changed = true + break + } + } + } + forRenderedContainers(live, intent, func(current, desired *corev1.Container) { + changed = clearIncompatibleHandlers(current, desired) || changed + for i := range current.Env { + env := ¤t.Env[i] + j := slices.IndexFunc(desired.Env, func(wanted corev1.EnvVar) bool { return wanted.Name == env.Name }) + if j < 0 { + continue + } + wanted := desired.Env[j] + // An omitted literal on a rendered name means empty, not unowned. + if (wanted.ValueFrom != nil || wanted.Value == "") && env.Value != "" { + env.Value = "" + changed = true + } + if env.ValueFrom != nil && (wanted.ValueFrom == nil || sourceKind(*env.ValueFrom) != sourceKind(*wanted.ValueFrom)) { + env.ValueFrom = nil + changed = true + } + } + }) + return changed +} + +func volumeSourceKind(source corev1.VolumeSource) int { + // Kubernetes defaults a missing volume source to EmptyDir. + if source == (corev1.VolumeSource{}) { + source.EmptyDir = &corev1.EmptyDirVolumeSource{} + } + return sourceKind(source) +} + +func clearIncompatibleHandlers(live, desired *corev1.Container) bool { + changed := false + for _, pair := range [][2]*corev1.Probe{{live.ReadinessProbe, desired.ReadinessProbe}, {live.LivenessProbe, desired.LivenessProbe}, {live.StartupProbe, desired.StartupProbe}} { + if pair[0] != nil && pair[1] != nil { + changed = clearIncompatibleHandler(&pair[0].ProbeHandler, &pair[1].ProbeHandler) || changed + } + } + if live.Lifecycle != nil && desired.Lifecycle != nil { + changed = clearIncompatibleHandler(live.Lifecycle.PostStart, desired.Lifecycle.PostStart) || changed + changed = clearIncompatibleHandler(live.Lifecycle.PreStop, desired.Lifecycle.PreStop) || changed + } + return changed +} + +func clearIncompatibleHandler[T any](live, desired *T) bool { + if live == nil || desired == nil || sourceKind(*live) == sourceKind(*desired) { + return false + } + var zero T + *live = zero + return true +} + +// Kubernetes models volume/env sources and probe/lifecycle handlers as unions. +func sourceKind(source any) int { + fields := reflect.ValueOf(source) + for i := range fields.NumField() { + if !fields.Field(i).IsZero() { + return i + } + } + return -1 +} + +func forRenderedContainers(live, desired *corev1.PodSpec, apply func(live, desired *corev1.Container)) { + for _, pair := range [][2][]corev1.Container{{live.Containers, desired.Containers}, {live.InitContainers, desired.InitContainers}} { + for i := range pair[0] { + for j := range pair[1] { + if pair[0][i].Name == pair[1][j].Name { + apply(&pair[0][i], &pair[1][j]) + break + } + } + } + } +} diff --git a/controllers/statefulset_repairs_test.go b/controllers/statefulset_repairs_test.go new file mode 100644 index 000000000..70498d9e4 --- /dev/null +++ b/controllers/statefulset_repairs_test.go @@ -0,0 +1,54 @@ +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func TestClearInheritedSecurityOverrides(t *testing.T) { + root, requestedUID, sidecarUID := int64(0), int64(1000), int64(2000) + yes, no := true, false + intent := &corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{ + RunAsUser: &requestedUID, RunAsNonRoot: &yes, + SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}, + }, + Containers: []corev1.Container{{Name: "zookeeper"}, {Name: "sidecar", SecurityContext: &corev1.SecurityContext{RunAsUser: &sidecarUID}}}, + InitContainers: []corev1.Container{{Name: "init"}}, + } + live := intent.DeepCopy() + live.Containers[0].SecurityContext = &corev1.SecurityContext{Privileged: &yes, ReadOnlyRootFilesystem: &yes} + live.Containers[1].SecurityContext.RunAsNonRoot = &no + live.InitContainers[0].SecurityContext = &corev1.SecurityContext{RunAsUser: &root} + live.Containers = append(live.Containers, corev1.Container{Name: "unmanaged", SecurityContext: &corev1.SecurityContext{RunAsUser: &sidecarUID}}) + if !clearInheritedSecurityOverrides(live, intent) { + t.Fatal("expected inherited settings to be reconciled") + } + zoo, sidecar, init := live.Containers[0].SecurityContext, live.Containers[1].SecurityContext, live.InitContainers[0].SecurityContext + if zoo.Privileged != nil || zoo.ReadOnlyRootFilesystem == nil || sidecar.RunAsNonRoot != nil || sidecar.RunAsUser == nil || *sidecar.RunAsUser != sidecarUID || init.RunAsUser != nil || live.Containers[2].SecurityContext.RunAsUser == nil { + t.Fatal("repair did not enforce inheritance while keeping explicit and unrelated settings") + } +} + +func TestClearInheritedRunAsNonRootOverrides(t *testing.T) { + for _, requested := range []bool{false, true} { + liveFlag, root := !requested, int64(0) + intent := &corev1.PodSpec{ + SecurityContext: &corev1.PodSecurityContext{RunAsNonRoot: &requested}, + Containers: []corev1.Container{{Name: "zookeeper"}}, + } + live := intent.DeepCopy() + live.Containers[0].SecurityContext = &corev1.SecurityContext{RunAsNonRoot: &liveFlag, RunAsUser: &root} + if !clearInheritedSecurityOverrides(live, intent) { + t.Fatalf("expected conflicting runAsNonRoot to be repaired for %v", requested) + } + sc := live.Containers[0].SecurityContext + if sc.RunAsNonRoot != nil || (sc.RunAsUser == nil) != requested { + t.Fatalf("unexpected overrides for requested runAsNonRoot=%v: %+v", requested, sc) + } + } +} diff --git a/controllers/zookeepercluster_controller.go b/controllers/zookeepercluster_controller.go index 139a7624a..dbb4e4974 100644 --- a/controllers/zookeepercluster_controller.go +++ b/controllers/zookeepercluster_controller.go @@ -12,6 +12,7 @@ package controllers import ( "context" "fmt" + "sort" "strconv" "time" @@ -33,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" @@ -103,6 +105,7 @@ func (r *ZookeeperClusterReconciler) Reconcile(ctx context.Context, request ctrl if err := instance.Spec.ValidateLogging(); err != nil { return reconcile.Result{}, err } + original := instance.DeepCopy() changed := instance.WithDefaults() if instance.GetTriggerRollingRestart() { ctrl.LoggerFrom(ctx).Info("Restarting zookeeper cluster") @@ -116,7 +119,7 @@ func (r *ZookeeperClusterReconciler) Reconcile(ctx context.Context, request ctrl } if changed { ctrl.LoggerFrom(ctx).Info("Setting default settings for zookeeper-cluster") - if err := r.Client.Update(ctx, instance); err != nil { + if err := r.Client.Patch(ctx, instance, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return reconcile.Result{}, err } return reconcile.Result{Requeue: true}, nil @@ -146,41 +149,6 @@ func getRollingRestartAnnotation() (string, string) { return "restartTime", time.Now().Format(time.RFC850) } -// compareResourceVersion compare resoure versions for the supplied ZookeeperCluster and StatefulSet -// resources -// Returns: -// 0 if versions are equal -// -1 if ZookeeperCluster version is less than StatefulSet version -// 1 if ZookeeperCluster version is greater than StatefulSet version -func compareResourceVersion(zk *zookeeperv1beta1.ZookeeperCluster, sts *appsv1.StatefulSet) int { - - zkResourceVersion, zkErr := strconv.Atoi(zk.ResourceVersion) - stsVersion, stsVersionFound := sts.Labels["owner-rv"] - - if !stsVersionFound { - if zkErr != nil { - log.Info("Fail to parse ZookeeperCluster version. Cannot decide zookeeper StatefulSet version") - return 0 - } - return 1 - } - stsResourceVersion, err := strconv.Atoi(stsVersion) - if err != nil { - if zkErr != nil { - log.Info("Fail to parse ZookeeperCluster version. Cannot decide zookeeper StatefulSet version") - return 0 - } - log.Info("Fail to convert StatefulSet version %s to integer; setting it to ZookeeperCluster version", stsVersion) - return 1 - } - if zkResourceVersion < stsResourceVersion { - return -1 - } else if zkResourceVersion > stsResourceVersion { - return 1 - } - return 0 -} - func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, instance *zookeeperv1beta1.ZookeeperCluster) (err error) { // we cannot upgrade if cluster is in UpgradeFailed @@ -195,8 +163,7 @@ func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, i Namespace: sts.Namespace, }, foundSts) if err == nil { - err = r.Client.Update(ctx, foundSts) - if err != nil { + if err := requireClusterOwner(instance, foundSts, "statefulset"); err != nil { return err } if foundSts.Status.Replicas == foundSts.Status.ReadyReplicas && foundSts.Status.CurrentRevision == foundSts.Status.UpdateRevision { @@ -232,9 +199,10 @@ func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, i return err } else { if !equality.Semantic.DeepEqual(foundServiceAccount.ImagePullSecrets, serviceAccount.ImagePullSecrets) { + original := foundServiceAccount.DeepCopy() foundServiceAccount.ImagePullSecrets = serviceAccount.ImagePullSecrets ctrl.LoggerFrom(ctx).Info("Updating ServiceAccount", "ServiceAccount.Namespace", serviceAccount.Namespace, "ServiceAccount.Name", serviceAccount.Name) - err = r.Client.Update(ctx, foundServiceAccount) + err = r.Client.Patch(ctx, foundServiceAccount, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})) if err != nil { return err } @@ -255,23 +223,16 @@ func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, i "StatefulSet.Namespace", sts.Namespace, "StatefulSet.Name", sts.Name) zk.EnsureLogbackProjection(instance, nil, sts) - // label the RV of the zookeeperCluster when creating the sts - sts.Labels["owner-rv"] = instance.ResourceVersion - err = r.Client.Create(ctx, sts) - if err != nil { - return err - } - return nil + return r.Client.Create(ctx, sts, client.FieldOwner(createFieldManager)) } else if err != nil { return err } else { - // check whether zookeeperCluster is updated before updating the sts - cmp := compareResourceVersion(instance, foundSts) - if cmp < 0 { - return fmt.Errorf("Staleness: cr.ResourceVersion %s is smaller than labeledRV %s", instance.ResourceVersion, foundSts.Labels["owner-rv"]) - } else if cmp > 0 { - // Zookeeper StatefulSet version inherits ZookeeperCluster resource version - foundSts.Labels["owner-rv"] = instance.ResourceVersion + if err := requireClusterOwner(instance, foundSts, "statefulset"); err != nil { + return err + } + // Read the parent after the child. Its resource version cannot fence stale parent intent. + if err := r.requireCurrentClusterSpec(ctx, instance); err != nil { + return err } zk.EnsureLogbackProjection(instance, foundSts, sts) foundSTSSize := *foundSts.Spec.Replicas @@ -293,6 +254,12 @@ func (r *ZookeeperClusterReconciler) reconcileStatefulSet(ctx context.Context, i if err != nil { return fmt.Errorf("error doing exists check for znode %s: %w", path, err) } + if err := r.requireCurrentClusterSpec(zkCtx, instance); err != nil { + return err + } + if err := r.requireCurrentStatefulSet(zkCtx, foundSts); err != nil { + return err + } data := "CLUSTER_SIZE=" + strconv.Itoa(int(newSTSSize)) ctrl.LoggerFrom(ctx).Info("Updating Cluster Size.", "New Data:", data, "Version", version) @@ -312,10 +279,7 @@ func (r *ZookeeperClusterReconciler) updateStatefulSet(ctx context.Context, inst ctrl.LoggerFrom(ctx).Info("Updating StatefulSet", "StatefulSet.Namespace", foundSts.Namespace, "StatefulSet.Name", foundSts.Name) - zk.SyncStatefulSet(foundSts, sts) - - err = r.Client.Update(ctx, foundSts) - if err != nil { + if err := r.applyStatefulSet(ctx, foundSts, sts); err != nil { return err } instance.Status.Replicas = foundSts.Status.Replicas @@ -366,30 +330,20 @@ func (r *ZookeeperClusterReconciler) upgradeStatefulSet(ctx context.Context, ins err = checkSyncTimeout(instance, zookeeperv1beta1.UpdatingZookeeperReason, foundSts.Status.UpdatedReplicas, 10*time.Minute) if err != nil { instance.Status.SetErrorConditionTrue("UpgradeFailed", err.Error()) - return r.Client.Status().Update(ctx, instance) + return r.patchClusterStatus(ctx, instance) } else { return nil } } } } - return r.Client.Status().Update(ctx, instance) + return r.patchClusterStatus(ctx, instance) } func (r *ZookeeperClusterReconciler) clearUpgradeStatus(ctx context.Context, z *zookeeperv1beta1.ZookeeperCluster) (err error) { z.Status.SetUpgradingConditionFalse() z.Status.TargetVersion = "" - // need to deep copy the status struct, otherwise it will be overwritten - // when updating the CR below - status := z.Status.DeepCopy() - - err = r.Client.Update(ctx, z) - if err != nil { - return err - } - - z.Status = *status - return nil + return r.patchClusterStatus(ctx, z) } func checkSyncTimeout(z *zookeeperv1beta1.ZookeeperCluster, reason string, updatedReplicas int32, t time.Duration) error { @@ -423,19 +377,14 @@ func (r *ZookeeperClusterReconciler) reconcileClientService(ctx context.Context, ctrl.LoggerFrom(ctx).Info("Creating new client service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err = r.Client.Create(ctx, svc) - if err != nil { - return err - } - return nil + return r.Client.Create(ctx, svc, client.FieldOwner(createFieldManager)) } else if err != nil { return err } else { ctrl.LoggerFrom(ctx).Info("Updating existing client service", "Service.Namespace", foundSvc.Namespace, "Service.Name", foundSvc.Name) - zk.SyncService(foundSvc, svc) - err = r.Client.Update(ctx, foundSvc) + err = r.applyService(ctx, instance, foundSvc, svc) if err != nil { return err } @@ -470,19 +419,14 @@ func (r *ZookeeperClusterReconciler) reconcileHeadlessService(ctx context.Contex ctrl.LoggerFrom(ctx).Info("Creating new headless service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err = r.Client.Create(ctx, svc) - if err != nil { - return err - } - return nil + return r.Client.Create(ctx, svc, client.FieldOwner(createFieldManager)) } else if err != nil { return err } else { ctrl.LoggerFrom(ctx).Info("Updating existing headless service", "Service.Namespace", foundSvc.Namespace, "Service.Name", foundSvc.Name) - zk.SyncService(foundSvc, svc) - err = r.Client.Update(ctx, foundSvc) + err = r.applyService(ctx, instance, foundSvc, svc) if err != nil { return err } @@ -504,19 +448,14 @@ func (r *ZookeeperClusterReconciler) reconcileAdminServerService(ctx context.Con ctrl.LoggerFrom(ctx).Info("Creating admin server service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err = r.Client.Create(ctx, svc) - if err != nil { - return err - } - return nil + return r.Client.Create(ctx, svc, client.FieldOwner(createFieldManager)) } else if err != nil { return err } else { ctrl.LoggerFrom(ctx).Info("Updating existing admin server service", "Service.Namespace", foundSvc.Namespace, "Service.Name", foundSvc.Name) - zk.SyncService(foundSvc, svc) - err = r.Client.Update(ctx, foundSvc) + err = r.applyService(ctx, instance, foundSvc, svc) if err != nil { return err } @@ -553,7 +492,7 @@ func (r *ZookeeperClusterReconciler) reconcilePodDisruptionBudget(ctx context.Co if equality.Semantic.DeepEqual(original.Spec, foundPdb.Spec) { return nil } - return r.Client.Patch(ctx, foundPdb, client.MergeFrom(original)) + return r.Client.Patch(ctx, foundPdb, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})) } func (r *ZookeeperClusterReconciler) reconcileConfigMap(ctx context.Context, instance *zookeeperv1beta1.ZookeeperCluster) (err error) { @@ -585,8 +524,9 @@ func (r *ZookeeperClusterReconciler) reconcileConfigMap(ctx context.Context, ins ctrl.LoggerFrom(ctx).Info("Updating existing config-map", "ConfigMap.Namespace", foundCm.Namespace, "ConfigMap.Name", foundCm.Name) + original := foundCm.DeepCopy() zk.SyncConfigMap(foundCm, cm) - err = r.Client.Update(ctx, foundCm) + err = r.Client.Patch(ctx, foundCm, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})) if err != nil { return err } @@ -626,6 +566,8 @@ func (r *ZookeeperClusterReconciler) reconcileClusterStatus(ctx context.Context, unreadyMembers = append(unreadyMembers, p.Name) } } + sort.Strings(readyMembers) + sort.Strings(unreadyMembers) instance.Status.Members.Ready = readyMembers instance.Status.Members.Unready = unreadyMembers @@ -660,7 +602,66 @@ func (r *ZookeeperClusterReconciler) reconcileClusterStatus(ctx context.Context, if instance.Status.CurrentVersion == "" && instance.Status.IsClusterInReadyState() { instance.Status.CurrentVersion = instance.Spec.Image.Tag } - return r.Client.Status().Update(ctx, instance) + return r.patchClusterStatus(ctx, instance) +} + +func (r *ZookeeperClusterReconciler) patchClusterStatus(ctx context.Context, instance *zookeeperv1beta1.ZookeeperCluster) error { + current := &zookeeperv1beta1.ZookeeperCluster{} + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := reader.Get(ctx, client.ObjectKeyFromObject(instance), current); err != nil { + return err + } + if current.ResourceVersion != instance.ResourceVersion || current.UID != instance.UID { + return staleCluster(instance) + } + if equality.Semantic.DeepEqual(current.Status, instance.Status) { + return nil + } + desired := current.DeepCopy() + desired.Status = instance.Status + if err := r.Client.Status().Patch(ctx, desired, client.MergeFromWithOptions(current, client.MergeFromWithOptimisticLock{})); err != nil { + return err + } + instance.ResourceVersion = desired.ResourceVersion + instance.Status = desired.Status + return nil +} + +func staleCluster(instance *zookeeperv1beta1.ZookeeperCluster) error { + return errors.NewConflict(schema.GroupResource{Group: zookeeperv1beta1.GroupVersion.Group, Resource: "zookeeperclusters"}, instance.Name, fmt.Errorf("reconcile snapshot is stale")) +} + +func (r *ZookeeperClusterReconciler) requireCurrentClusterSpec(ctx context.Context, instance *zookeeperv1beta1.ZookeeperCluster) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + current := &zookeeperv1beta1.ZookeeperCluster{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(instance), current); err != nil { + return err + } + if current.UID != instance.UID || current.Generation != instance.Generation || !current.DeletionTimestamp.IsZero() { + return staleCluster(instance) + } + return nil +} + +func (r *ZookeeperClusterReconciler) requireCurrentStatefulSet(ctx context.Context, observed *appsv1.StatefulSet) error { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + current := &appsv1.StatefulSet{} + if err := reader.Get(ctx, client.ObjectKeyFromObject(observed), current); err != nil { + return err + } + if current.UID != observed.UID || current.ResourceVersion != observed.ResourceVersion || !current.DeletionTimestamp.IsZero() { + return errors.NewConflict(appsv1.Resource("statefulsets"), observed.Name, fmt.Errorf("resize snapshot is stale")) + } + return nil } // YAMLExporterReconciler returns a fake Reconciler which is being used for generating YAML files @@ -755,8 +756,9 @@ func (r *ZookeeperClusterReconciler) reconcileFinalizers(ctx context.Context, in return nil } if !utils.ContainsString(instance.ObjectMeta.Finalizers, utils.ZkFinalizer) && !config.DisableFinalizer { + original := instance.DeepCopy() instance.ObjectMeta.Finalizers = append(instance.ObjectMeta.Finalizers, utils.ZkFinalizer) - if err = r.Client.Update(ctx, instance); err != nil { + if err = r.Client.Patch(ctx, instance, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return err } } @@ -768,8 +770,9 @@ func (r *ZookeeperClusterReconciler) reconcileFinalizers(ctx context.Context, in return err } } + original := instance.DeepCopy() instance.Finalizers = utils.RemoveString(instance.Finalizers, utils.ZkFinalizer) - if err = r.Client.Update(ctx, instance); err != nil { + if err = r.Client.Patch(ctx, instance, client.MergeFromWithOptions(original, client.MergeFromWithOptimisticLock{})); err != nil { return err } } diff --git a/controllers/zookeepercluster_controller_test.go b/controllers/zookeepercluster_controller_test.go index 871622504..b8bc6507f 100644 --- a/controllers/zookeepercluster_controller_test.go +++ b/controllers/zookeepercluster_controller_test.go @@ -35,6 +35,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/event" "sigs.k8s.io/controller-runtime/pkg/reconcile" @@ -62,7 +63,7 @@ func TestReconcileEventPredicate(t *testing.T) { stsSpec.Generation++ metadata := sts.DeepCopy() metadata.ResourceVersion = "2" - metadata.Labels = map[string]string{"owner-rv": "2"} + metadata.Labels = map[string]string{"external": "value"} cluster := &v1beta1.ZookeeperCluster{ObjectMeta: metav1.ObjectMeta{Generation: 1}} clusterSpec := cluster.DeepCopy() clusterSpec.Generation++ @@ -367,12 +368,12 @@ func TestFinalizerDeletionSkipsNormalReconciliation(t *testing.T) { Create: func(context.Context, client.WithWatch, client.Object, ...client.CreateOption) error { return errors.New("terminating clusters must not create resources") }, - Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { cluster, ok := obj.(*v1beta1.ZookeeperCluster) if !ok || !reflect.DeepEqual(cluster.Spec, originalSpec) || utils.ContainsString(cluster.Finalizers, utils.ZkFinalizer) { return errors.New("termination must only remove the operator finalizer") } - return c.Update(ctx, obj, opts...) + return c.Patch(ctx, obj, patch, opts...) }, }) result, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: client.ObjectKeyFromObject(z)}) @@ -576,11 +577,15 @@ func TestCanceledReconcileStopsClientWork(t *testing.T) { } other := z.DeepCopy() other.Name = "other" + other.UID = "other-metadata-test" other.ResourceVersion = "" if err := r.Client.Create(context.Background(), other); err != nil { t.Fatal(err) } otherSts := zk.MakeStatefulSet(other) + if err := controllerutil.SetControllerReference(other, otherSts, r.Scheme); err != nil { + t.Fatal(err) + } replicas := int32(3) otherSts.Spec.Replicas = &replicas if err := r.Client.Create(context.Background(), otherSts); err != nil { @@ -872,7 +877,7 @@ func TestScaleMetadataStatefulSetConflictRetries(t *testing.T) { z.Spec.Replicas = 5 cl := r.Client r.Client = interceptor.NewClient(cl.(client.WithWatch), interceptor.Funcs{ - Update: func(context.Context, client.WithWatch, client.Object, ...client.UpdateOption) error { + Patch: func(context.Context, client.WithWatch, client.Object, client.Patch, ...client.PatchOption) error { return apierrors.NewConflict(schema.GroupResource{Group: "apps", Resource: "statefulsets"}, sts.Name, errors.New("stale writer")) }, }) @@ -904,7 +909,10 @@ func newScaleMetadataTestReconciler(t *testing.T, replicas int32) (*ZookeeperClu z.WithDefaults() z.Spec.Replicas = replicas sts := zk.MakeStatefulSet(z) - cl := fake.NewClientBuilder().WithScheme(s).WithObjects(z, sts).WithStatusSubresource(z, sts).Build() + if err := controllerutil.SetControllerReference(z, sts, s); err != nil { + t.Fatal(err) + } + cl := fake.NewClientBuilder().WithScheme(s).WithObjects(z, sts).WithStatusSubresource(z, sts).WithReturnManagedFields().Build() return &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return &MockZookeeperClient{} }, Log: log}, z, sts } @@ -953,6 +961,11 @@ var _ = Describe("ZookeeperCluster Controller", func() { mockZkClient = new(MockZookeeperClient) r *ZookeeperClusterReconciler ) + makeOwnedStatefulSet := func(z *v1beta1.ZookeeperCluster) *appsv1.StatefulSet { + sts := zk.MakeStatefulSet(z) + Ω(controllerutil.SetControllerReference(z, sts, s)).To(Succeed()) + return sts + } Context("Reconcile", func() { var ( @@ -972,6 +985,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { ObjectMeta: metav1.ObjectMeta{ Name: Name, Namespace: Namespace, + UID: "test-zookeeper-uid", }, } s.AddKnownTypes(v1beta1.GroupVersion, z) @@ -985,7 +999,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { z.Status.Init() z.Spec.Pod.Annotations = map[string]string{"custom": "keep"} cl = fake.NewClientBuilder().WithScheme(s). - WithRuntimeObjects(z, zk.MakeConfigMap(z), zk.MakeStatefulSet(z)). + WithRuntimeObjects(z, zk.MakeConfigMap(z), makeOwnedStatefulSet(z)). WithStatusSubresource(z).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} }) @@ -1055,11 +1069,11 @@ var _ = Describe("ZookeeperCluster Controller", func() { z.Spec.Logging.Level = "WARN" Ω(cl.Update(context.TODO(), z)).To(Succeed()) r.Client = interceptor.NewClient(cl, interceptor.Funcs{ - Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + Patch: func(ctx context.Context, c client.WithWatch, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { if _, ok := obj.(*corev1.ConfigMap); ok { return errors.New("config update failed") } - return c.Update(ctx, obj, opts...) + return c.Patch(ctx, obj, patch, opts...) }, }) _, err := r.Reconcile(context.TODO(), req) @@ -1252,7 +1266,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { z.Spec.Pod.ServiceAccountName = "zookeeper" z.Status.Init() next := z.DeepCopy() - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) next.Spec.Replicas = 6 cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} @@ -1281,7 +1295,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { z.WithDefaults() z.Status.Init() next := z.DeepCopy() - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) @@ -1354,7 +1368,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next.Spec.Image.Tag = "0.2.7" next.Status.CurrentVersion = "0.2.6" next.Status.SetPodsReadyConditionTrue() - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() st = &appsv1.StatefulSet{} err = cl.Get(context.TODO(), req.NamespacedName, st) @@ -1414,7 +1428,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next.Status.CurrentVersion = "0.2.6" next.Status.TargetVersion = "0.2.7" next.Status.SetUpgradingConditionTrue(" ", " ") - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() st = &appsv1.StatefulSet{} err = cl.Get(context.TODO(), req.NamespacedName, st) @@ -1461,7 +1475,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next := z.DeepCopy() next.Status.SetUpgradingConditionTrue(" ", "1") next.Status.TargetVersion = "0.2.7" - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() st = &appsv1.StatefulSet{} err = cl.Get(context.TODO(), req.NamespacedName, st) @@ -1507,8 +1521,8 @@ var _ = Describe("ZookeeperCluster Controller", func() { next.Status.ReadyReplicas = 3 next.Spec.Replicas = 3 next.Spec.Image.Tag = "0.2.7" - st := zk.MakeStatefulSet(z) - cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).Build() + st := makeOwnedStatefulSet(z) + cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() st = &appsv1.StatefulSet{} err = cl.Get(context.TODO(), req.NamespacedName, st) // changing the Revision value to simulate the upgrade scenario @@ -1551,7 +1565,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next.Status.CurrentVersion = "0.2.6" next.Status.TargetVersion = "" next.Status.IsClusterInUpgradingState() - st := zk.MakeStatefulSet(z) + st := makeOwnedStatefulSet(z) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, st).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) @@ -1670,6 +1684,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { next := z.DeepCopy() next.Spec.Ports[0].ContainerPort = 2182 svc := zk.MakeClientService(z) + Ω(controllerutil.SetControllerReference(next, svc, s)).To(Succeed()) cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) @@ -1730,64 +1745,18 @@ var _ = Describe("ZookeeperCluster Controller", func() { }) }) - Context("Checking resource version", func() { - var ( - sts *appsv1.StatefulSet - ) - - BeforeEach(func() { - z.WithDefaults() - z.ResourceVersion = "100" - sts = &appsv1.StatefulSet{} - sts.Labels = make(map[string]string) - }) - - It("should return 1 as 100 > 99", func() { - sts.Labels["owner-rv"] = "99" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(1)) - }) - - It("should return 0 as 100 == 100", func() { - sts.Labels["owner-rv"] = "100" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(0)) - }) - - It("should return -1 as 100 < 101", func() { - sts.Labels["owner-rv"] = "101" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(-1)) - }) - - It("should return 1 as 100 > invalid numeric version 'xoxo'", func() { - sts.Labels["owner-rv"] = "xoxo" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(1)) - }) - - It("should return 0 as invalid zk version z0z0 == invalid numeric version 'x0x0'", func() { - sts.Labels["owner-rv"] = "x0x0" - z.ResourceVersion = "z0z0" - updated := compareResourceVersion(z, sts) - Ω(updated).To(Equal(0)) - }) - }) - Context("trigger rolling restart", func() { var ( cl client.Client err error foundZk = &v1beta1.ZookeeperCluster{} next *v1beta1.ZookeeperCluster - svc *corev1.Service ) BeforeEach(func() { z.WithDefaults() next = z.DeepCopy() next.Spec.TriggerRollingRestart = true - svc = zk.MakeClientService(z) - cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build() + cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) err = cl.Get(context.TODO(), req.NamespacedName, foundZk) @@ -1806,8 +1775,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { Ω(restartTimeExists).To(Equal(true)) next.Spec.TriggerRollingRestart = false - svc = zk.MakeClientService(z) - cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build() + cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) @@ -1827,7 +1795,6 @@ var _ = Describe("ZookeeperCluster Controller", func() { // update the crd instance next.Spec.TriggerRollingRestart = false - svc = zk.MakeClientService(z) r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) err = cl.Get(context.TODO(), req.NamespacedName, foundZk) @@ -1844,8 +1811,7 @@ var _ = Describe("ZookeeperCluster Controller", func() { // update the crd instance to trigger rolling restart next.Spec.TriggerRollingRestart = true - svc = zk.MakeClientService(z) - cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next, svc).WithStatusSubresource(next).Build() + cl = fake.NewClientBuilder().WithScheme(scheme.Scheme).WithRuntimeObjects(next).WithStatusSubresource(next).Build() r = &ZookeeperClusterReconciler{Client: cl, Scheme: s, NewZkClient: func() zk.ZookeeperClient { return mockZkClient }} res, err = r.Reconcile(context.TODO(), req) err = cl.Get(context.TODO(), req.NamespacedName, foundZk) diff --git a/doc/operator-upgrade.md b/doc/operator-upgrade.md index b4a640f68..c1f3a42de 100644 --- a/doc/operator-upgrade.md +++ b/doc/operator-upgrade.md @@ -1,4 +1,28 @@ # Upgrading Operator + +## Before upgrading to a release that applies StatefulSets and Services + +- Check the controller owner reference on each existing ZooKeeper StatefulSet and + client, headless, and admin Service. Its UID must match the current + `ZookeeperCluster` UID. The operator does not automatically adopt restored, + manually created, or orphaned resources. If references do not match, pause the + upgrade. Identify the resources and their storage before attempting recovery; + do not delete the StatefulSet or PVCs to clear the error. Explicit orphan + recovery is tracked in [#122](https://github.com/adobe/zookeeper-operator/issues/122). +- Check any custom selectors on the existing Services. The operator restores + their selector to `app: `, even when the owner UID matches. + If clients need custom routing, create a separately managed Service with a + different name and switch the clients to it before the upgrade. +- If you manage your own operator RBAC (for example `rbac.create: false`), + compare it with the release's bundled role. In addition to existing permissions, + grant `patch` on `statefulsets` (`apps`), `services`, `configmaps`, and + `serviceaccounts` (core API group), `poddisruptionbudgets` (`policy`), and + `zookeeperclusters` and `zookeeperclusters/status` (`zookeeper.pravega.io`). + The bundled roles already include these permissions. A Ready operator Pod + does not prove that the operator can write these resources. + +## Upgrade the operator + zookeeperoperator can be upgraded to a version **[VERSION]** via helm using the following command ``` diff --git a/go.mod b/go.mod index 774d8fb4f..d57bc7f40 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( k8s.io/apimachinery v0.37.2 k8s.io/client-go v0.37.2 sigs.k8s.io/controller-runtime v0.25.2 + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 ) require ( @@ -81,6 +82,5 @@ require ( k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/pkg/zk/generators.go b/pkg/zk/generators.go index e9cf34c53..a148e00ae 100644 --- a/pkg/zk/generators.go +++ b/pkg/zk/generators.go @@ -14,6 +14,7 @@ import ( "crypto/sha256" "fmt" "reflect" + "sort" "strconv" "strings" @@ -333,8 +334,13 @@ func makeZkConfigString(z *v1beta1.ZookeeperCluster) string { ports := z.ZookeeperPorts() var zkConfig = "" - for key, value := range z.Spec.Conf.AdditionalConfig { - zkConfig = zkConfig + fmt.Sprintf("%s=%s\n", key, value) + keys := make([]string, 0, len(z.Spec.Conf.AdditionalConfig)) + for key := range z.Spec.Conf.AdditionalConfig { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + zkConfig = zkConfig + fmt.Sprintf("%s=%s\n", key, z.Spec.Conf.AdditionalConfig[key]) } return zkConfig + "4lw.commands.whitelist=cons, envi, conf, crst, srvr, stat, mntr, ruok\n" + "dataDir=/data\n" + @@ -469,6 +475,7 @@ func makeService(name string, ports []v1.ServicePort, clusterIP bool, external b Annotations: annotationMap, }, Spec: v1.ServiceSpec{ + Type: v1.ServiceTypeClusterIP, Ports: ports, Selector: map[string]string{"app": z.GetName()}, }, diff --git a/pkg/zk/synchronizers.go b/pkg/zk/synchronizers.go index 77a37f89f..11cbf3a9a 100644 --- a/pkg/zk/synchronizers.go +++ b/pkg/zk/synchronizers.go @@ -11,24 +11,9 @@ package zk import ( - appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" ) -// SyncStatefulSet synchronizes any updates to the stateful-set -func SyncStatefulSet(curr *appsv1.StatefulSet, next *appsv1.StatefulSet) { - curr.Spec.Replicas = next.Spec.Replicas - curr.Spec.Template = next.Spec.Template - curr.Spec.UpdateStrategy = next.Spec.UpdateStrategy -} - -// SyncService synchronizes a service with an updated spec and validates it -func SyncService(curr *v1.Service, next *v1.Service) { - curr.Spec.Ports = next.Spec.Ports - curr.Spec.Type = next.Spec.Type - curr.SetAnnotations(next.GetAnnotations()) -} - // SyncConfigMap synchronizes a configmap with an updated spec and validates it func SyncConfigMap(curr *v1.ConfigMap, next *v1.ConfigMap) { curr.Data = next.Data diff --git a/pkg/zk/synchronizers_test.go b/pkg/zk/synchronizers_test.go index 9e909b8f6..d00517920 100644 --- a/pkg/zk/synchronizers_test.go +++ b/pkg/zk/synchronizers_test.go @@ -13,9 +13,6 @@ package zk_test import ( "github.com/pravega/zookeeper-operator/api/v1beta1" "github.com/pravega/zookeeper-operator/pkg/zk" - appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" . "github.com/onsi/ginkgo" @@ -24,62 +21,6 @@ import ( var _ = Describe("Synchronizers", func() { - Context("with a valid update specs", func() { - var sts1 *appsv1.StatefulSet - - BeforeEach(func() { - z := &v1beta1.ZookeeperCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "example", - Namespace: "default", - }, - } - z.WithDefaults() - z.Spec.Pod.Resources.Limits = v1.ResourceList{ - v1.ResourceStorage: resource.MustParse("20Gi"), - } - sts1 = zk.MakeStatefulSet(z) - sts2 := zk.MakeStatefulSet(z) - reps := int32(4) - sts2.Spec.Replicas = &reps - sts2.Spec.Template.Spec.Containers[0].Image = "repo/newimage:latest" - zk.SyncStatefulSet(sts1, sts2) - }) - - It("should have the updated fields", func() { - Ω(*sts1.Spec.Replicas).To(BeEquivalentTo(4)) - Ω(sts1.Spec.Template.Spec.Containers[0].Image). - To(Equal("repo/newimage:latest")) - }) - }) - - Context("with a valid update of Service port", func() { - var port int32 - var value string - - BeforeEach(func() { - z := &v1beta1.ZookeeperCluster{ - ObjectMeta: metav1.ObjectMeta{ - Name: "example", - Namespace: "default", - }, - } - z.WithDefaults() - svc1 := zk.MakeClientService(z) - svc2 := svc1.DeepCopy() - svc2.Spec.Ports[0].Port = int32(4000) - svc2.Spec.Type = "temp" - zk.SyncService(svc1, svc2) - port = svc1.Spec.Ports[0].Port - value = string(svc1.Spec.Type) - }) - - It("should have the updated fields for service", func() { - Ω(port).To(BeEquivalentTo(4000)) - Ω(value).To(BeEquivalentTo("temp")) - }) - }) - Context("with a valid update of config map", func() { var value string BeforeEach(func() { diff --git a/pkg/zk/zookeeper_client_integration_test.go b/pkg/zk/zookeeper_client_integration_test.go index a90e16786..d16388a36 100644 --- a/pkg/zk/zookeeper_client_integration_test.go +++ b/pkg/zk/zookeeper_client_integration_test.go @@ -33,6 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) func TestMetadataCreateWaitsForServingZooKeeper(t *testing.T) { @@ -228,17 +229,20 @@ func TestMetadataCreateRetryPreservesExistingData(t *testing.T) { cluster.WithDefaults() cluster.Spec.Replicas = 1 sts := operatorzk.MakeStatefulSet(cluster) + if err := controllerutil.SetControllerReference(cluster, sts, s); err != nil { + t.Fatal(err) + } sts.Status.Replicas = 1 sts.Status.ReadyReplicas = 1 cl := fake.NewClientBuilder().WithScheme(s).WithObjects(cluster, sts).WithStatusSubresource(cluster, sts).Build() conflict := true faultClient := interceptor.NewClient(cl, interceptor.Funcs{ - SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + SubResourcePatch: func(ctx context.Context, c client.Client, subresource string, obj client.Object, patch client.Patch, opts ...client.SubResourcePatchOption) error { if subresource == "status" && conflict { conflict = false return apierrors.NewConflict(schema.GroupResource{Group: v1beta1.GroupVersion.Group, Resource: "zookeeperclusters"}, obj.GetName(), errors.New("stale writer")) } - return c.SubResource(subresource).Update(ctx, obj, opts...) + return c.SubResource(subresource).Patch(ctx, obj, patch, opts...) }, }) r := &controllers.ZookeeperClusterReconciler{ From 69a735fec3c80835ed299cd46c0c103e4dde32d9 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Fri, 20 Nov 2026 21:22:30 +0100 Subject: [PATCH 2/2] fix(controller): handle optional legacy make run-local upgrades Migrate the generic main field manager only when it owns the legacy StatefulSet marker. Test logging and scheduling migration and verify that unrelated main-owned fields remain untouched. Document a targeted cleanup for verified legacy Service DNS annotations without claiming arbitrary Service fields from main. --- controllers/compatibility_envtest_test.go | 46 ++++++++++++++++++ .../legacy_local_runner_envtest_test.go | 47 +++++++++++++++++++ controllers/server_side_apply.go | 19 +++++++- controllers/statefulset_compatibility.go | 8 ++-- controllers/statefulset_repairs.go | 2 +- doc/operator-upgrade.md | 19 ++++++++ 6 files changed, 134 insertions(+), 7 deletions(-) create mode 100644 controllers/legacy_local_runner_envtest_test.go diff --git a/controllers/compatibility_envtest_test.go b/controllers/compatibility_envtest_test.go index ea4adb324..c74939777 100644 --- a/controllers/compatibility_envtest_test.go +++ b/controllers/compatibility_envtest_test.go @@ -8,9 +8,11 @@ package controllers import ( "context" "encoding/json" + "maps" "slices" "testing" + api "github.com/pravega/zookeeper-operator/api/v1beta1" "github.com/pravega/zookeeper-operator/pkg/zk" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -688,3 +690,47 @@ func TestEnvtestSeccompProfileReplacesConflictingLegacyAnnotation(t *testing.T) } assertEnvtestSettled(t, ctx, r, cl, z, 3) } + +func TestEnvtestMigratesLegacyLocalRunner(t *testing.T) { + for _, initialLogging := range []string{"INFO", "WARN"} { + t.Run(initialLogging, func(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + z.Spec.Logging = api.LoggingConfig{Level: initialLogging} + z.Spec.Pod.NodeSelector = map[string]string{"pool": "old"} + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + sts := zk.MakeStatefulSet(z) + sts.Labels = maps.Clone(sts.Labels) + sts.Labels["owner-rv"] = z.ResourceVersion + if err := controllerutil.SetControllerReference(z, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, sts, client.FieldOwner("main")); err != nil { + t.Fatal(err) + } + z.Spec.Pod.NodeSelector = nil + if initialLogging == "INFO" { + z.Spec.Logging.Level = "WARN" + } else { + z.Spec.Logging.Level = "INFO" + } + if err := cl.Update(ctx, z); err != nil { + t.Fatal(err) + } + if err := convergenceReconciler(cl).reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + template := sts.Spec.Template + if len(template.Spec.NodeSelector) != 0 || template.Spec.Volumes[0].Projected == nil { + t.Fatal("local runner did not converge scheduling and logging fields") + } + if _, present := template.Annotations["zookeeper.pravega.io/logback-hash"]; present != (z.Spec.Logging.Level == "WARN") { + t.Fatal("logging hash did not follow the requested configuration") + } + }) + } +} diff --git a/controllers/legacy_local_runner_envtest_test.go b/controllers/legacy_local_runner_envtest_test.go new file mode 100644 index 000000000..5ef36db34 --- /dev/null +++ b/controllers/legacy_local_runner_envtest_test.go @@ -0,0 +1,47 @@ +//go:build envtest + +// Copyright 2026 Adobe. All rights reserved. +// Licensed under the Apache License, Version 2.0. + +package controllers + +import ( + "maps" + "slices" + "testing" + + "github.com/pravega/zookeeper-operator/pkg/zk" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/csaupgrade" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" +) + +func TestEnvtestPreservesUnrelatedMainManager(t *testing.T) { + ctx, cl, z := newEnvtestCluster(t) + sts := zk.MakeStatefulSet(z) + sts.Labels = maps.Clone(sts.Labels) + sts.Labels["owner-rv"] = z.ResourceVersion + if err := controllerutil.SetControllerReference(z, sts, cl.Scheme()); err != nil { + t.Fatal(err) + } + if err := cl.Create(ctx, sts, client.FieldOwner("zookeeper-operator")); err != nil { + t.Fatal(err) + } + const key = "example.com/unrelated" + sts.Annotations = map[string]string{key: "keep"} + if err := cl.Update(ctx, sts, client.FieldOwner("main")); err != nil { + t.Fatal(err) + } + if err := convergenceReconciler(cl).reconcileStatefulSet(ctx, z); err != nil { + t.Fatal(err) + } + if err := cl.Get(ctx, client.ObjectKeyFromObject(sts), sts); err != nil { + t.Fatal(err) + } + owners := csaupgrade.FindFieldsOwners(sts.ManagedFields, metav1.ManagedFieldsOperationUpdate, fieldpath.NewSet(fieldpath.MakePathOrDie("metadata", "annotations", key))) + if sts.Annotations[key] != "keep" || !slices.ContainsFunc(owners, func(owner metav1.ManagedFieldsEntry) bool { return owner.Manager == "main" }) { + t.Fatal("an unrelated main manager must retain its annotation and ownership") + } +} diff --git a/controllers/server_side_apply.go b/controllers/server_side_apply.go index 9916bf44b..62ac25334 100644 --- a/controllers/server_side_apply.go +++ b/controllers/server_side_apply.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/csaupgrade" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/structured-merge-diff/v6/fieldpath" ) // applyFieldManager owns the StatefulSet and Service fields that the operator renders. @@ -29,6 +30,20 @@ const createFieldManager = "zookeeper-operator-create" // can remove fields the operator no longer renders. var legacyFieldManagers = sets.New("zookeeper-operator", createFieldManager) +func legacyManagersFor(current client.Object) sets.Set[string] { + // make run-local wrote as "main". Only migrate that generic manager when + // it owns the legacy operator's StatefulSet bookkeeping label. + if _, ok := current.(*appsv1.StatefulSet); ok && current.GetLabels()["owner-rv"] != "" { + marker := fieldpath.NewSet(fieldpath.MakePathOrDie("metadata", "labels", "owner-rv")) + for _, owner := range csaupgrade.FindFieldsOwners(current.GetManagedFields(), metav1.ManagedFieldsOperationUpdate, marker) { + if owner.Manager == "main" && owner.Subresource == "" { + return legacyFieldManagers.Union(sets.New("main")) + } + } + } + return legacyFieldManagers +} + // applyStatefulSet updates an existing StatefulSet without changing its immutable fields. func (r *ZookeeperClusterReconciler) applyStatefulSet(ctx context.Context, current, desired *appsv1.StatefulSet) error { intent := desired.DeepCopy() @@ -92,7 +107,7 @@ func (r *ZookeeperClusterReconciler) applyService(ctx context.Context, cluster * } // Publish the type, ports, and annotations together or not at all. next := &corev1.Service{} - if err := r.mergeWorkloadLocally(prepared, intent, next); err != nil { + if err := r.mergeWorkloadLocally(current, prepared, intent, next); err != nil { return err } next.Status = current.Status @@ -117,7 +132,7 @@ func (r *ZookeeperClusterReconciler) apply(ctx context.Context, current, intent if current.GetResourceVersion() == "" { return fmt.Errorf("cannot apply %T %s without a resource version", current, client.ObjectKeyFromObject(current)) } - migration, err := csaupgrade.UpgradeManagedFieldsPatch(current, legacyFieldManagers, applyFieldManager) + migration, err := csaupgrade.UpgradeManagedFieldsPatch(current, legacyManagersFor(current), applyFieldManager) if err != nil { return err } diff --git a/controllers/statefulset_compatibility.go b/controllers/statefulset_compatibility.go index 170103284..7542fbb28 100644 --- a/controllers/statefulset_compatibility.go +++ b/controllers/statefulset_compatibility.go @@ -79,13 +79,13 @@ func uniqueBy[T any, K comparable](values []T, key func(T) K) []T { // then restore the exact ordered lists and let the API server validate one guarded patch. func (r *ZookeeperClusterReconciler) applyCompatibleStatefulSet(ctx context.Context, current, prepared, intent, desired *appsv1.StatefulSet, retiringVolumes []corev1.Volume) error { next := &appsv1.StatefulSet{} - if err := r.mergeWorkloadLocally(prepared, intent, next); err != nil { + if err := r.mergeWorkloadLocally(current, prepared, intent, next); err != nil { return err } if resolveRetiringVolumes(prepared, intent, next, retiringVolumes) { // Keep complete sources while referenced; don't persist orphaned fragments. next = &appsv1.StatefulSet{} - if err := r.mergeWorkloadLocally(prepared, intent, next); err != nil { + if err := r.mergeWorkloadLocally(current, prepared, intent, next); err != nil { return err } } @@ -99,10 +99,10 @@ func (r *ZookeeperClusterReconciler) applyCompatibleStatefulSet(ctx context.Cont return nil } -func (r *ZookeeperClusterReconciler) mergeWorkloadLocally(prepared, intent, next client.Object) error { +func (r *ZookeeperClusterReconciler) mergeWorkloadLocally(current, prepared, intent, next client.Object) error { gvk := intent.GetObjectKind().GroupVersionKind() prepared.GetObjectKind().SetGroupVersionKind(gvk) - if err := csaupgrade.UpgradeManagedFields(prepared, legacyFieldManagers, applyFieldManager); err != nil { + if err := csaupgrade.UpgradeManagedFields(prepared, legacyManagersFor(current), applyFieldManager); err != nil { return err } manager, err := managedfields.NewDefaultFieldManager(applyconfigs.NewTypeConverter(r.Scheme), r.Scheme, r.Scheme, r.Scheme, gvk, gvk.GroupVersion(), "", nil) diff --git a/controllers/statefulset_repairs.go b/controllers/statefulset_repairs.go index f69954dcc..489ab2444 100644 --- a/controllers/statefulset_repairs.go +++ b/controllers/statefulset_repairs.go @@ -113,7 +113,7 @@ func matchesSeccompProfile(annotation string, profile *corev1.SeccompProfile) bo func statefulSetFields(current *appsv1.StatefulSet) (operator, external *fieldpath.Set) { operator, external = fieldpath.NewSet(), fieldpath.NewSet() - legacy := legacyFieldManagers + legacy := legacyManagersFor(current) for _, entry := range current.ManagedFields { if entry.Subresource != "" || entry.FieldsV1 == nil || entry.APIVersion != "apps/v1" { continue diff --git a/doc/operator-upgrade.md b/doc/operator-upgrade.md index c1f3a42de..ea553faee 100644 --- a/doc/operator-upgrade.md +++ b/doc/operator-upgrade.md @@ -21,6 +21,25 @@ The bundled roles already include these permissions. A Ready operator Pod does not prove that the operator can write these resources. +### Headless DNS on clusters created with `make run-local` + +Older local binaries recorded Service writes under the generic `main` field +manager. The upgrade does not transfer Service fields from `main`, because that +name can also represent an unrelated writer. If you clear `spec.domainName`, a +legacy `external-dns.alpha.kubernetes.io/hostname` annotation may remain on the +headless Service. + +If the Service belongs to the current cluster UID, neither `spec.domainName` nor +`spec.headlessService.annotations` requests that hostname, and your old cluster +configuration confirms that the operator generated the live annotation, remove +just that key from the headless Service. Do not remove it if another controller +or owner manages it intentionally. + +```sh +kubectl -n annotate service -headless \ + external-dns.alpha.kubernetes.io/hostname- +``` + ## Upgrade the operator zookeeperoperator can be upgraded to a version **[VERSION]** via helm using the following command