diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03c6d451f..6468a15c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,27 @@ jobs: helm lint charts/kafka-operator --strict --kube-version "$version" helm template kafka-operator charts/kafka-operator --kube-version "$version" > /dev/null done + rendered=$(helm template kafka-operator charts/kafka-operator) + if grep -q '^kind: PodMonitor$' <<< "$rendered"; then exit 1; fi + grep -F 'kafka.banzaicloud.io/alert-leader: "true"' <<< "$rendered" + grep -E 'fieldPath: metadata\.name$' <<< "$rendered" + grep -E 'fieldPath: metadata\.uid$' <<< "$rendered" + for proxy in true false; do + for port in 8080 9090; do + args=() + if [ "$port" != 8080 ]; then args+=(--set metricEndpoint.port="$port"); fi + rendered=$(helm template kafka-operator charts/kafka-operator \ + --set prometheusMetrics.podMonitor.enabled=true \ + --set prometheusMetrics.authProxy.enabled="$proxy" "${args[@]}") + grep -F ' port: metrics' <<< "$rendered" + if grep -q ' scheme: https' <<< "$rendered"; then exit 1; fi + grep -F -- "--metrics-addr=:$port" <<< "$rendered" + grep -F "containerPort: $port" <<< "$rendered" + if [ "$proxy" = true ]; then + grep -F -- "--upstream=http://127.0.0.1:$port/" <<< "$rendered" + fi + done + done IFS=. read -r major minor _ <<< "$KUBERNETES_MIN_VERSION" unsupported="${major}.$((minor - 1)).99" if output=$(helm template kafka-operator charts/kafka-operator --kube-version "$unsupported" 2>&1); then diff --git a/charts/kafka-operator/README.md b/charts/kafka-operator/README.md index 47c5fe046..a7ec04551 100644 --- a/charts/kafka-operator/README.md +++ b/charts/kafka-operator/README.md @@ -9,6 +9,71 @@ The [Koperator](https://github.com/adobe/koperator) is a Kubernetes operator to CI covers Kubernetes 1.35.0 and 1.37.0. The chart's `kubeVersion` constraint also accepts distribution-specific version suffixes. +## Alert routing, readiness, and election constraints + +The chart enables leader election. Only its elected listener publishes +`kafka.banzaicloud.io/alert-leader: "true"`, which the alert Service requires. +Never set this label on Pod templates. Webhook-serving standbys remain ready; +elected replicas require usable alert serving/routing, and enabled webhooks +require local serving readiness. Liveness is independent of Kafka and election. +Upgrades from label-unaware versions have an alert endpoint gap until a new +replica becomes leader; plan alert retries during that handoff. + +The chart supplies the operator Pod's `POD_NAMESPACE`, `POD_NAME` and `POD_UID` +through the downward API. Routing uses uncached API reads and UID-checked +updates within that namespace and the exact installation selector: +`control-plane=controller-manager`, `controller-tools.k8s.io=1.0`, +`app.kubernetes.io/component=operator`, and matching +`app.kubernetes.io/name`/`app.kubernetes.io/instance`. Existing chart RBAC grants +Pod get/list/update; custom RBAC must include the operator namespace even when +Kafka is watched elsewhere. Identity/read/write failures fail startup closed. + +For local/non-Helm use, absence of **both** `POD_NAME` and `POD_UID` preserves the +legacy listener, including with `POD_NAMESPACE` alone. Partial identity is +rejected. Opting in requires all three inputs, installation labels and matching +Service selector above. `alertManager.enable` controls exposure, not startup. +Observed cancellation closes admission, drains HTTP requests for up to five +seconds and bounds self-label cleanup to two seconds; cleanup failure cannot +keep the listener open. Endpoint updates are asynchronous; stale requests may +fail or receive HTTP 503. + +Election identity is unchanged. Replicas must enable election, use byte-identical +raw namespace input and share the election namespace. Nonempty `--namespaces` +wins over `WATCH_NAMESPACE`; order, whitespace and duplicates affect the lock +ID even for equivalent watch sets. Do not normalize inputs during rolling +upgrades. Election namespace defaults to the in-cluster service-account +namespace, not watched namespaces or `POD_NAMESPACE`. +Overlapping watch sets (including all-namespaces overlap), differently spelled +equivalent sets and independent election namespaces are unsupported concurrency +topologies. Use replicas of one installation, not separate releases with +different Service selectors sharing a Lease. + +Election and routing labels are not external-action fencing or exactly-once +execution. Pauses/partitions before observed cancellation, admitted work and +ambiguous submissions can outlive routing changes; HTTP draining and conflict +retries do not automatically adopt external operations. + +## Metrics compatibility + +Raw metrics remain bound to all interfaces over unauthenticated HTTP. The +managed PodMonitor, when enabled, scrapes the `metrics` port directly, including +when the existing auth proxy is enabled. The proxy does not protect this raw +endpoint. Listener, container and proxy-upstream ports follow `metricEndpoint.port` +(default 8080). + +Managed TLS/authenticated scraping and its certificate/credential lifecycle are +deferred. This preserves baseline HTTP scraping; it does not secure metrics. +Do not treat the proxy's presence as authentication for the raw endpoint. + +## Existing webhook certificate renewal limits + +Generated webhook serving certificates last 365 days (CA: 3650 days), without +continuous renewal. A generation-enabled Helm upgrade replaces CA and serving +material, not necessarily without downtime. Renew before expiry and coordinate +the Secret, webhook CA bundle and Pod rollout. With `webhook.certs.generate=false`, +refresh supplied CA/certificate/key through existing values and deployment. +No metrics certificate reuse, issuance or automatic renewal is added. + ## Installing the chart Before installing the chart, you must first install the Koperator CustomResourceDefinition resources. diff --git a/charts/kafka-operator/templates/alertmanager-service.yaml b/charts/kafka-operator/templates/alertmanager-service.yaml index 7be7f3913..82369e157 100644 --- a/charts/kafka-operator/templates/alertmanager-service.yaml +++ b/charts/kafka-operator/templates/alertmanager-service.yaml @@ -18,6 +18,7 @@ metadata: app.kubernetes.io/component: alertmanager spec: selector: + kafka.banzaicloud.io/alert-leader: "true" control-plane: controller-manager controller-tools.k8s.io: "1.0" app.kubernetes.io/name: {{ include "kafka-operator.name" . }} diff --git a/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml b/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml index 15944d993..8f67d3bc7 100644 --- a/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml +++ b/charts/kafka-operator/templates/operator-deployment-with-webhook.yaml @@ -1,3 +1,4 @@ +{{- $metricsPort := (.Values.metricEndpoint).port | default 8080 }} {{- $tlsCrt := "" }} {{- $tlsKey := "" }} {{- $caCrt := "" }} @@ -157,7 +158,7 @@ spec: imagePullPolicy: {{ .Values.prometheusMetrics.authProxy.image.pullPolicy }} args: - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" + - "--upstream=http://127.0.0.1:{{ $metricsPort }}/" - "--v=10" ports: - containerPort: 8443 @@ -192,9 +193,7 @@ spec: {{- if .Values.operator.developmentLogging }} - --development {{- end }} - {{- if (.Values.metricEndpoint).port }} - - --metrics-addr=":{{ .Values.metricEndpoint.port }}" - {{- end }} + - --metrics-addr=:{{ $metricsPort }} {{- if .Values.healthProbes.port }} - --health-probes-addr=:{{ .Values.healthProbes.port }} {{- end }} @@ -216,6 +215,16 @@ spec: port: health-probes path: /readyz env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.uid - name: POD_NAMESPACE valueFrom: fieldRef: @@ -230,7 +239,7 @@ spec: name: webhook-server protocol: TCP {{- end}} - - containerPort: {{ (.Values.metricEndpoint).port | default 8080 }} + - containerPort: {{ $metricsPort }} name: metrics protocol: TCP - containerPort: {{ .Values.alertManager.port }} diff --git a/controllers/alertmanager_controller.go b/controllers/alertmanager_controller.go index 037b48b9b..a36eaa3e4 100644 --- a/controllers/alertmanager_controller.go +++ b/controllers/alertmanager_controller.go @@ -16,9 +16,18 @@ package controllers import ( "context" + "errors" + "fmt" "net" "net/http" + "os" + "sync" + "time" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -28,18 +37,42 @@ import ( ) const ( - receiverAddr = ":9001" + receiverAddr = ":9001" + alertLeaderLabel = "kafka.banzaicloud.io/alert-leader" ) // AController implements Runnable type AController struct { - Client client.Client + Client client.Client + reader client.Reader + elected <-chan struct{} + state *alertServingState + pod types.NamespacedName + uid types.UID +} + +type alertServingState struct { + sync.RWMutex + started bool + ready bool + ctx context.Context } // SetAlertManagerWithManager creates a new Alertmanager Controller and adds it to the Manager with default RBAC. The Manager will set fields on the Controller // and Start it when the Manager is Started. func SetAlertManagerWithManager(mgr manager.Manager) error { - return mgr.Add(AController{Client: mgr.GetClient()}) + name, uid, namespace := os.Getenv("POD_NAME"), os.Getenv("POD_UID"), os.Getenv("POD_NAMESPACE") + if (name == "") != (uid == "") || (name != "" && namespace == "") { + return errors.New("managed alert routing requires POD_NAMESPACE, POD_NAME and POD_UID together") + } + c := &AController{ + Client: mgr.GetClient(), reader: mgr.GetAPIReader(), elected: mgr.Elected(), + state: &alertServingState{}, pod: types.NamespacedName{Namespace: namespace, Name: name}, uid: types.UID(uid), + } + if err := mgr.Add(c); err != nil { + return err + } + return mgr.AddReadyzCheck("alerts", c.readyz) } // Start initiates the alertmanager controller @@ -47,7 +80,174 @@ func (c AController) Start(ctx context.Context) error { logf.SetLogger(util.CreateLogger(false, false)) log := logf.Log.WithName("alertmanager") - ln, _ := net.Listen("tcp", receiverAddr) - httpServer := &http.Server{Handler: alertmanager.NewApp(log, c.Client)} - return httpServer.Serve(ln) + ln, err := net.Listen("tcp", receiverAddr) + if err != nil { + return fmt.Errorf("listen for alerts: %w", err) + } + return c.run(ctx, ln, alertmanager.NewApp(log, c.Client)) +} + +func (c AController) readyz(_ *http.Request) error { + if c.state == nil { + return errors.New("alert serving state is unavailable") + } + c.state.RLock() + defer c.state.RUnlock() + if !c.state.started { + select { + case <-c.elected: + return errors.New("elected alert listener has not started") + default: + return nil + } + } + if !c.state.ready || c.state.ctx.Err() != nil { + return errors.New("elected alert listener is not ready") + } + return nil +} + +func (c AController) run(ctx context.Context, ln net.Listener, handler http.Handler) (err error) { + if c.state == nil { + c.state = &alertServingState{} + } + c.state.Lock() + c.state.started, c.state.ready, c.state.ctx = true, false, ctx + c.state.Unlock() + var scope client.MatchingLabels + defer func() { + c.state.Lock() + c.state.ready = false + c.state.Unlock() + _ = ln.Close() + if len(scope) != 0 { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if cleanupErr := c.updateRouteLabel(cleanupCtx, c.pod, c.uid, scope, false); cleanupErr != nil { + err = errors.Join(err, fmt.Errorf("remove alert routing label: %w", cleanupErr)) + } + } + }() + if c.pod.Name != "" { + scope, err = c.prepareRouting(ctx) + if err != nil { + return err + } + } + c.state.Lock() + c.state.ready = true + c.state.Unlock() + return serveAlerts(ctx, ln, c.gatedHandler(ctx, handler), 5*time.Second) +} + +func (c AController) gatedHandler(ctx context.Context, handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + c.state.RLock() + ready := c.state.ready + c.state.RUnlock() + if ctx.Err() != nil || !ready { + http.Error(w, "alert listener is not accepting requests", http.StatusServiceUnavailable) + return + } + handler.ServeHTTP(w, r) + }) +} + +func (c AController) prepareRouting(ctx context.Context) (client.MatchingLabels, error) { + self := &corev1.Pod{} + if err := c.reader.Get(ctx, c.pod, self); err != nil { + return nil, fmt.Errorf("read alert listener Pod: %w", err) + } + if self.UID != c.uid { + return nil, errors.New("alert listener Pod UID does not match POD_UID") + } + scope := client.MatchingLabels{} + for _, key := range []string{ + "control-plane", "controller-tools.k8s.io", "app.kubernetes.io/name", + "app.kubernetes.io/instance", "app.kubernetes.io/component", + } { + if self.Labels[key] == "" { + return nil, fmt.Errorf("alert listener Pod requires installation label %s", key) + } + scope[key] = self.Labels[key] + } + if scope["control-plane"] != "controller-manager" || scope["controller-tools.k8s.io"] != "1.0" || + scope["app.kubernetes.io/component"] != "operator" { + return nil, errors.New("alert listener Pod labels do not match the alert Service selector") + } + pods := &corev1.PodList{} + if err := c.reader.List(ctx, pods, client.InNamespace(c.pod.Namespace), scope); err != nil { + return scope, fmt.Errorf("list alert routing Pods: %w", err) + } + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Labels[alertLeaderLabel] == True { + if err := c.updateRouteLabel(ctx, client.ObjectKeyFromObject(pod), pod.UID, scope, false); err != nil { + return scope, fmt.Errorf("clear stale alert routing label: %w", err) + } + } + } + if err := c.updateRouteLabel(ctx, c.pod, c.uid, scope, true); err != nil { + return scope, fmt.Errorf("publish alert routing label: %w", err) + } + return scope, nil +} + +func (c AController) updateRouteLabel(ctx context.Context, key client.ObjectKey, uid types.UID, scope client.MatchingLabels, active bool) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + pod := &corev1.Pod{} + if err := c.reader.Get(ctx, key, pod); err != nil { + if !active && apierrors.IsNotFound(err) { + return nil + } + return err + } + matches := pod.UID == uid + for label, value := range scope { + matches = matches && pod.Labels[label] == value + } + if !matches { + if !active { + return nil + } + return errors.New("alert routing Pod identity or installation labels changed") + } + if err := ctx.Err(); err != nil { + return err + } + if active { + if pod.DeletionTimestamp != nil { + return errors.New("cannot route alerts to a terminating Pod") + } + pod.Labels[alertLeaderLabel] = True + } else { + if pod.Labels[alertLeaderLabel] != True { + return nil + } + delete(pod.Labels, alertLeaderLabel) + } + return c.Client.Update(ctx, pod) + }) +} + +func serveAlerts(ctx context.Context, ln net.Listener, handler http.Handler, shutdownTimeout time.Duration) error { + httpServer := &http.Server{Handler: handler} + defer httpServer.Close() + defer ln.Close() + + serveDone := make(chan error, 1) + go func() { serveDone <- httpServer.Serve(ln) }() + select { + case err := <-serveDone: + return err + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + shutdownErr := httpServer.Shutdown(shutdownCtx) + serveErr := <-serveDone + if errors.Is(serveErr, http.ErrServerClosed) { + serveErr = nil + } + return errors.Join(shutdownErr, serveErr) + } } diff --git a/controllers/controller_common_test.go b/controllers/controller_common_test.go index eb049668c..e20ccb2d2 100644 --- a/controllers/controller_common_test.go +++ b/controllers/controller_common_test.go @@ -16,15 +16,34 @@ package controllers import ( + "context" "errors" + "fmt" + "io" + "net" + "net/http" + "net/http/httptest" + "net/http/httptrace" "reflect" + "sync" + "sync/atomic" "testing" "time" emperrors "emperror.dev/errors" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + //nolint:staticcheck "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/healthz" logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/manager" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" @@ -38,6 +57,460 @@ const ( testNamespace = "test-namespace" ) +type alertTestManager struct { + manager.Manager + client client.Client + reader client.Reader + elected chan struct{} + runnable manager.Runnable + check healthz.Checker +} + +func (m *alertTestManager) GetClient() client.Client { return m.client } +func (m *alertTestManager) GetAPIReader() client.Reader { return m.reader } +func (m *alertTestManager) Elected() <-chan struct{} { return m.elected } +func (m *alertTestManager) Add(r manager.Runnable) error { m.runnable = r; return nil } +func (m *alertTestManager) AddReadyzCheck(_ string, check healthz.Checker) error { + m.check = check + return nil +} + +func TestAlertManagerIdentityRegistration(t *testing.T) { + for _, tc := range []struct { + name, namespace, pod, uid string + wantErr bool + }{ + {name: "legacy"}, + {name: "namespace alone remains legacy", namespace: "operators"}, + {name: "name alone rejected", namespace: "operators", pod: "operator", wantErr: true}, + {name: "UID alone rejected", namespace: "operators", uid: "uid", wantErr: true}, + {name: "missing namespace rejected", pod: "operator", uid: "uid", wantErr: true}, + {name: "complete", namespace: "operators", pod: "operator", uid: "uid"}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("POD_NAMESPACE", tc.namespace) + t.Setenv("POD_NAME", tc.pod) + t.Setenv("POD_UID", tc.uid) + mgr := &alertTestManager{elected: make(chan struct{})} + err := SetAlertManagerWithManager(mgr) + if (err != nil) != tc.wantErr { + t.Fatalf("registration error = %v, want error = %t", err, tc.wantErr) + } + if !tc.wantErr && mgr.check == nil { + t.Fatal("alert readiness was not registered") + } + }) + } +} + +type alertTestClient struct { + client.Client + update func(context.Context, client.Object, ...client.UpdateOption) error +} + +func (c *alertTestClient) Get(context.Context, client.ObjectKey, client.Object, ...client.GetOption) error { + return errors.New("operator Pod is outside cached namespaces") +} + +func (c *alertTestClient) List(context.Context, client.ObjectList, ...client.ListOption) error { + return errors.New("operator Pods are outside cached namespaces") +} + +func (c *alertTestClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if c.update != nil { + return c.update(ctx, obj, opts...) + } + return c.Client.Update(ctx, obj, opts...) +} + +type alertTestReader struct { + client.Reader + getErr, listErr error +} + +func (r alertTestReader) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if r.getErr != nil { + return r.getErr + } + return r.Reader.Get(ctx, key, obj, opts...) +} + +func (r alertTestReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if r.listErr != nil { + return r.listErr + } + return r.Reader.List(ctx, list, opts...) +} + +func alertTestPod(name, release string) *corev1.Pod { + return &corev1.Pod{ObjectMeta: metav1.ObjectMeta{ + Name: name, Namespace: "operators", UID: types.UID(name), + Labels: map[string]string{ + "control-plane": "controller-manager", "controller-tools.k8s.io": "1.0", + "app.kubernetes.io/name": "kafka-operator", "app.kubernetes.io/instance": release, + "app.kubernetes.io/component": "operator", "kafka.banzaicloud.io/alert-leader": "true", + }, + }} +} + +func newAlertTestController(pods ...client.Object) (AController, *alertTestClient) { + storage := fake.NewClientBuilder().WithObjects(pods...).Build() + writer := &alertTestClient{Client: storage} + elected := make(chan struct{}) + close(elected) + return AController{ + Client: writer, reader: storage, elected: elected, state: &alertServingState{}, + pod: types.NamespacedName{Namespace: "operators", Name: "self"}, uid: "self", + }, writer +} + +func runAlertForTest(t *testing.T, c AController, handler http.Handler) (context.CancelFunc, <-chan error, net.Listener) { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + done, stopped := make(chan error, 1), make(chan struct{}) + go func() { + done <- c.run(ctx, listener, handler) + close(stopped) + }() + t.Cleanup(func() { + cancel() + _ = listener.Close() + select { + case <-stopped: + case <-time.After(4 * time.Second): + t.Error("alert controller did not stop within its cleanup bound") + } + }) + return cancel, done, listener +} + +func TestAlertRoutingUsesScopedUncachedIdentity(t *testing.T) { + self, stale, other := alertTestPod("self", "release"), alertTestPod("stale", "release"), alertTestPod("other", "other-release") + foreign := alertTestPod("foreign", "release") + foreign.Namespace = "other-namespace" + c, writer := newAlertTestController(self, stale, other, foreign) + cancel, done, _ := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + for _, expected := range []struct { + pod *corev1.Pod + active bool + }{{self, true}, {stale, false}, {other, true}, {foreign, true}} { + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), client.ObjectKeyFromObject(expected.pod), current)) + require.Equal(t, expected.active, current.Labels["kafka.banzaicloud.io/alert-leader"] == "true", expected.pod.Name) + } + cancel() + require.NoError(t, <-done) + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Empty(t, current.Labels["kafka.banzaicloud.io/alert-leader"]) +} + +func TestAlertRoutingStartupFailures(t *testing.T) { + for _, failure := range []string{"UID", "labels", "get", "deleted", "list", "write", "conflicts", "ambiguous"} { + t.Run(failure, func(t *testing.T) { + self := alertTestPod("self", "release") + delete(self.Labels, "kafka.banzaicloud.io/alert-leader") + if failure == "labels" { + delete(self.Labels, "app.kubernetes.io/instance") + } + c, writer := newAlertTestController(self) + switch failure { + case "UID": + c.uid = "replacement" + case "get": + c.reader = alertTestReader{Reader: c.reader, getErr: errors.New("read denied")} + case "deleted": + c.reader = alertTestReader{Reader: c.reader, getErr: apierrors.NewNotFound(schema.GroupResource{Resource: "pods"}, "self")} + case "list": + c.reader = alertTestReader{Reader: c.reader, listErr: errors.New("list denied")} + case "write", "conflicts", "ambiguous": + writer.update = func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if obj.GetLabels()["kafka.banzaicloud.io/alert-leader"] == "true" { + if failure == "ambiguous" { + if err := writer.Client.Update(ctx, obj, opts...); err != nil { + return err + } + } + if failure == "conflicts" { + return apierrors.NewConflict(schema.GroupResource{Resource: "pods"}, "self", errors.New("conflict")) + } + return errors.New("publication failed") + } + return writer.Client.Update(ctx, obj, opts...) + } + } + _, done, _ := runAlertForTest(t, c, http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("failed startup admitted an alert") + })) + select { + case err := <-done: + require.Error(t, err) + case <-time.After(time.Second): + t.Fatal("startup failure was not returned") + } + require.Error(t, c.readyz(nil)) + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Empty(t, current.Labels["kafka.banzaicloud.io/alert-leader"]) + }) + } +} + +func TestAlertRoutingConflictPreservesMetadata(t *testing.T) { + self := alertTestPod("self", "release") + delete(self.Labels, "kafka.banzaicloud.io/alert-leader") + c, writer := newAlertTestController(self) + var updates atomic.Int32 + writer.update = func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if updates.Add(1) == 1 { + current := &corev1.Pod{} + if err := writer.Client.Get(ctx, c.pod, current); err != nil { + return err + } + current.Labels["concurrent-label"] = "preserved" + if err := writer.Client.Update(ctx, current); err != nil { + return err + } + return apierrors.NewConflict(schema.GroupResource{Resource: "pods"}, "self", errors.New("concurrent update")) + } + return writer.Client.Update(ctx, obj, opts...) + } + cancel, done, _ := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + cancel() + require.NoError(t, <-done) + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Equal(t, "preserved", current.Labels["concurrent-label"]) + require.GreaterOrEqual(t, updates.Load(), int32(3)) +} + +func TestAlertReadinessPreservesStandbysAndLegacy(t *testing.T) { + c, _ := newAlertTestController() + elected := make(chan struct{}) + c.elected = elected + require.NoError(t, c.readyz(nil), "standby must not require an alert listener") + close(elected) + require.Error(t, c.readyz(nil), "elected startup must not claim serving readiness") + c.pod, c.uid = types.NamespacedName{}, "" + cancel, done, _ := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + cancel() + require.Error(t, c.readyz(nil)) + require.NoError(t, <-done) +} + +func TestAlertRoutingBoundedCleanup(t *testing.T) { + c, writer := newAlertTestController(alertTestPod("self", "release")) + cleanupStarted := make(chan struct{}) + cleanupOnce := sync.OnceFunc(func() { close(cleanupStarted) }) + writer.update = func(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + if obj.GetLabels()["kafka.banzaicloud.io/alert-leader"] != "true" { + // Only block final cleanup, not startup's stale-label removal. + c.state.RLock() + stopping := c.state.ctx != nil && c.state.ctx.Err() != nil + c.state.RUnlock() + if stopping { + cleanupOnce() + <-ctx.Done() + return ctx.Err() + } + } + return writer.Client.Update(ctx, obj, opts...) + } + cancel, done, listener := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + cancel() + select { + case <-cleanupStarted: + case <-time.After(time.Second): + t.Fatal("cleanup did not start") + } + connection, err := net.DialTimeout("tcp", listener.Addr().String(), time.Second) + if connection != nil { + _ = connection.Close() + } + require.Error(t, err, "listener remained open while cleanup was blocked") + select { + case err := <-done: + require.ErrorIs(t, err, context.DeadlineExceeded) + case <-time.After(3 * time.Second): + t.Fatal("blocked label cleanup exceeded its bound") + } +} + +func TestAlertRoutingRejectsCanceledKeepalive(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := AController{state: &alertServingState{started: true, ready: true, ctx: ctx}} + var admitted atomic.Int32 + // Keep the connection open to model stale routing during listener shutdown. + server := httptest.NewServer(c.gatedHandler(ctx, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + admitted.Add(1) + w.WriteHeader(http.StatusNoContent) + }))) + defer server.Close() + transport := &http.Transport{} + defer transport.CloseIdleConnections() + httpClient := &http.Client{Transport: transport, Timeout: time.Second} + response, err := httpClient.Post(server.URL, "application/json", nil) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + require.Equal(t, http.StatusNoContent, response.StatusCode) + + cancel() + var reused atomic.Bool + trace := &httptrace.ClientTrace{GotConn: func(info httptrace.GotConnInfo) { reused.Store(info.Reused) }} + request, err := http.NewRequestWithContext(httptrace.WithClientTrace(context.Background(), trace), http.MethodPost, server.URL, nil) + require.NoError(t, err) + response, err = httpClient.Do(request) + require.NoError(t, err) + defer response.Body.Close() + require.True(t, reused.Load(), "request did not reuse the stale connection") + require.Equal(t, http.StatusServiceUnavailable, response.StatusCode) + require.Equal(t, int32(1), admitted.Load(), "canceled keepalive request reached the receiver") +} + +func TestAlertRoutingCleanupPreservesReplacementUID(t *testing.T) { + self := alertTestPod("self", "release") + c, writer := newAlertTestController(self) + cancel, done, _ := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + require.NoError(t, writer.Delete(context.Background(), self)) + replacement := alertTestPod("self", "release") + replacement.UID = "replacement" + require.NoError(t, writer.Create(context.Background(), replacement)) + cancel() + require.NoError(t, <-done) + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Equal(t, "true", current.Labels["kafka.banzaicloud.io/alert-leader"]) +} + +func TestAlertRoutingServeFailureRemovesLabel(t *testing.T) { + c, writer := newAlertTestController(alertTestPod("self", "release")) + _, done, listener := runAlertForTest(t, c, http.NotFoundHandler()) + require.Eventually(t, func() bool { return c.readyz(nil) == nil }, time.Second, time.Millisecond) + require.NoError(t, listener.Close()) + select { + case err := <-done: + require.ErrorIs(t, err, net.ErrClosed) + case <-time.After(3 * time.Second): + t.Fatal("Serve failure did not stop routing") + } + current := &corev1.Pod{} + require.NoError(t, writer.Client.Get(context.Background(), c.pod, current)) + require.Empty(t, current.Labels["kafka.banzaicloud.io/alert-leader"]) + require.Error(t, c.readyz(nil)) +} + +func TestAlertManagerReportsBindError(t *testing.T) { + listener, err := net.Listen("tcp", receiverAddr) + if err != nil { + t.Skipf("cannot reserve alert listener for bind-error regression: %v", err) + } + defer listener.Close() + + c, writer := newAlertTestController(alertTestPod("self", "release")) + writer.update = func(context.Context, client.Object, ...client.UpdateOption) error { + t.Error("bind failure attempted to change a routing label") + return errors.New("unexpected update") + } + require.Error(t, c.Start(context.Background())) + require.Error(t, c.readyz(nil)) +} + +func TestAlertServerLifecycle(t *testing.T) { + for _, tc := range []struct { + name string + timeout time.Duration + finish bool + serveError bool + expectedErr error + }{ + {name: "drains in-flight request", timeout: 5 * time.Second, finish: true}, + {name: "bounds shutdown and closes connection", timeout: 50 * time.Millisecond, expectedErr: context.DeadlineExceeded}, + {name: "returns serve error and closes connection", serveError: true, expectedErr: net.ErrClosed}, + } { + t.Run(tc.name, func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer listener.Close() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + started, stopped, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + finish := sync.OnceFunc(func() { close(release) }) + defer finish() + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer close(stopped) + close(started) + select { + case <-release: + _, _ = w.Write([]byte("drained")) + case <-r.Context().Done(): + } + }) + serverDone := make(chan error, 1) + go func() { serverDone <- serveAlerts(ctx, listener, handler, tc.timeout) }() + requestDone := make(chan error, 1) + go func() { + httpClient := &http.Client{Timeout: 5 * time.Second} + response, requestErr := httpClient.Get("http://" + listener.Addr().String()) + if requestErr == nil { + defer response.Body.Close() + body, readErr := io.ReadAll(response.Body) + requestErr = readErr + if requestErr == nil && (response.StatusCode != http.StatusOK || string(body) != "drained") { + requestErr = fmt.Errorf("unexpected response: status=%d body=%q", response.StatusCode, body) + } + } + requestDone <- requestErr + }() + select { + case <-started: + case <-time.After(2 * time.Second): + t.Fatal("alert handler did not start") + } + if tc.serveError { + require.NoError(t, listener.Close()) + } else { + cancel() + } + if tc.finish { + select { + case err := <-serverDone: + t.Fatalf("server returned before draining the request: %v", err) + case <-time.After(25 * time.Millisecond): + } + finish() + } + select { + case err := <-serverDone: + require.ErrorIs(t, err, tc.expectedErr) + case <-time.After(2 * time.Second): + t.Fatal("alert server did not stop") + } + select { + case <-stopped: + case <-time.After(2 * time.Second): + t.Fatal("alert handler connection was not closed") + } + select { + case err := <-requestDone: + require.Equal(t, tc.finish, err == nil, "request error: %v", err) + case <-time.After(2 * time.Second): + t.Fatal("alert request did not finish") + } + rebound, err := net.Listen("tcp", listener.Addr().String()) + require.NoError(t, err, "alert listener was not released") + _ = rebound.Close() + }) + } +} + func TestRequeueAfter(t *testing.T) { sec := 5 res, err := requeueAfter(sec) diff --git a/controllers/cruisecontroloperation_controller.go b/controllers/cruisecontroloperation_controller.go index d7da19d39..0bf53b9e8 100644 --- a/controllers/cruisecontroloperation_controller.go +++ b/controllers/cruisecontroloperation_controller.go @@ -243,24 +243,41 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques } } - conflictRetryFunction := func() error { - if err = updateResult(log, cruseControlTaskResult, ccOperationExecution, true); err != nil { - return err - } - err = r.Status().Update(ctx, ccOperationExecution) - if apiErrors.IsConflict(err) { - err = r.Get(ctx, client.ObjectKey{Name: ccOperationExecution.GetName(), Namespace: ccOperationExecution.GetNamespace()}, ccOperationExecution) - } - return err - } - // Protection to avoid re-execution task when status update is failed - if err := util.RetryOnConflict(util.DefaultBackOffForConflict, conflictRetryFunction); err != nil { + // Retry persistence without resubmitting the external task. + if err := r.persistExecutionResult(ctx, log, ccOperationExecution, cruseControlTaskResult); err != nil { return requeueWithError(log, "could not update the result of the Cruise Control user task execution to the CruiseControlOperation status", err) } return reconciled() } +func (r *CruiseControlOperationReconciler) persistExecutionResult( + ctx context.Context, + log logr.Logger, + operation *banzaiv1alpha1.CruiseControlOperation, + result *scale.Result, +) error { + firstAttempt := true + return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error { + latest := &banzaiv1alpha1.CruiseControlOperation{} + if firstAttempt { + latest = operation.DeepCopy() + } else if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil { + return err + } + firstAttempt = false + if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() { + return errors.New("CruiseControlOperation identity or current task changed after external task submission") + } + // Preserve the stop operation selected in memory during finalization. + latest.Status.CurrentTask.Operation = operation.CurrentTaskOperation() + if err := updateResult(log, result, latest, true); err != nil { + return err + } + return r.Status().Update(ctx, latest) + }) +} + func (r *CruiseControlOperationReconciler) addFinalizer(ctx context.Context, currentCCOperation *banzaiv1alpha1.CruiseControlOperation) error { // examine DeletionTimestamp to determine if object is under deletion if currentCCOperation.DeletionTimestamp.IsZero() { diff --git a/controllers/cruisecontroloperation_controller_test.go b/controllers/cruisecontroloperation_controller_test.go index d9a68fd87..770ffa962 100644 --- a/controllers/cruisecontroloperation_controller_test.go +++ b/controllers/cruisecontroloperation_controller_test.go @@ -17,16 +17,22 @@ package controllers import ( "context" + "errors" "testing" "time" "github.com/go-logr/logr" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "go.uber.org/mock/gomock" + apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" "github.com/banzaicloud/koperator/api/v1alpha1" "github.com/banzaicloud/koperator/api/v1beta1" @@ -34,6 +40,183 @@ import ( "github.com/banzaicloud/koperator/pkg/scale" ) +func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) { + for _, tt := range []struct { + name string + retry bool + deleting bool + replaced bool + taskRemoved bool + taskChanged bool + exhausted bool + conflicts int + wantRetries int + wantFailed int + wantOp v1alpha1.CruiseControlTaskOperation + }{ + {name: "first submission without conflict", wantOp: v1alpha1.OperationAddBroker}, + {name: "retry without conflict preserves history once", retry: true, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission without conflict retains operation", deleting: true, wantOp: v1alpha1.OperationStopExecution}, + {name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker}, + {name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker}, + {name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution}, + {name: "replacement object rejects stale result", replaced: true, conflicts: 1}, + {name: "removed task rejects stale result", taskRemoved: true, conflicts: 1}, + {name: "newer task rejects stale result", taskChanged: true, conflicts: 1}, + {name: "exhausted conflicts return error", exhausted: true, conflicts: 1000}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + scheme := runtime.NewScheme() + if err := v1alpha1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + if err := v1beta1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "cluster", Namespace: "ns"}} + operation := &v1alpha1.CruiseControlOperation{ + ObjectMeta: v1.ObjectMeta{ + Name: "operation", Namespace: "ns", UID: "operation-uid", + Labels: map[string]string{v1beta1.KafkaCRLabelKey: "cluster"}, + }, + Spec: v1alpha1.CruiseControlOperationSpec{ErrorPolicy: v1alpha1.ErrorPolicyRetry}, + } + operation.Status.ErrorPolicy = v1alpha1.ErrorPolicyRetry + operation.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationAddBroker} + taskResult := &scale.Result{ + TaskID: "submitted-task", State: v1beta1.CruiseControlTaskInExecution, + StartedAt: "Mon, 02 Jan 2006 15:04:05 GMT", ResponseStatusCode: 202, + RequestURL: "http://cruise-control/add_broker", + } + var previous []*scale.Result + if tt.retry { + operation.Status.CurrentTask.ID = "failed-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskCompletedWithError + operation.Status.CurrentTask.Finished = &v1.Time{Time: time.Date(2006, 1, 2, 15, 5, 0, 0, time.UTC)} + operation.Status.RetryCount = 2 + operation.Status.FailedTasks = []v1alpha1.CruiseControlTask{{ID: "earlier-failed-task"}} + previous = []*scale.Result{{TaskID: "failed-task", State: v1beta1.CruiseControlTaskCompletedWithError}} + } + if tt.deleting { + operation.DeletionTimestamp = &v1.Time{Time: time.Now()} + operation.Finalizers = []string{ccOperationFinalizerGroup} + operation.Status.CurrentTask.ID = "running-task" + operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskInExecution + previous = []*scale.Result{{TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}} + } + scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(t)) + scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{ + AnalyzerReady: true, MonitorReady: true, ExecutorReady: true, + }}, nil) + var taskIDs []any + if len(previous) != 0 { + taskIDs = []any{previous[0].TaskID} + } + scaler.EXPECT().UserTasks(gomock.Any(), taskIDs...).Return(previous, nil) + if tt.deleting { + scaler.EXPECT().StopExecution(gomock.Any()).Return(taskResult, nil).Times(1) + } else { + scaler.EXPECT().AddBrokersWithParams(gomock.Any(), operation.CurrentTaskParameters()).Return(taskResult, nil).Times(1) + } + var resultWrites int + kubeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(cluster, operation). + WithStatusSubresource(operation). + WithInterceptorFuncs(interceptor.Funcs{ + SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error { + if op, ok := obj.(*v1alpha1.CruiseControlOperation); ok && subresource == "status" && op.CurrentTaskID() == "submitted-task" { + resultWrites++ + if resultWrites <= tt.conflicts { + latest := &v1alpha1.CruiseControlOperation{} + if err := c.Get(ctx, client.ObjectKeyFromObject(obj), latest); err != nil { + return err + } + latest.Annotations = map[string]string{"other-writer": "preserved"} + if err := c.Update(ctx, latest); err != nil { + return err + } + if tt.replaced { + latest.Finalizers = nil + if err := c.Update(ctx, latest); err != nil { + return err + } + if err := c.Delete(ctx, latest); err != nil { + return err + } + latest.ResourceVersion = "" + latest.UID = "replacement-uid" + if err := c.Create(ctx, latest); err != nil { + return err + } + } + if tt.taskRemoved || tt.taskChanged { + if tt.taskRemoved { + latest.Status.CurrentTask = nil + } else { + latest.Status.CurrentTask.ID = "newer-task" + } + if err := c.Status().Update(ctx, latest); err != nil { + return err + } + } + return apierrors.NewConflict(schema.GroupResource{Group: "kafka.banzaicloud.io", Resource: "cruisecontroloperations"}, obj.GetName(), errors.New("injected concurrent update")) + } + } + return c.SubResource(subresource).Update(ctx, obj, opts...) + }, + }).Build() + var directReads int + directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + directReads++ + assert.Positive(t, resultWrites, "direct GET must follow a conflicting result write") + return c.Get(ctx, key, obj, opts...) + }, + }) + reconciler := &CruiseControlOperationReconciler{ + Client: kubeClient, DirectClient: directClient, Scheme: scheme, + ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) { + return scaler, nil + }, + } + _, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(operation)}) + if err != nil && !tt.replaced && !tt.taskRemoved && !tt.taskChanged && !tt.exhausted { + t.Fatal(err) + } + persisted := &v1alpha1.CruiseControlOperation{} + if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(operation), persisted); err != nil { + t.Fatal(err) + } + if tt.replaced || tt.taskRemoved || tt.taskChanged || tt.exhausted { + if err == nil { + t.Error("reported success without persisting the result to the original task") + } + if persisted.CurrentTaskID() == "submitted-task" { + t.Error("attached stale external result to a replaced operation or removed task") + } + return + } + assert.Equal(t, "submitted-task", persisted.CurrentTaskID()) + assert.Equal(t, v1beta1.CruiseControlTaskInExecution, persisted.CurrentTaskState()) + assert.Equal(t, tt.wantOp, persisted.CurrentTaskOperation()) + task := persisted.CurrentTask() + require.NotNil(t, task) + assert.Equal(t, taskResult.RequestURL, task.HTTPRequest) + assert.Equal(t, &taskResult.ResponseStatusCode, task.HTTPResponseCode) + require.NotNil(t, task.Started) + assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC))) + assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount) + assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed) + if tt.conflicts > 0 { + assert.Equal(t, "preserved", persisted.Annotations["other-writer"]) + } + assert.Equal(t, tt.conflicts+1, resultWrites) + assert.Equal(t, tt.conflicts, directReads) + }) + } +} + func createCCRetryExecutionOperation(createTime time.Time, id string, operation v1alpha1.CruiseControlTaskOperation) *v1alpha1.CruiseControlOperation { return &v1alpha1.CruiseControlOperation{ ObjectMeta: v1.ObjectMeta{ diff --git a/docs/monitoring.md b/docs/monitoring.md index b6f802f1c..8d917bade 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -37,4 +37,21 @@ sum by (verb) (rate(rest_client_rate_limiter_duration_seconds_sum[5m])) Record cluster size, resource counts, workload, test duration, leader changes, and error rates with each measurement. These metrics establish instrumentation, not evidence that an optimization improves performance. This change does not -alter metrics authentication, scraper routing, worker counts, or cache scope. \ No newline at end of file +alter metrics authentication, scraper routing, worker counts, or cache scope. + +## Cruise Control execution-result persistence + +After an external task is submitted, the operator retries conflicting status +writes using fresh API-server reads. These persistence retries do not submit +the external task again. They retain the task identity, response metadata, +retry history, and selected stop-operation type. A replaced operation, removed +current task, or different task identity produces an error rather than accepting +a stale result. + +This is not exactly-once execution. If all writes fail, or the process stops +after Cruise Control accepts a request but before its task ID is recorded, the +external task can outlive the recorded operation state. Recorded task IDs allow +subsequent reconciliation to query task progress. Automatic adoption of an +unrecorded submission is not implemented; inspect Cruise Control's task history +when investigating this window, and do not infer that a missing status means +the external operation never ran. \ No newline at end of file diff --git a/main.go b/main.go index a854dd863..88b43f844 100644 --- a/main.go +++ b/main.go @@ -174,6 +174,10 @@ func main() { setupLog.Error(err, "unable to start /readyz endpoint") os.Exit(1) } + if err = addWebhookReadyzCheck(mgr, webhookDisabled); err != nil { + setupLog.Error(err, "unable to add webhook readiness check") + os.Exit(1) + } if err := certv1.AddToScheme(mgr.GetScheme()); err != nil { setupLog.Error(err, "") @@ -288,3 +292,10 @@ func main() { os.Exit(1) } } + +func addWebhookReadyzCheck(mgr ctrl.Manager, disabled bool) error { + if disabled { + return nil + } + return mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()) +} diff --git a/main_test.go b/main_test.go index b3841021d..98b521b88 100644 --- a/main_test.go +++ b/main_test.go @@ -16,6 +16,7 @@ package main import ( "context" + "errors" "net/http" "net/http/httptest" "testing" @@ -23,9 +24,71 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics" + "sigs.k8s.io/controller-runtime/pkg/webhook" ) +type webhookReadyTestServer struct { + webhook.Server + err error +} + +func (s *webhookReadyTestServer) StartedChecker() healthz.Checker { + return func(*http.Request) error { return s.err } +} + +type webhookReadyTestManager struct { + ctrl.Manager + server *webhookReadyTestServer + check healthz.Checker + serverRequests, healthRegistrations int +} + +func (m *webhookReadyTestManager) GetWebhookServer() webhook.Server { + m.serverRequests++ + return m.server +} + +func (m *webhookReadyTestManager) AddReadyzCheck(_ string, check healthz.Checker) error { + m.check = check + return nil +} + +func (m *webhookReadyTestManager) AddHealthzCheck(string, healthz.Checker) error { + m.healthRegistrations++ + return nil +} + +func TestWebhookReadinessConditionalRegistration(t *testing.T) { + for _, disabled := range []bool{false, true} { + mgr := &webhookReadyTestManager{server: &webhookReadyTestServer{err: errors.New("not serving TLS")}} + if err := addWebhookReadyzCheck(mgr, disabled); err != nil { + t.Fatal(err) + } + if mgr.healthRegistrations != 0 { + t.Fatal("serving readiness changed liveness registration") + } + if disabled { + if mgr.serverRequests != 0 || mgr.check != nil { + t.Fatal("disabled webhooks started or registered a TLS server") + } + continue + } + if mgr.check == nil || mgr.serverRequests != 1 { + t.Fatal("enabled webhook serving readiness was not registered") + } + if err := mgr.check(httptest.NewRequest(http.MethodGet, "/readyz", nil)); err == nil { + t.Fatal("unusable webhook standby reported ready") + } + mgr.server.err = nil + if err := mgr.check(httptest.NewRequest(http.MethodGet, "/readyz", nil)); err != nil { + t.Fatalf("usable webhook standby is not ready: %v", err) + } + } +} + func TestRESTRequestMetricsAreExposed(t *testing.T) { apiServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json")