Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ setup-envtest: $(LOCALBIN)
test-envtest: setup-envtest
@assets="$$("$(SETUP_ENVTEST)" use "$(ENVTEST_K8S_VERSION)" --bin-dir "$(LOCALBIN)/k8s" -p path)" && \
test -x "$$assets/kube-apiserver" && test -x "$$assets/etcd" && test -x "$$assets/kubectl" && \
KUBEBUILDER_ASSETS="$$assets" go test -tags=envtest ./controllers -run '^TestEnvtest' -count=1 -v
KUBEBUILDER_ASSETS="$$assets" go test -tags=envtest . ./controllers -count=1 -v

test-e2e: test-e2e-remote

Expand Down
20 changes: 14 additions & 6 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,14 @@ NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE
zookeeper-operator 1 1 1 1 12m
```

The operator serves `/healthz` (process liveness) and `/readyz` on
`--health-probe-bind-address` (default `:8081`), separately from metrics.
Helm and Kustomize deployments probe this container port without exposing a
Service. Readiness requires manager cache initialization and becomes false on
shutdown; standby managers can be ready without holding the leader lease.
Neither probe reports ZooKeeper ensemble readiness or requires ZooKeeper to be
available. Use each `ZookeeperCluster`'s status to check its ensemble.

### Deploy a sample Zookeeper cluster

#### Manual deployment
Expand Down Expand Up @@ -486,17 +494,17 @@ The list of available commands are
### API-server integration tests

`make test` runs the fast unit suite without Kubernetes binaries.
Run `make test-envtest` for the opt-in controller API-server tests. This target
Run `make test-envtest` for the opt-in manager and controller API-server tests. This target
installs the `ENVTEST_TOOL_VERSION` pinned in the Makefile and downloads
its `ENVTEST_K8S_VERSION` assets into `bin/k8s`.
Missing or unusable assets fail the target instead of skipping tests.
The same target gates CI and releases.

The tagged suite verifies status-subresource isolation, stale-update conflicts,
and deletion finalizers using the repository CRD and isolated namespaces.
It does not run the operator, a StatefulSet controller, garbage collection, or
a storage provisioner; full reconciliation and storage behavior still require
the live kind e2e suite.
The tagged suite verifies operator probe startup/shutdown behavior,
status-subresource isolation, stale-update conflicts, and deletion finalizers
using the repository CRD and isolated namespaces. It runs the manager in-process
without a StatefulSet controller, garbage collection, or a storage provisioner;
full reconciliation and storage behavior still require the live kind e2e suite.

### Update dependencies

Expand Down
14 changes: 14 additions & 0 deletions charts/zookeeper-operator/templates/operator.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,20 @@ spec:
ports:
- containerPort: {{ int .Values.metricsPort }}
name: metrics
- containerPort: 8081
name: health
livenessProbe:
httpGet:
path: /healthz
port: health
periodSeconds: 10
failureThreshold: 3
readinessProbe:
httpGet:
path: /readyz
port: health
periodSeconds: 5
failureThreshold: 3
command:
- zookeeper-operator
args:
Expand Down
14 changes: 14 additions & 0 deletions config/manager/manager.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,20 @@ spec:
ports:
- containerPort: 60000
name: metrics
- containerPort: 8081
name: health
livenessProbe:
httpGet:
path: /healthz
port: health
periodSeconds: 10
failureThreshold: 3
readinessProbe:
httpGet:
path: /readyz
port: health
periodSeconds: 5
failureThreshold: 3
command:
- zookeeper-operator
imagePullPolicy: Always
Expand Down
19 changes: 14 additions & 5 deletions controllers/zookeepercluster_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/event"
Expand Down Expand Up @@ -837,11 +838,19 @@ func (r *ZookeeperClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
if r.APIReader == nil {
r.APIReader = mgr.GetAPIReader()
}
return ctrl.NewControllerManagedBy(mgr).
For(&zookeeperv1beta1.ZookeeperCluster{}).
Owns(&appsv1.StatefulSet{}).
Owns(&corev1.Service{}).
Owns(&corev1.Pod{}).
primary := &zookeeperv1beta1.ZookeeperCluster{}
owned := []client.Object{&appsv1.StatefulSet{}, &corev1.Service{}, &corev1.Pod{}}
controller := ctrl.NewControllerManagedBy(mgr).For(primary)
for _, object := range owned {
controller.Owns(object)
}
// Initialize the same watches eagerly so manager startup waits for their caches.
for _, object := range append([]client.Object{primary}, owned...) {
if _, err := mgr.GetCache().GetInformer(context.TODO(), object, cache.BlockUntilSynced(false)); err != nil {
return fmt.Errorf("initialize cache for %T: %w", object, err)
}
}
return controller.
WithEventFilter(reconcileEventPredicate()).
Complete(r)
}
Expand Down
62 changes: 56 additions & 6 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,13 +11,16 @@
package main

import (
"context"
"crypto/md5"
"flag"
"fmt"
"io"
"net/http"
"os"
"runtime"
"strings"
"sync/atomic"

zkConfig "github.com/pravega/zookeeper-operator/pkg/controller/config"
"github.com/pravega/zookeeper-operator/pkg/version"
Expand All @@ -30,7 +33,9 @@ import (
_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/metrics/server"

api "github.com/pravega/zookeeper-operator/api/v1beta1"
Expand Down Expand Up @@ -61,8 +66,10 @@ func printVersion() {

func main() {
var metricsAddr string
var healthProbeAddr string
var enableLeaderElection bool
flag.StringVar(&metricsAddr, "metrics-bind-address", "127.0.0.1:6000", "The address the metric endpoint binds to.")
flag.StringVar(&healthProbeAddr, "health-probe-bind-address", ":8081", "The address the health probe endpoint binds to.")
flag.BoolVar(&enableLeaderElection, "enable-leader-election", true,
"Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
opts := zap.Options{}
Expand Down Expand Up @@ -113,11 +120,12 @@ func main() {
}

mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Cache: cacheOpts,
Metrics: server.Options{BindAddress: metricsAddr},
LeaderElection: enableLeaderElection,
LeaderElectionID: leaderElectionID,
Scheme: scheme,
Cache: cacheOpts,
Metrics: server.Options{BindAddress: metricsAddr},
HealthProbeBindAddress: healthProbeAddr,
LeaderElection: enableLeaderElection,
LeaderElectionID: leaderElectionID,
})
if err != nil {
log.Error(err, "unable to start manager")
Expand All @@ -135,13 +143,55 @@ func main() {
}
// +kubebuilder:scaffold:builder

ctx := ctrl.SetupSignalHandler()
if err := configureHealthChecks(ctx, mgr); err != nil {
log.Error(err, "unable to configure health probes")
os.Exit(1)
}

log.Info("starting manager")
if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
if err := mgr.Start(ctx); err != nil {
log.Error(err, "problem running manager")
os.Exit(1)
}
}

func configureHealthChecks(ctx context.Context, mgr manager.Manager) error {
if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil {
return err
}
readiness := &managerReadiness{}
if err := mgr.AddReadyzCheck("manager", func(_ *http.Request) error {
if err := ctx.Err(); err != nil {
return err
}
if !readiness.ready.Load() {
return fmt.Errorf("manager cache is not initialized")
}
return nil
}); err != nil {
return err
}
return mgr.Add(readiness)
}

type managerReadiness struct {
ready atomic.Bool
}

func (r *managerReadiness) Start(ctx context.Context) error {
// The manager starts non-leader runnables only after its caches sync.
r.ready.Store(true)
<-ctx.Done()
r.ready.Store(false)
return nil
}

func (*managerReadiness) NeedLeaderElection() bool {
// Standbys must become ready so a rolling update can replace the old leader.
return false
}

// getWatchNamespace returns the Namespace the operator should be watching for changes
func getWatchNamespace() (string, error) {
// WatchNamespaceEnvVar is the constant for env variable WATCH_NAMESPACE
Expand Down
Loading
Loading