Skip to content

[control-operator] finalizer blocks deletion of task until all pods are gone #830

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
justonedev1 wants to merge 2 commits into
base: master
Choose a base branch
from
+50 −21
Open

[control-operator] finalizer blocks deletion of task until all pods are gone #830

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9b26f2a
[control-operator] finalizer blocks deletion of task until all pods a…
justonedev1 Jul 3, 2026
9b20acf
fixup! [control-operator] finalizer blocks deletion of task until all…
justonedev1 Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 50 additions & 21 deletions control-operator/internal/controller/task_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,31 +274,60 @@ func (r *TaskReconciler) consumeGRPCConsumerIfReady(ctx context.Context, t *alie
// kubectl patch task --type=json -p='[{"op": "remove", "path": "/metadata/finalizers"}]'
func (r *TaskReconciler) handleFinalizer(ctx context.Context, t *aliecsv1alpha1.Task, log logr.Logger) (ctrl.Result, bool, error) {
if t.DeletionTimestamp.IsZero() {
if !controllerutil.ContainsFinalizer(t, taskFinalizer) {
controllerutil.AddFinalizer(t, taskFinalizer)
if err := r.Update(ctx, t); err != nil {
return ctrl.Result{}, true, err
}
return ctrl.Result{}, true, nil
if controllerutil.AddFinalizer(t, taskFinalizer) {
return ctrl.Result{}, true, r.Update(ctx, t)
}
} else {
if controllerutil.ContainsFinalizer(t, taskFinalizer) {
log.Info("Cleaning up gRPC connection before deletion")
if client, exists := clientsForContainers[t.Name]; exists {
if err := client.Close(); err != nil {
log.Error(err, "Failed to close gRPC client during deletion")
}
delete(clientsForContainers, t.Name)
}
return ctrl.Result{}, false, nil
}

controllerutil.RemoveFinalizer(t, taskFinalizer)
if err := r.Update(ctx, t); err != nil {
return ctrl.Result{}, true, err
}
}
if !controllerutil.ContainsFinalizer(t, taskFinalizer) {
return ctrl.Result{}, true, nil
}
return ctrl.Result{}, false, nil

log.Info("Finalizer found, starting cleanup")
r.cleargRPC(t, log)

podGone, err := r.podDeleted(ctx, t, log)
if err != nil || !podGone {
return ctrl.Result{}, true, err
}

controllerutil.RemoveFinalizer(t, taskFinalizer)
return ctrl.Result{}, true, r.Update(ctx, t)
}

// podDeleted deletes the Task's Pod if it still exists, and reports whether it is
// fully gone yet so the caller can wait for termination before removing the finalizer.
func (r *TaskReconciler) podDeleted(ctx context.Context, t *aliecsv1alpha1.Task, log logr.Logger) (bool, error) {
pod := &v1.Pod{}
err := r.Get(ctx, types.NamespacedName{Name: podNameFromTask(t.Name), Namespace: t.Namespace}, pod)
if errors.IsNotFound(err) {
log.Info("POD cleaned")
return true, nil
}
if err != nil {
return false, err
}

if pod.DeletionTimestamp.IsZero() {
log.Info("Deleting pod before removing finalizer")
if err := r.Delete(ctx, pod); err != nil && !errors.IsNotFound(err) {
return false, err
}
}
log.Info("Waiting for pod to terminate before removing finalizer")
return false, nil
}

func (*TaskReconciler) cleargRPC(t *aliecsv1alpha1.Task, log logr.Logger) {
if client, exists := clientsForContainers[t.Name]; exists {
log.Info("Cleaning up gRPC connection")
if err := client.Close(); err != nil {
log.Error(err, "Failed to close gRPC client during deletion")
}
delete(clientsForContainers, t.Name)
log.Info("gRPC cleaned")
}
}

func podNameFromTask(name string) string {
Expand Down
Loading