Skip to content

Add remove-ace.sh for Flux and fargocd (Argo CD) setups #1387

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ArnobKumarSaha wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Add remove-ace.sh for Flux and fargocd (Argo CD) setups #1387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
85d5ad2
Add remove-ace.sh for Flux and fargocd (Argo CD) setups
ArnobKumarSaha Oct 30, 2026
74ac920
Remove leftover hook RBAC, ACE GatewayClass and operator-created RBAC…
ArnobKumarSaha Oct 30, 2026
3261f99
Make remove-ace.sh executable
ArnobKumarSaha Oct 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 117 additions & 0 deletions hack/deploy/delete-helmrelease.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
#!/bin/bash

# Copyright AppsCode Inc. and Contributors
#
# Licensed under the AppsCode Community License 1.0.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -euo pipefail

hr=helmreleases.helm.toolkit.fluxcd.io
app=applications.argoproj.io
fargocdFinalizer=fargocd.appscode.com/finalizer
argoNS=${ARGO_NS:-argocd}
timeout=2m
force=false
args=()

for arg in "$@"; do
case $arg in
--timeout=*) timeout=${arg#*=} ;;
--force) force=true ;;
*) args+=("$arg") ;;
esac
done
if [[ ${#args[@]} -lt 1 ]]; then
echo "usage: $(basename "$0") <namespace> [helmrelease...] [--timeout=2m] [--force]" >&2
exit 1
fi
ns=${args[0]}

stripTerminating() {
local kind group objNS name ref
local -a nsArgs
while IFS='|' read -r kind group objNS name; do
[[ -n $name ]] || continue
ref=${kind,,}${group:+.$group}
nsArgs=()
[[ -n $objNS ]] && nsArgs=(-n "$objNS")
if [[ -n $(kubectl get "$ref" "$name" "${nsArgs[@]}" --ignore-not-found -o jsonpath='{.metadata.deletionTimestamp}') ]]; then
echo " removing finalizers of terminating $ref ${objNS:+$objNS/}$name"
kubectl patch "$ref" "$name" "${nsArgs[@]}" --type=merge -p '{"metadata":{"finalizers":null}}'
fi
done
}

# Argo CD never clears argocd.argoproj.io/hook-finalizer of Helm-hook objects when their Application is deleted
unblockApplication() {
kubectl get "$app" "$1" -n "$argoNS" -o json |
jq -r '.status.resources[]? | "\(.kind)|\(.group // "")|\(.namespace // "")|\(.name)"' |
stripTerminating
}

# Operators are uninstalled in parallel, so finalizers on a release's objects may have no controller left to clear them
unblockFluxRelease() {
local release
release=$(kubectl get "$hr" "$1" -n "$ns" -o json | jq -r '.status.history[0] // empty | "\(.namespace) \(.name)"')
if [[ -z $release ]]; then
echo " $ns/$1 has no release history"
return 0
fi
# shellcheck disable=SC2086
helm get manifest -n $release | kubectl get -f - -o json --ignore-not-found |
jq -r '(.items // [.])[] | select(.metadata.deletionTimestamp)
| "\(.kind)|\(if (.apiVersion | contains("/")) then (.apiVersion | split("/")[0]) else "" end)|\(.metadata.namespace // "")|\(.metadata.name)"' |
stripTerminating
}

waitGone() {
kubectl wait --for=delete "$hr/$1" -n "$ns" --timeout="$2" >/dev/null 2>&1
}

for name in "${args[@]:1}"; do
hrJSON=$(kubectl get "$hr" "$name" -n "$ns" -o json --ignore-not-found)
if [[ -z $hrJSON ]]; then
echo "$hr $ns/$name not found, skipping"
continue
fi
appName=""
if [[ $(jq --arg f "$fargocdFinalizer" '.metadata.finalizers // [] | index($f) != null' <<<"$hrJSON") == true ]]; then
appName=$(kubectl get "$app" -n "$argoNS" -o json |
jq -r --arg key "$ns/$name" '.items[] | select(.metadata.annotations["fargocd.appscode.com/helmrelease"] == $key) | .metadata.name')
fi
echo "deleting $hr $ns/$name${appName:+ (Application $argoNS/$appName)}"
kubectl delete "$hr" "$name" -n "$ns" --wait=false
waitGone "$name" "$timeout" && continue

echo "$ns/$name is stuck"
if [[ -n $appName ]] && kubectl get "$app" "$appName" -n "$argoNS" >/dev/null 2>&1; then
unblockApplication "$appName"
waitGone "$name" 1m && continue
if [[ $force != true ]]; then
echo "still stuck; inspect: kubectl get $app $appName -n $argoNS -o yaml; rerun with --force to remove the Application finalizer" >&2
exit 1
fi
echo " --force: removing finalizers of Application $argoNS/$appName"
kubectl patch "$app" "$appName" -n "$argoNS" --type=merge -p '{"metadata":{"finalizers":null}}'
elif [[ -z $appName ]]; then
unblockFluxRelease "$name"
fi
waitGone "$name" 1m && continue

if [[ $force != true ]]; then
echo "$hr $ns/$name still stuck; rerun with --force to remove its finalizers" >&2
exit 1
fi
echo " --force: removing finalizers of $hr $ns/$name"
kubectl patch "$hr" "$name" -n "$ns" --type=merge -p '{"metadata":{"finalizers":null}}'
done
276 changes: 276 additions & 0 deletions hack/deploy/remove-ace.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,276 @@
#!/bin/bash

# Copyright AppsCode Inc. and Contributors
#
# Licensed under the AppsCode Community License 1.0.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://github.com/appscode/licenses/raw/1.0.0/AppsCode-Community-1.0.0.md
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -euo pipefail

hr=helmreleases.helm.toolkit.fluxcd.io
deleteHR=$(dirname "$(readlink -f "$0")")/delete-helmrelease.sh
fullPrune=false
forceArgs=()
timeout=2m
# removed last, as they serve webhooks/APIServices the other releases depend on; aceshifter is the root of their dependsOn
coreHRs="cert-manager|cert-manager-csi-driver-cacerts|license-proxyserver|kube-ui-server|akp-crd-manager|aceshifter"
# dependencies missing from spec.dependsOn: kubestash clears the finalizer of stash-presets' BackupStorage
extraDeps='{"stash-presets":["kubestash"]}'
# third-party CRDs (cert-manager, prometheus-operator, gateway-api, OCM, ...) may be vendor-managed
appscodeGroups='(.*\.)?(appscode\.com|kubedb\.com|kubestash\.com|x-helm\.dev|openviz\.dev|klusters\.dev|voyagermesh\.com)|cacerts\.csi\.cert-manager\.io|fluxcd\.open-cluster-management\.io'

for arg in "$@"; do
case $arg in
--full-prune) fullPrune=true ;;
--force) forceArgs=(--force) ;;
*)
echo "usage: $(basename "$0") [--full-prune] [--force]" >&2
exit 1
;;
esac
done

for bin in kubectl helm jq; do
if ! command -v "$bin" >/dev/null; then
echo "$bin is required" >&2
exit 1
fi
done

crdExists() {
kubectl get crd "$1" >/dev/null 2>&1
}

deleteHRs() {
local ns=$1
shift
[[ $# -gt 0 ]] || return 0
"$deleteHR" "$ns" "$@" --timeout="$timeout" "${forceArgs[@]}"
}

deleteAndWait() {
local resource=$1 ns=$2 name
shift 2
[[ $# -gt 0 ]] || return 0
echo "deleting $resource in $ns: $*"
kubectl delete "$resource" -n "$ns" "$@" --ignore-not-found --wait=false
for name in "$@"; do
kubectl wait --for=delete "$resource/$name" -n "$ns" --timeout="$timeout" >/dev/null 2>&1 && continue
echo "$resource $ns/$name is stuck, removing finalizers"
kubectl patch "$resource" "$name" -n "$ns" --type=merge -p '{"metadata":{"finalizers":null}}' || echo "failed to patch $resource $ns/$name"
done
}

deleteAll() {
local resource=$1 ns
for ns in $(kubectl get "$resource" -A -o json | jq -r '[.items[].metadata.namespace // "default"] | unique[]'); do
# shellcheck disable=SC2046
deleteAndWait "$resource" "$ns" $(kubectl get "$resource" -n "$ns" -o jsonpath='{.items[*].metadata.name}')
done
}

patchAll() {
local resource=$1 patch=$2 filter=$3 ns name
kubectl get "$resource" -A -o json | jq -r ".items[] | select($filter) | \"\(.metadata.namespace // \"default\") \(.metadata.name)\"" |
while read -r ns name; do
kubectl patch "$resource" "$name" -n "$ns" --type=merge -p "$patch"
done
}

requireController() {
local kind=$1 selector=$2 found ns name replicas
found=$(kubectl get "$kind" -A -l "$selector" -o json | jq -r '.items[0] // empty | "\(.metadata.namespace) \(.metadata.name) \(.spec.replicas)"')
if [[ -z $found ]]; then
echo "$kind with labels $selector not found" >&2
exit 1
fi
read -r ns name replicas <<<"$found"
if [[ $replicas == 0 ]]; then
kubectl scale "$kind" "$name" -n "$ns" --replicas=1
fi
kubectl rollout status "$kind" "$name" -n "$ns" --timeout="$timeout"
}

# Deletes the HRs of a {name: [dependsOn...]} map, dependents before their dependencies
deleteInWaves() {
local deps=$1 wave
while [[ $deps != "{}" ]]; do
wave=$(jq -r '([.[][]] | unique) as $needed | keys[] | select(. as $n | $needed | index($n) | not)' <<<"$deps")
if [[ -z $wave ]]; then
echo "dependency cycle, deleting the rest together"
wave=$(jq -r 'keys[]' <<<"$deps")
fi
# the cluster-manager operator tears down the hub it created only while it is still running
if grep -qx cluster-manager-hub <<<"$wave" && crdExists clustermanagers.operator.open-cluster-management.io; then
# their namespaces' ManifestWork finalizers are cleared by the hub's work-controller
crdExists managedclusters.cluster.open-cluster-management.io && timeout=30s deleteAll managedclusters.cluster.open-cluster-management.io
deleteAll clustermanagers.operator.open-cluster-management.io
kubectl delete ns open-cluster-management-hub --ignore-not-found --timeout="$timeout"
fi
# shellcheck disable=SC2086
deleteHRs kubeops $wave
deps=$(jq -c --arg wave "$wave" '($wave | split("\n")) as $w | with_entries(select(.key as $k | $w | index($k) | not))' <<<"$deps")
done
}

owners='["ace-installer"]'
if crdExists featuresets.ui.k8s.appscode.com; then
owners=$(kubectl get featuresets.ui.k8s.appscode.com -o json | jq -c '["ace-installer"] + [.items[].metadata.name]')
fi
# HRs created outside the ace-installer/FeatureSet charts (e.g. by b3) carry only the part-of label
aceHRJSON=$(kubectl get "$hr" -n kubeops -o json | jq --argjson owners "$owners" \
'[.items[] | select((.metadata.annotations["meta.helm.sh/release-name"] // .metadata.labels["app.kubernetes.io/part-of"]) as $r | $owners | index($r))]')
gatewayHR=$(kubectl get "$hr" gateway -n ace -o json --ignore-not-found)
allHRJSON=$(jq --argjson gw "${gatewayHR:-null}" '. + ([$gw] - [null])' <<<"$aceHRJSON")

# fargocd mirrors HRs into Argo CD Applications; the rest are reconciled by Flux helm-controller
if [[ $(jq 'any(.[]; .metadata.finalizers // [] | index("fargocd.appscode.com/finalizer") | not)' <<<"$allHRJSON") == true ]]; then
requireController deploy app.kubernetes.io/component=helm-controller,app.kubernetes.io/part-of=flux
requireController deploy app.kubernetes.io/component=source-controller,app.kubernetes.io/part-of=flux
fi
if [[ $(jq 'any(.[]; .metadata.finalizers // [] | index("fargocd.appscode.com/finalizer"))' <<<"$allHRJSON") == true ]]; then
requireController deploy app.kubernetes.io/name=fargocd
requireController statefulset app.kubernetes.io/name=argocd-application-controller
fi

# Flux is kept: helm-controller skips the uninstall (and HelmChart cleanup) of a suspended HR
if [[ $(jq 'any(.[]; .metadata.name == "flux2")' <<<"$aceHRJSON") == true ]]; then
fluxChart=$(kubectl get "$hr" flux2 -n kubeops -o jsonpath='{.status.helmChart}')
kubectl patch "$hr" flux2 -n kubeops --type=merge -p '{"spec":{"suspend":true}}'
deleteHRs kubeops flux2
if [[ -n $fluxChart ]]; then
kubectl delete helmcharts.source.toolkit.fluxcd.io "${fluxChart#*/}" -n "${fluxChart%%/*}" --ignore-not-found
fi
aceHRJSON=$(jq '[.[] | select(.metadata.name != "flux2")]' <<<"$aceHRJSON")
fi

aceHRs=$(jq -r '.[].metadata.name' <<<"$aceHRJSON")
aceNamespaces=$(jq -c '["ace", "kubeops"] + [.[].spec.targetNamespace // empty] | unique' <<<"$aceHRJSON")
echo "ACE HelmReleases: $(echo $aceHRs)"

if [[ $fullPrune != true ]]; then
userObjects=$(for kind in $(kubectl api-resources --api-group=kubedb.com -o name) backupconfigurations.core.kubestash.com; do
crdExists "$kind" || continue
kubectl get "$kind" -A -o json | jq -r --argjson ns "$aceNamespaces" --arg kind "$kind" \
'.items[] | select(.metadata.namespace as $n | $ns | index($n) | not) | " \($kind) \(.metadata.namespace)/\(.metadata.name)"'
done)
if [[ -n $userObjects ]]; then
printf 'WARNING: these objects outside ACE namespaces lose their operator and keep their finalizers:\n%s\n' "$userObjects" >&2
fi
fi

if [[ $fullPrune == true ]]; then
# their finalizers wait on spoke agents / backup backends; don't block on them
crdExists managedclusters.cluster.open-cluster-management.io && timeout=30s deleteAll managedclusters.cluster.open-cluster-management.io
for kind in $(kubectl api-resources --api-group=kubedb.com -o name); do
patchAll "$kind" '{"spec":{"deletionPolicy":"WipeOut"}}' true
deleteAll "$kind"
done
for kind in backupconfigurations.core.kubestash.com backupsessions.core.kubestash.com restoresessions.core.kubestash.com; do
crdExists "$kind" && deleteAll "$kind"
done
crdExists repositories.storage.kubestash.com && timeout=30s deleteAll repositories.storage.kubestash.com
crdExists backupstorages.storage.kubestash.com && deleteAll backupstorages.storage.kubestash.com
fi

# dashboard finalizers call ACE's Grafana/Perses, which go away with the ace release
for kind in grafanadashboards.openviz.dev persesdashboards.openviz.dev; do
crdExists "$kind" && deleteAll "$kind"
done

# ace runs the platform-api that reconciles FeatureSets; catalog-manager creates ace/gateway
# shellcheck disable=SC2046
deleteHRs kubeops $(grep -x ace <<<"$aceHRs" || true)
# shellcheck disable=SC2046
deleteHRs kubeops $(grep -x catalog-manager <<<"$aceHRs" || true)
deleteHRs ace gateway

# Releases go in reverse dependsOn order, so operators outlive the objects whose finalizers or CRDs they own
# (e.g. OCM addon managers' ClusterManagementAddOns need the hub CRDs).
deleteInWaves "$(jq -c --arg pinned "^(ace|catalog-manager|opscenter-features|$coreHRs)$" --argjson extra "$extraDeps" \
'[.[] | .metadata.name as $n | select($n | test($pinned) | not)
| {key: $n, value: ([.spec.dependsOn[]?.name] + ($extra[$n] // []))}] | from_entries' <<<"$aceHRJSON")"

# before kube-ui-server, which clears the Feature finalizers
# shellcheck disable=SC2046
deleteHRs kubeops $(grep -x opscenter-features <<<"$aceHRs" || true)
for kind in features.ui.k8s.appscode.com featuresets.ui.k8s.appscode.com; do
crdExists "$kind" && deleteAll "$kind"
done

deleteInWaves "$(jq -c --arg core "^($coreHRs)$" \
'[.[] | select(.metadata.name | test($core))] | (map(.metadata.name)) as $names
| map({key: .metadata.name, value: [.spec.dependsOn[]?.name | select(. as $d | $names | index($d))]}) | from_entries' <<<"$aceHRJSON")"

if helm status -n kubeops ace-installer >/dev/null 2>&1; then
helm uninstall -n kubeops ace-installer --wait --timeout "$timeout"
fi

echo "deleting ACE data"
kubectl delete jobs,pvc -n ace --all --ignore-not-found --timeout="$timeout"

for name in $(kubectl get apiservices -o json | jq -r --argjson ns "$aceNamespaces" '.items[]
| select(.spec.service.namespace as $n | $n != null and ($ns | index($n)))
| select(any(.status.conditions[]?; .type == "Available" and .status == "False")) | .metadata.name'); do
kubectl delete apiservice "$name"
done

# helm keeps hook-created webhook configs (e.g. kubestash) on uninstall; their dead services block later installs
for kind in mutatingwebhookconfigurations validatingwebhookconfigurations; do
kubectl get "$kind" -o json | jq -r --argjson ns "$aceNamespaces" '.items[]
| select(.metadata.labels["helm.toolkit.fluxcd.io/namespace"] == "kubeops" or any(.webhooks[]?; .clientConfig.service.namespace as $n | $n != null and ($ns | index($n))))
| .metadata.name as $name | .webhooks[]?.clientConfig.service | select(. != null)
| "\($name) \(.namespace) \(.name)"' | sort -u |
while read -r name ns svc; do
kubectl get svc "$svc" -n "$ns" >/dev/null 2>&1 || kubectl delete "$kind" "$name" --ignore-not-found
done
done

# helm never deletes hook objects on uninstall (e.g. kube-ui-server's aggregated ClusterRoles)
kubectl get clusterroles,clusterrolebindings -l helm.toolkit.fluxcd.io/name -o json | jq -r '.items[]
| select(.metadata.annotations["helm.sh/hook"])
| "\(.kind | ascii_downcase) \(.metadata.name) \(.metadata.labels["helm.toolkit.fluxcd.io/namespace"]) \(.metadata.labels["helm.toolkit.fluxcd.io/name"])"' |
while read -r kind name ns release; do
kubectl get "$hr" "$release" -n "$ns" >/dev/null 2>&1 || kubectl delete "$kind" "$name" --ignore-not-found
done

if [[ $fullPrune == true ]]; then
# the default GatewayClass is kept on uninstall (helm.sh/resource-policy: keep); Envoy Gateway is no longer
# around to clear its gateway-exists finalizer
if crdExists gatewayclasses.gateway.networking.k8s.io; then
# shellcheck disable=SC2046
timeout=30s deleteAndWait gatewayclasses.gateway.networking.k8s.io default $(kubectl get gatewayclasses.gateway.networking.k8s.io -o json |
jq -r '.items[] | select(.metadata.annotations["catalog.appscode.com/gateway-config"]) | .metadata.name')
fi
# created at runtime by the KubeDB and KubeStash operators, outside any release
kubectl get clusterroles,clusterrolebindings -o json | jq -r '.items[]
| select((.metadata.name | test("^kubedb:[a-z-]+-reader$")) or .metadata.labels["app.kubernetes.io/managed-by"] == "kubestash.com")
| "\(.kind | ascii_downcase) \(.metadata.name)"' |
while read -r kind name; do
kubectl delete "$kind" "$name" --ignore-not-found
done

crds=$(kubectl get crd -o json | jq -r --arg re "^($appscodeGroups)$" '.items[] | select(.spec.group | test($re)) | .metadata.name')
for crd in $crds; do
patchAll "$crd" '{"metadata":{"finalizers":null}}' .metadata.finalizers
done
# shellcheck disable=SC2086
deleteAndWait crd default $crds

for ns in $(jq -r '.[]' <<<"$aceNamespaces"); do
if [[ -n $(kubectl get secret -n "$ns" -l owner=helm -o name) || -n $(kubectl get "$hr" -n "$ns" -o name) ]]; then
echo "keeping namespace $ns: it still has Helm or Flux releases"
continue
fi
kubectl delete ns "$ns" --ignore-not-found --timeout="$timeout"
done
fi
Loading