From b6994acdbe77187885058f63a8796e4f9cff2766 Mon Sep 17 00:00:00 2001 From: Chris Butler Date: Thu, 19 Nov 2026 03:49:39 +0000 Subject: [PATCH] fix(waves): gating-safe sync-wave layout for every cluster group MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit main.gitops.applicationHealthCheckEnabled is set globally in values-global.yaml, so app-of-apps wave gating is now ON for every cluster group — but the two escape hatches that make a gated first sync work existed only in values-baremetal.yaml: - the wave -1 ClusterRoleBinding bridge (patterns-operator#887): ArgoCD applies wave-0 resources by kind, so the imperative ServiceAccounts are attempted before the chart's own cluster-admin binding; ungated installs self-heal on the next auto-sync, gated ones park on wave-0 health and deadlock. - the wave 15 acm application: the ACM chart renders the MultiClusterHub whose CRD is registered by the ACM operator — a wave-10 subscription. At wave 0 the acm app fails ArgoCD discovery on the not-yet-served operator.open-cluster-management.io/v1 and gates wave 10 behind itself: a hard deadlock. Give every cluster group the same shape as the live-validated values-baremetal.yaml layout (bridge at -1, vault/secrets/kyverno at 0, wait-for-vault-unsealed hook at 5, operator subscriptions at 10, acm at 15, trustee/policies at 20, workloads at 30): - values-baremetal-hub.yaml: bridge; acm -> wave 15; the acm.mce_operator.source tpl-conditional (mirrored catalog in airgap, redhat-operators otherwise); stale commented-out override block removed. - values-azure.yaml, values-trusted-hub.yaml: bridge + wave-5 wait-for-vault-unsealed hook (both run vault + ESO in-cluster) + full wave layout + the mce conditional. - values-azure-spoke.yaml: bridge + wave layout (no vault, no ACM). - values-baremetal-spoke.yaml: bridge; the airgap overlay becomes conditional like every other baremetal file — a CONNECTED spoke was previously pointed at cs-redhat-operator-index-v4-22, a mirrored catalog that does not exist outside airgap, leaving its sourceless subscriptions unresolvable. Connected renders verified unchanged apart from the additions above: full helm render of all six cluster groups diffed against a golden baseline captured at 7efafc8 (clustergroup chart 0.9.58). The spoke's connected render moves from 6 cs-* catalog references to 0. --- values-azure-spoke.yaml | 42 +++++++++++++++ values-azure.yaml | 104 ++++++++++++++++++++++++++++++++++++ values-baremetal-hub.yaml | 44 +++++++++++++-- values-baremetal-spoke.yaml | 31 ++++++++++- values-trusted-hub.yaml | 88 ++++++++++++++++++++++++++++++ 5 files changed, 304 insertions(+), 5 deletions(-) diff --git a/values-azure-spoke.yaml b/values-azure-spoke.yaml index 03066023..78ad6ce0 100644 --- a/values-azure-spoke.yaml +++ b/values-azure-spoke.yaml @@ -27,6 +27,8 @@ clusterGroup: channel: stable installPlanApproval: Manual csv: sandboxed-containers-operator.v1.13.1 + annotations: + argocd.argoproj.io/sync-wave: "10" cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator @@ -45,6 +47,36 @@ clusterGroup: sharedValueFiles: - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' - '/overrides/values-{{ $.Values.global.clusterVersion }}.yaml' + + extraObjects: + # Bridge for the first-sync ordering race (patterns-operator#887): ArgoCD + # applies wave-0 resources by kind, so the imperative ServiceAccounts are + # attempted before the cluster-admin ClusterRoleBinding that authorises the + # controller to create them. Ungated installs self-heal on the next + # auto-sync; with sync-wave gating the first operation parks on wave-0 + # health and deadlocks. This binding applies at wave -1, before any wave-0 + # kind, which the controller can create from the start (the operator's + # instance role already grants rbac.authorization.k8s.io: */*). + # Distinct name on purpose — matching the clustergroup chart's own + # vp-gitops-cluster-admin-rolebinding would collide in the manifest. + # TODO: remove once patterns-operator creates the binding (or the + # clustergroup chart moves its own to wave -1) — see patterns-operator#887. + argocd-cluster-admin-bootstrap: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: coco-pattern-argocd-cluster-admin-bootstrap + annotations: + argocd.argoproj.io/sync-wave: "-1" + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: vp-gitops-argocd-application-controller + namespace: vp-gitops + applications: secrets-operator: @@ -53,6 +85,8 @@ clusterGroup: project: external-secrets chart: openshift-external-secrets chartVersion: 0.0.* + annotations: + argocd.argoproj.io/sync-wave: "0" sandbox: name: sandbox @@ -60,6 +94,8 @@ clusterGroup: project: sandbox chart: sandboxed-containers chartVersion: 0.2.* + annotations: + argocd.argoproj.io/sync-wave: "10" overrides: - name: global.secretStore.backend value: vault @@ -73,18 +109,24 @@ clusterGroup: namespace: hello-openshift project: workloads path: charts/coco-supported/hello-openshift + annotations: + argocd.argoproj.io/sync-wave: "30" kbs-access-curl: name: kbs-access-curl namespace: kbs-access project: workloads path: charts/coco-supported/kbs-access-curl + annotations: + argocd.argoproj.io/sync-wave: "30" kbs-access-sealed: name: kbs-access-sealed namespace: kbs-access project: workloads path: charts/coco-supported/kbs-access-sealed + annotations: + argocd.argoproj.io/sync-wave: "30" imperative: jobs: diff --git a/values-azure.yaml b/values-azure.yaml index e7c2d0ad..ef7d599f 100644 --- a/values-azure.yaml +++ b/values-azure.yaml @@ -27,6 +27,8 @@ clusterGroup: acm: name: advanced-cluster-management namespace: open-cluster-management + annotations: + argocd.argoproj.io/sync-wave: "10" eso: name: openshift-external-secrets-operator namespace: external-secrets-operator @@ -37,16 +39,22 @@ clusterGroup: channel: stable installPlanApproval: Manual csv: sandboxed-containers-operator.v1.13.1 + annotations: + argocd.argoproj.io/sync-wave: "10" trustee: name: trustee-operator namespace: trustee-operator-system channel: stable installPlanApproval: Manual csv: trustee-operator.v1.2.1 + annotations: + argocd.argoproj.io/sync-wave: "10" cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator channel: stable-v1 + annotations: + argocd.argoproj.io/sync-wave: "10" projects: - hub @@ -61,6 +69,69 @@ clusterGroup: sharedValueFiles: - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' - '/overrides/values-{{ $.Values.global.clusterVersion }}.yaml' + + extraObjects: + # Bridge for the first-sync ordering race (patterns-operator#887): ArgoCD + # applies wave-0 resources by kind, so the imperative ServiceAccounts are + # attempted before the cluster-admin ClusterRoleBinding that authorises the + # controller to create them. Ungated installs self-heal on the next + # auto-sync; with sync-wave gating the first operation parks on wave-0 + # health and deadlocks. This binding applies at wave -1, before any wave-0 + # kind, which the controller can create from the start (the operator's + # instance role already grants rbac.authorization.k8s.io: */*). + # Distinct name on purpose — matching the clustergroup chart's own + # vp-gitops-cluster-admin-rolebinding would collide in the manifest. + # TODO: remove once patterns-operator creates the binding (or the + # clustergroup chart moves its own to wave -1) — see patterns-operator#887. + argocd-cluster-admin-bootstrap: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: coco-pattern-argocd-cluster-admin-bootstrap + annotations: + argocd.argoproj.io/sync-wave: "-1" + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: vp-gitops-argocd-application-controller + namespace: vp-gitops + wait-for-vault-unsealed: + apiVersion: batch/v1 + kind: Job + metadata: + name: wait-for-vault-unsealed + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation + argocd.argoproj.io/sync-wave: "5" + spec: + backoffLimit: 10 + activeDeadlineSeconds: 1800 + template: + spec: + restartPolicy: OnFailure + containers: + - name: wait-for-vault + image: registry.redhat.io/openshift4/ose-cli:latest + command: + - /bin/bash + - -c + - | + echo "Waiting for vault to be unsealed..." + while true; do + if curl -sk https://vault.vault.svc:8200/v1/sys/seal-status | grep -q '"sealed":false'; then + echo "Vault unsealed. Waiting 120s for secrets to load..." + sleep 120 + echo "Done. Proceeding." + exit 0 + fi + echo "Vault not ready. Retrying in 15s..." + sleep 15 + done + applications: acm: name: acm @@ -68,6 +139,19 @@ clusterGroup: project: hub chart: acm chartVersion: 0.1.* + # Wave 15: after the wave-10 ACM operator subscription (MCH CRD), + # before trustee at 20. + annotations: + argocd.argoproj.io/sync-wave: "15" + # The MCH operator creates the multicluster-engine subscription at + # runtime from the mce-subscription-spec annotation (rendered from this + # value), so it must be born on the right catalog. Connected: the + # default redhat-operators. Airgap: the mirrored catalog (the default + # catalogs are disabled). Kept as a single tpl-conditional list entry — + # an overlay list would replace this one wholesale. + overrides: + - name: acm.mce_operator.source + value: '{{ if $.Values.global.airgap }}cs-redhat-operator-index-v4-22{{ else }}redhat-operators{{ end }}' vault: name: vault @@ -76,6 +160,8 @@ clusterGroup: chart: hashicorp-vault # Require 0.1.10 or newer within the 0.1.x release series. chartVersion: "~0.1.10" + annotations: + argocd.argoproj.io/sync-wave: "0" secrets-operator: name: openshift-external-secrets @@ -83,12 +169,16 @@ clusterGroup: project: external-secrets chart: openshift-external-secrets chartVersion: 0.0.* + annotations: + argocd.argoproj.io/sync-wave: "0" trustee: name: trustee namespace: trustee-operator-system project: trustee chart: trustee chartVersion: 0.10.* + annotations: + argocd.argoproj.io/sync-wave: "20" extraValueFiles: - '/overrides/values-trustee.yaml' - '/overrides/values-trustee-azure.yaml' @@ -101,12 +191,16 @@ clusterGroup: project: sandbox chart: sandboxed-containers chartVersion: 0.2.* + annotations: + argocd.argoproj.io/sync-wave: "10" sandbox-policies: name: sandbox-policies namespace: openshift-sandboxed-containers-operator project: sandbox chart: sandboxed-policies chartVersion: 0.3.* + annotations: + argocd.argoproj.io/sync-wave: "20" overrides: - name: global.coco.azure.tags value: "key1=value1,key2=value2" @@ -117,6 +211,8 @@ clusterGroup: namespace: hello-openshift project: workloads path: charts/coco-supported/hello-openshift + annotations: + argocd.argoproj.io/sync-wave: "30" syncPolicy: automated: prune: true @@ -126,6 +222,8 @@ clusterGroup: namespace: kbs-access project: workloads path: charts/coco-supported/kbs-access-curl + annotations: + argocd.argoproj.io/sync-wave: "30" syncPolicy: automated: prune: true @@ -135,6 +233,8 @@ clusterGroup: namespace: kbs-access project: workloads path: charts/coco-supported/kbs-access-sealed + annotations: + argocd.argoproj.io/sync-wave: "30" syncPolicy: automated: prune: true @@ -144,6 +244,8 @@ clusterGroup: namespace: kyverno project: hub path: charts/vendor/kyverno + annotations: + argocd.argoproj.io/sync-wave: "0" syncPolicy: automated: {} retry: @@ -232,6 +334,8 @@ clusterGroup: namespace: openshift-sandboxed-containers-operator project: sandbox path: charts/all/coco-kyverno-policies + annotations: + argocd.argoproj.io/sync-wave: "20" imperative: # NOTE: We *must* use lists and not hashes. As hashes lose ordering once parsed by helm diff --git a/values-baremetal-hub.yaml b/values-baremetal-hub.yaml index 9edccf07..fbc1b127 100644 --- a/values-baremetal-hub.yaml +++ b/values-baremetal-hub.yaml @@ -114,6 +114,33 @@ clusterGroup: - '/overrides/values-baremetal-{{ if $.Values.global.airgap }}airgap{{ else }}connected{{ end }}.yaml' extraObjects: + # Bridge for the first-sync ordering race (patterns-operator#887): ArgoCD + # applies wave-0 resources by kind, so the imperative ServiceAccounts are + # attempted before the cluster-admin ClusterRoleBinding that authorises the + # controller to create them. Ungated installs self-heal on the next + # auto-sync; with sync-wave gating the first operation parks on wave-0 + # health and deadlocks. This binding applies at wave -1, before any wave-0 + # kind, which the controller can create from the start (the operator's + # instance role already grants rbac.authorization.k8s.io: */*). + # Distinct name on purpose — matching the clustergroup chart's own + # vp-gitops-cluster-admin-rolebinding would collide in the manifest. + # TODO: remove once patterns-operator creates the binding (or the + # clustergroup chart moves its own to wave -1) — see patterns-operator#887. + argocd-cluster-admin-bootstrap: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: coco-pattern-argocd-cluster-admin-bootstrap + annotations: + argocd.argoproj.io/sync-wave: "-1" + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: vp-gitops-argocd-application-controller + namespace: vp-gitops wait-for-vault-unsealed: apiVersion: batch/v1 kind: Job @@ -155,10 +182,19 @@ clusterGroup: project: hub chart: acm chartVersion: 0.1.* - # Required for airgap -- set to your mirrored catalog source name - # overrides: - # - name: acm.mce_operator.source - # value: "cs-redhat-operator-index-v4-22" + # Wave 15: after the wave-10 ACM operator subscription (MCH CRD), + # before trustee at 20. + annotations: + argocd.argoproj.io/sync-wave: "15" + # The MCH operator creates the multicluster-engine subscription at + # runtime from the mce-subscription-spec annotation (rendered from this + # value), so it must be born on the right catalog. Connected: the + # default redhat-operators. Airgap: the mirrored catalog (the default + # catalogs are disabled). Kept as a single tpl-conditional list entry — + # an overlay list would replace this one wholesale. + overrides: + - name: acm.mce_operator.source + value: '{{ if $.Values.global.airgap }}cs-redhat-operator-index-v4-22{{ else }}redhat-operators{{ end }}' vault: name: vault diff --git a/values-baremetal-spoke.yaml b/values-baremetal-spoke.yaml index fb07b693..0e160003 100644 --- a/values-baremetal-spoke.yaml +++ b/values-baremetal-spoke.yaml @@ -59,7 +59,36 @@ clusterGroup: - '/overrides/values-storage-{{ $.Values.global.storageProvider }}.yaml' - '/overrides/values-{{ $.Values.global.clusterVersion }}.yaml' - '/overrides/values-hw-{{ $.Values.global.hardware.profile }}.yaml' - - '/overrides/values-baremetal-airgap.yaml' + - '/overrides/values-baremetal-{{ if $.Values.global.airgap }}airgap{{ else }}connected{{ end }}.yaml' + + extraObjects: + # Bridge for the first-sync ordering race (patterns-operator#887): ArgoCD + # applies wave-0 resources by kind, so the imperative ServiceAccounts are + # attempted before the cluster-admin ClusterRoleBinding that authorises the + # controller to create them. Ungated installs self-heal on the next + # auto-sync; with sync-wave gating the first operation parks on wave-0 + # health and deadlocks. This binding applies at wave -1, before any wave-0 + # kind, which the controller can create from the start (the operator's + # instance role already grants rbac.authorization.k8s.io: */*). + # Distinct name on purpose — matching the clustergroup chart's own + # vp-gitops-cluster-admin-rolebinding would collide in the manifest. + # TODO: remove once patterns-operator creates the binding (or the + # clustergroup chart moves its own to wave -1) — see patterns-operator#887. + argocd-cluster-admin-bootstrap: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: coco-pattern-argocd-cluster-admin-bootstrap + annotations: + argocd.argoproj.io/sync-wave: "-1" + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: vp-gitops-argocd-application-controller + namespace: vp-gitops applications: secrets-operator: diff --git a/values-trusted-hub.yaml b/values-trusted-hub.yaml index d4d6c217..f95fa788 100644 --- a/values-trusted-hub.yaml +++ b/values-trusted-hub.yaml @@ -22,6 +22,8 @@ clusterGroup: acm: name: advanced-cluster-management namespace: open-cluster-management + annotations: + argocd.argoproj.io/sync-wave: "10" eso: name: openshift-external-secrets-operator namespace: external-secrets-operator @@ -32,6 +34,8 @@ clusterGroup: channel: stable installPlanApproval: Manual csv: trustee-operator.v1.2.1 + annotations: + argocd.argoproj.io/sync-wave: "10" cert-manager: name: openshift-cert-manager-operator namespace: cert-manager-operator @@ -50,6 +54,69 @@ clusterGroup: sharedValueFiles: - '/overrides/values-{{ $.Values.global.clusterPlatform }}.yaml' - '/overrides/values-{{ $.Values.global.clusterVersion }}.yaml' + + extraObjects: + # Bridge for the first-sync ordering race (patterns-operator#887): ArgoCD + # applies wave-0 resources by kind, so the imperative ServiceAccounts are + # attempted before the cluster-admin ClusterRoleBinding that authorises the + # controller to create them. Ungated installs self-heal on the next + # auto-sync; with sync-wave gating the first operation parks on wave-0 + # health and deadlocks. This binding applies at wave -1, before any wave-0 + # kind, which the controller can create from the start (the operator's + # instance role already grants rbac.authorization.k8s.io: */*). + # Distinct name on purpose — matching the clustergroup chart's own + # vp-gitops-cluster-admin-rolebinding would collide in the manifest. + # TODO: remove once patterns-operator creates the binding (or the + # clustergroup chart moves its own to wave -1) — see patterns-operator#887. + argocd-cluster-admin-bootstrap: + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRoleBinding + metadata: + name: coco-pattern-argocd-cluster-admin-bootstrap + annotations: + argocd.argoproj.io/sync-wave: "-1" + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin + subjects: + - kind: ServiceAccount + name: vp-gitops-argocd-application-controller + namespace: vp-gitops + wait-for-vault-unsealed: + apiVersion: batch/v1 + kind: Job + metadata: + name: wait-for-vault-unsealed + annotations: + argocd.argoproj.io/hook: Sync + argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation + argocd.argoproj.io/sync-wave: "5" + spec: + backoffLimit: 10 + activeDeadlineSeconds: 1800 + template: + spec: + restartPolicy: OnFailure + containers: + - name: wait-for-vault + image: registry.redhat.io/openshift4/ose-cli:latest + command: + - /bin/bash + - -c + - | + echo "Waiting for vault to be unsealed..." + while true; do + if curl -sk https://vault.vault.svc:8200/v1/sys/seal-status | grep -q '"sealed":false'; then + echo "Vault unsealed. Waiting 120s for secrets to load..." + sleep 120 + echo "Done. Proceeding." + exit 0 + fi + echo "Vault not ready. Retrying in 15s..." + sleep 15 + done + applications: acm: name: acm @@ -57,6 +124,19 @@ clusterGroup: project: hub chart: acm chartVersion: 0.1.* + # Wave 15: after the wave-10 ACM operator subscription (MCH CRD), + # before trustee at 20. + annotations: + argocd.argoproj.io/sync-wave: "15" + # The MCH operator creates the multicluster-engine subscription at + # runtime from the mce-subscription-spec annotation (rendered from this + # value), so it must be born on the right catalog. Connected: the + # default redhat-operators. Airgap: the mirrored catalog (the default + # catalogs are disabled). Kept as a single tpl-conditional list entry — + # an overlay list would replace this one wholesale. + overrides: + - name: acm.mce_operator.source + value: '{{ if $.Values.global.airgap }}cs-redhat-operator-index-v4-22{{ else }}redhat-operators{{ end }}' vault: name: vault @@ -65,6 +145,8 @@ clusterGroup: chart: hashicorp-vault # Require 0.1.10 or newer within the 0.1.x release series. chartVersion: "~0.1.10" + annotations: + argocd.argoproj.io/sync-wave: "0" secrets-operator: name: openshift-external-secrets @@ -72,6 +154,8 @@ clusterGroup: project: external-secrets chart: openshift-external-secrets chartVersion: 0.0.* + annotations: + argocd.argoproj.io/sync-wave: "0" trustee: name: trustee @@ -79,6 +163,8 @@ clusterGroup: project: trustee chart: trustee chartVersion: 0.10.* + annotations: + argocd.argoproj.io/sync-wave: "20" extraValueFiles: - '/overrides/values-trustee.yaml' overrides: @@ -90,6 +176,8 @@ clusterGroup: project: sandbox chart: sandboxed-policies chartVersion: 0.2.* + annotations: + argocd.argoproj.io/sync-wave: "20" overrides: - name: global.coco.azure.tags value: "key1=value1,key2=value2"