diff --git a/terraform/modules/bootstrap/main.tf b/terraform/modules/bootstrap/main.tf index 989cdee2..680d28be 100644 --- a/terraform/modules/bootstrap/main.tf +++ b/terraform/modules/bootstrap/main.tf @@ -53,6 +53,14 @@ resource "helm_release" "argocd" { namespace = "argocd" create_namespace = true + # Resilience: a failed/interrupted upgrade rolls back instead of getting + # stuck in `pending-upgrade` (which blocks the next apply with "another + # operation in progress"). Longer timeout absorbs transient API blips + # (e.g. an in-flight k3s node upgrade briefly bouncing the API server). + timeout = 900 + atomic = true + cleanup_on_fail = true + values = [ yamlencode({ server = { @@ -85,6 +93,10 @@ resource "helm_release" "argo_workflows" { namespace = "argo" create_namespace = true + timeout = 900 + atomic = true + cleanup_on_fail = true + values = [ yamlencode({ server = { @@ -108,6 +120,10 @@ resource "helm_release" "cnpg" { version = "0.28.0" namespace = "cnpg-system" create_namespace = true + + timeout = 900 + atomic = true + cleanup_on_fail = true } # Let's Encrypt ClusterIssuer (cert-manager ships in the kube-hetzner module) diff --git a/terraform/modules/cluster/main.tf b/terraform/modules/cluster/main.tf index 2f70cd28..4ddf9633 100644 --- a/terraform/modules/cluster/main.tf +++ b/terraform/modules/cluster/main.tf @@ -71,6 +71,14 @@ module "kube-hetzner" { network_region = "eu-central" + # Don't let the in-cluster system-upgrade-controller auto-upgrade k3s/OS. + # Those upgrades cordon+drain a node and briefly bounce the API server; when + # they fire during a terraform apply the run can be interrupted mid-upgrade + # and leave a node cordoned (as happened on the first rollouts). Pin them so + # node upgrades are a deliberate, supervised action instead. + automatically_upgrade_k3s = false + automatically_upgrade_os = false + # Use existing x86 snapshot, skip ARM microos_x86_snapshot_id = "374341457" microos_arm_snapshot_id = "374341457"