From 7f3375a55e705e3b3d43cada4eec756800f21274 Mon Sep 17 00:00:00 2001
From: Dmitriy Zhuk <dmitriyzhuk@gmail.com>
Date: Thu, 30 Jul 2026 02:20:22 +0300
Subject: [PATCH] fix(terraform): make apply resilient to interrupted upgrades

Two interrupted applies left the cluster in a bad state: the argocd helm
release got stuck in pending-upgrade (blocking the next apply with "another
operation in progress"), and in-cluster k3s node upgrades that fired mid-apply
cordoned+drained nodes and briefly bounced the API server, then failed before
uncordoning.

- helm_release argocd/argo_workflows/cnpg: add atomic + cleanup_on_fail +
  timeout=900 so a failed/slow upgrade rolls back cleanly instead of getting
  stuck pending-upgrade.
- kube-hetzner: automatically_upgrade_k3s=false, automatically_upgrade_os=false
  so node upgrades are deliberate/supervised and don't cordon nodes during an
  unrelated terraform apply.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 terraform/modules/bootstrap/main.tf | 16 ++++++++++++++++
 terraform/modules/cluster/main.tf   |  8 ++++++++
 2 files changed, 24 insertions(+)

diff --git a/terraform/modules/bootstrap/main.tf b/terraform/modules/bootstrap/main.tf
index 989cdee2..680d28be 100644
--- a/terraform/modules/bootstrap/main.tf
+++ b/terraform/modules/bootstrap/main.tf
@@ -53,6 +53,14 @@ resource "helm_release" "argocd" {
   namespace        = "argocd"
   create_namespace = true
 
+  # Resilience: a failed/interrupted upgrade rolls back instead of getting
+  # stuck in `pending-upgrade` (which blocks the next apply with "another
+  # operation in progress"). Longer timeout absorbs transient API blips
+  # (e.g. an in-flight k3s node upgrade briefly bouncing the API server).
+  timeout         = 900
+  atomic          = true
+  cleanup_on_fail = true
+
   values = [
     yamlencode({
       server = {
@@ -85,6 +93,10 @@ resource "helm_release" "argo_workflows" {
   namespace        = "argo"
   create_namespace = true
 
+  timeout         = 900
+  atomic          = true
+  cleanup_on_fail = true
+
   values = [
     yamlencode({
       server = {
@@ -108,6 +120,10 @@ resource "helm_release" "cnpg" {
   version          = "0.28.0"
   namespace        = "cnpg-system"
   create_namespace = true
+
+  timeout         = 900
+  atomic          = true
+  cleanup_on_fail = true
 }
 
 # Let's Encrypt ClusterIssuer (cert-manager ships in the kube-hetzner module)
diff --git a/terraform/modules/cluster/main.tf b/terraform/modules/cluster/main.tf
index 2f70cd28..4ddf9633 100644
--- a/terraform/modules/cluster/main.tf
+++ b/terraform/modules/cluster/main.tf
@@ -71,6 +71,14 @@ module "kube-hetzner" {
 
   network_region = "eu-central"
 
+  # Don't let the in-cluster system-upgrade-controller auto-upgrade k3s/OS.
+  # Those upgrades cordon+drain a node and briefly bounce the API server; when
+  # they fire during a terraform apply the run can be interrupted mid-upgrade
+  # and leave a node cordoned (as happened on the first rollouts). Pin them so
+  # node upgrades are a deliberate, supervised action instead.
+  automatically_upgrade_k3s = false
+  automatically_upgrade_os  = false
+
   # Use existing x86 snapshot, skip ARM
   microos_x86_snapshot_id = "374341457"
   microos_arm_snapshot_id = "374341457"