From 4884d008b2889ee3dda2ef78d25508173167ebd7 Mon Sep 17 00:00:00 2001 From: Martin Jackson Date: Fri, 13 Nov 2026 09:07:37 -0600 Subject: [PATCH] Make jobs drain-cancelable The external-secrets validation job is a poll loop and the config jobs run long ansible playbooks, so a node drain evicting one of these pods could burn backoffLimit retries or leave the work half-done. - Validation job: SIGTERM trap exits 143 so a cancelled run is never marked successful - All three Jobs (validation, bootstrap, config cronjob): podFailurePolicy ignores DisruptionTarget drain evictions and exit code 143, so evictions do not burn backoffLimit and a replacement pod resumes the work. The config jobs omit containerName since both the agof-init (bash) and agof-config (timeout-wrapped make) containers exit 143 on SIGTERM. - terminationGracePeriodSeconds: 60 on all job pods so a drain is never blocked waiting for a pod Note: the exit-code rule requires operator: In, and the drain-eviction rule must use onPodConditions with type DisruptionTarget (there is no onDisruptionConditions field). Co-Authored-By: Claude Code --- Chart.yaml | 2 +- README.md | 8 +- README.md.gotmpl | 6 ++ templates/_helpers.tpl | 3 + templates/agof-config-bootstrap-job.yaml | 14 +++ templates/agof-config-cronjob.yaml | 13 +++ .../external-secrets-validation-job.yaml | 22 ++++- tests/drain-cancelable_test.yaml | 91 +++++++++++++++++++ 8 files changed, 156 insertions(+), 3 deletions(-) create mode 100644 tests/drain-cancelable_test.yaml diff --git a/Chart.yaml b/Chart.yaml index 02c28bd..2d6ce2a 100644 --- a/Chart.yaml +++ b/Chart.yaml @@ -5,7 +5,7 @@ home: https://github.com/validatedpatterns/aap-config-chart.git keywords: - pattern name: aap-config -version: 0.2.7 +version: 0.2.8 dependencies: - name: vp-rbac version: '0.1.*' diff --git a/README.md b/README.md index 71841a8..858a8c2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # aap-config -![Version: 0.2.7](https://img.shields.io/badge/Version-0.2.7-informational?style=flat-square) +![Version: 0.2.8](https://img.shields.io/badge/Version-0.2.8-informational?style=flat-square) A Helm chart to build and deploy secrets using external-secrets for ansible-edge-gitops @@ -59,6 +59,12 @@ to 600 (10 min). Run the validation job as an Argo Sync hook at sync-wave 2 with `HookSucceeded,BeforeHookCreation` so ExternalSecrets apply in wave 1 before validation runs and each sync retry gets a fresh job without OutOfSync drift. +* v0.2.8: Make all jobs drain-cancelable: a `podFailurePolicy` ignores +DisruptionTarget drain evictions and exit code 143 (SIGTERM), so an evicted +pod does not burn `backoffLimit` and a replacement pod resumes the work, and +`terminationGracePeriodSeconds` is set to 60 so a node drain is never blocked +waiting for a job pod. + ### Git authentication secret (`agof.gitAuthSecret`) When `agof_repo` or the config-as-code repo (`agof.cac_repo` / `agof.iac_repo`) is diff --git a/README.md.gotmpl b/README.md.gotmpl index de07850..6ce041c 100644 --- a/README.md.gotmpl +++ b/README.md.gotmpl @@ -60,6 +60,12 @@ to 600 (10 min). Run the validation job as an Argo Sync hook at sync-wave 2 with `HookSucceeded,BeforeHookCreation` so ExternalSecrets apply in wave 1 before validation runs and each sync retry gets a fresh job without OutOfSync drift. +* v0.2.8: Make all jobs drain-cancelable: a `podFailurePolicy` ignores +DisruptionTarget drain evictions and exit code 143 (SIGTERM), so an evicted +pod does not burn `backoffLimit` and a replacement pod resumes the work, and +`terminationGracePeriodSeconds` is set to 60 so a node drain is never blocked +waiting for a job pod. + ### Git authentication secret (`agof.gitAuthSecret`) When `agof_repo` or the config-as-code repo (`agof.cac_repo` / `agof.iac_repo`) is diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index 831c076..fba7e9a 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -35,6 +35,9 @@ {{- define "aap-config.app.configjobspec" -}} restartPolicy: Never serviceAccountName: {{ $.Values.serviceAccountName }} +# SIGTERM'd containers exit promptly; keep the grace period short so a node +# drain is never blocked waiting for this pod. +terminationGracePeriodSeconds: 60 volumes: - name: agof-scratch-space emptyDir: diff --git a/templates/agof-config-bootstrap-job.yaml b/templates/agof-config-bootstrap-job.yaml index 277aa5b..ac392b9 100644 --- a/templates/agof-config-bootstrap-job.yaml +++ b/templates/agof-config-bootstrap-job.yaml @@ -7,6 +7,20 @@ metadata: annotations: argocd.argoproj.io/sync-wave: "3" spec: + # A drain-evicted (cancelled) pod is ignored rather than counted as a + # failure: the Job stays incomplete and a replacement pod retries the + # configuration. Only genuine failures count toward backoffLimit. + # No containerName: both the agof-init (bash) and agof-config + # (timeout-wrapped make) containers exit 143 on SIGTERM. + podFailurePolicy: + rules: + - action: Ignore + onExitCodes: + operator: In + values: [143] + - action: Ignore + onPodConditions: + - type: DisruptionTarget parallelism: 1 completions: 1 backoffLimit: 99 diff --git a/templates/agof-config-cronjob.yaml b/templates/agof-config-cronjob.yaml index 2c8fcd7..04e7522 100644 --- a/templates/agof-config-cronjob.yaml +++ b/templates/agof-config-cronjob.yaml @@ -12,6 +12,19 @@ spec: concurrencyPolicy: Forbid jobTemplate: spec: + # A drain-evicted (cancelled) pod is ignored rather than counted as a + # failure; only genuine failures count toward backoffLimit. + # No containerName: both the agof-init (bash) and agof-config + # (timeout-wrapped make) containers exit 143 on SIGTERM. + podFailurePolicy: + rules: + - action: Ignore + onExitCodes: + operator: In + values: [143] + - action: Ignore + onPodConditions: + - type: DisruptionTarget template: spec: activeDeadlineSeconds: {{ $.Values.configJob.activeDeadlineSeconds }} diff --git a/templates/external-secrets-validation-job.yaml b/templates/external-secrets-validation-job.yaml index 58bc48e..5fcd691 100644 --- a/templates/external-secrets-validation-job.yaml +++ b/templates/external-secrets-validation-job.yaml @@ -9,6 +9,19 @@ metadata: argocd.argoproj.io/hook: Sync argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation spec: + # A drain-evicted (cancelled) pod is ignored rather than counted as a + # failure: the Job stays incomplete and a replacement pod resumes the + # validation. Only genuine failures count toward backoffLimit. + podFailurePolicy: + rules: + - action: Ignore + onExitCodes: + containerName: secrets-validator + operator: In + values: [143] + - action: Ignore + onPodConditions: + - type: DisruptionTarget parallelism: 1 completions: 1 backoffLimit: {{ $.Values.validationJob.backoffLimit }} @@ -17,6 +30,9 @@ spec: spec: restartPolicy: Never serviceAccountName: {{ $.Values.serviceAccountName }} + # SIGTERM trap exits promptly; keep the grace period short so a node + # drain is never blocked waiting for this pod. + terminationGracePeriodSeconds: 60 containers: - name: secrets-validator image: {{ .Values.configJob.image }} @@ -26,7 +42,11 @@ spec: - -c - | set -euo pipefail - + # A cancelled run must never be marked successful: exit 143 on + # SIGTERM so the podFailurePolicy ignores it and a replacement + # pod resumes the wait. + trap 'echo "Validation cancelled (SIGTERM); exiting 143" >&2; exit 143' TERM + echo "Starting external secrets validation..." # List of external secrets to validate diff --git a/tests/drain-cancelable_test.yaml b/tests/drain-cancelable_test.yaml new file mode 100644 index 0000000..4e9904c --- /dev/null +++ b/tests/drain-cancelable_test.yaml @@ -0,0 +1,91 @@ +suite: Test jobs are drain-cancelable +templates: + - templates/external-secrets-validation-job.yaml + - templates/agof-config-bootstrap-job.yaml + - templates/agof-config-cronjob.yaml +release: + name: release-test +tests: + - it: Validation job ignores drain evictions and exit 143 + template: templates/external-secrets-validation-job.yaml + asserts: + # Drain evictions are ignored, not counted as failures + - equal: + path: spec.podFailurePolicy.rules[0].action + value: Ignore + - equal: + path: spec.podFailurePolicy.rules[0].onExitCodes.containerName + value: secrets-validator + - equal: + path: spec.podFailurePolicy.rules[0].onExitCodes.operator + value: In + - equal: + path: spec.podFailurePolicy.rules[0].onExitCodes.values + value: [143] + - equal: + path: spec.podFailurePolicy.rules[1].action + value: Ignore + - equal: + path: spec.podFailurePolicy.rules[1].onPodConditions[0].type + value: DisruptionTarget + # Short grace period so a drain is never blocked + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 60 + # SIGTERM trap exits 143 (nonzero) so the Job never completes on cancel + - matchRegex: + path: spec.template.spec.containers[0].command[2] + pattern: trap '.*exit 143' TERM + + - it: Bootstrap job ignores drain evictions and exit 143 + template: templates/agof-config-bootstrap-job.yaml + asserts: + - equal: + path: spec.podFailurePolicy.rules[0].action + value: Ignore + - equal: + path: spec.podFailurePolicy.rules[0].onExitCodes.operator + value: In + - equal: + path: spec.podFailurePolicy.rules[0].onExitCodes.values + value: [143] + - equal: + path: spec.podFailurePolicy.rules[1].action + value: Ignore + - equal: + path: spec.podFailurePolicy.rules[1].onPodConditions[0].type + value: DisruptionTarget + - equal: + path: spec.template.spec.terminationGracePeriodSeconds + value: 60 + + - it: Config cronjob ignores drain evictions and exit 143 + template: templates/agof-config-cronjob.yaml + asserts: + - equal: + path: spec.jobTemplate.spec.podFailurePolicy.rules[0].action + value: Ignore + - equal: + path: spec.jobTemplate.spec.podFailurePolicy.rules[0].onExitCodes.operator + value: In + - equal: + path: spec.jobTemplate.spec.podFailurePolicy.rules[0].onExitCodes.values + value: [143] + - equal: + path: spec.jobTemplate.spec.podFailurePolicy.rules[1].action + value: Ignore + - equal: + path: spec.jobTemplate.spec.podFailurePolicy.rules[1].onPodConditions[0].type + value: DisruptionTarget + - equal: + path: spec.jobTemplate.spec.template.spec.terminationGracePeriodSeconds + value: 60 + + - it: Does not render the validation job when disabled + template: templates/external-secrets-validation-job.yaml + set: + validationJob: + disabled: true + asserts: + - hasDocuments: + count: 0