lukepolo · Nov 3, 2026 · Nov 3, 2026 · Nov 3, 2026 · Nov 3, 2026
diff --git a/.github/scripts/image_updates.sh b/.github/scripts/image_updates.sh
@@ -0,0 +1,184 @@
+#!/bin/bash
+# Finds newer releases of the third-party images the panel pins, for
+# .github/workflows/image-updates.yml. Run it from inside the repository.
+#
+#   image_updates.sh plan           one line per update: <minor|major> <old ref> <new ref>
+#   image_updates.sh apply OLD NEW  rewrites every pin of OLD to NEW
+#   image_updates.sh title          the commit title for the plan lines on stdin
+#
+# A pin is a tracked `image: <ref>` line or a shell `<NAME>_IMAGE=<ref>`
+# assignment whose tag is a version. An update keeps the tag's shape: its `v`,
+# its number of version parts and the suffix after the version (`-alpine`,
+# `-pg18`, `-k3s1`) stay the same, so a tag only moves to another release of
+# the same build, and prereleases (`-rc1`, `.rc22`) never match. A pinned
+# digest is re-pinned to the new tag's. An update is major when the first
+# version part changes, or below 1.0 the second.
+set -euo pipefail
+
+# Pins that only move by hand.
+IGNORED_PATHS=(
+    # The upstream cert-manager release file, held on 1.17: from 1.18 the
+    # HTTP-01 solver's Ingress is rejected by the older ingress-nginx some
+    # panels still run, and renewals quietly stop.
+    ':!overlays/cert-manager-crds/cert-manager.yaml'
+    # The system-upgrade-controller's release assets, which
+    # tests/scripts/system_upgrade_test.sh checks against their sha256.
+    ':!base/system-upgrade/'
+    # The old releases the TimescaleDB upgrade tests start from.
+    ':!tests/timescaledb/'
+    # This script's own test fixtures.
+    ':!tests/scripts/image_updates_test.sh'
+    # Workflows, which the app's token can't push.
+    ':!.github/'
+)
+
+die() {
+    echo "image_updates.sh: $*" >&2
+    exit 1
+}
+
+cd "$(git rev-parse --show-toplevel)"
+
+# Every distinct ref pinned in the tracked files.
+pins() {
+    git grep -I -h -o -E \
+        -e 'image:[[:space:]]*"?[^[:space:]"]+' \
+        -e '^[[:space:]]*[A-Z0-9_]+_IMAGE=[^[:space:]"]+' \
+        -- "${IGNORED_PATHS[@]}" |
+        sed -E 's/^.*(image:[[:space:]]*"?|_IMAGE=)//' |
+        sort -u
+}
+
+# The newest tags in the list on stdin that keep the shape of tag $1, as
+# "minor <tag>" and "major <tag>" lines.
+newer_tags() {
+    awk -v current="$1" '
+        # Splits a tag into its v, version and suffix. 0 for a tag that is not
+        # a plain version, like latest, 19beta1-alpine or 31.0.rc22-arm64.
+        function parse(tag, out) {
+            out["v"] = ""
+            if (tag ~ /^v[0-9]/) {
+                out["v"] = "v"
+                tag = substr(tag, 2)
+            }
+            if (!match(tag, /^[0-9]+(\.[0-9]+)*/)) return 0
+            out["version"] = substr(tag, 1, RLENGTH)
+            out["suffix"] = substr(tag, RLENGTH + 1)
+            return out["suffix"] == "" || out["suffix"] ~ /^-/
+        }
+        # 1 when version a is newer than version b, which has as many parts.
+        function newer(a, b,   x, y, n, i) {
+            n = split(a, x, ".")
+            split(b, y, ".")
+            for (i = 1; i <= n; i++) {
+                if (x[i] + 0 != y[i] + 0) return x[i] + 0 > y[i] + 0
+            }
+            return 0
+        }
+        BEGIN {
+            if (!parse(current, cur)) exit
+            parts = split(cur["version"], c, ".")
+        }
+        {
+            if (!parse($0, cand)) next
+            if (cand["v"] != cur["v"] || cand["suffix"] != cur["suffix"]) next
+            if (split(cand["version"], n, ".") != parts) next
+            if (!newer(cand["version"], cur["version"])) next
+            kind = "minor"
+            if (n[1] + 0 != c[1] + 0 || (c[1] + 0 == 0 && parts > 1 && n[2] + 0 != c[2] + 0)) kind = "major"
+            if (!(kind in best) || newer(cand["version"], best[kind])) {
+                best[kind] = cand["version"]
+                best_tag[kind] = $0
+            }
+        }
+        END {
+            if ("minor" in best_tag) print "minor", best_tag["minor"]
+            if ("major" in best_tag) print "major", best_tag["major"]
+        }
+    '
+}
+
+plan() {
+    local ref name tag digest kind new_tag new_digest tags
+    pins | while IFS= read -r ref; do
+        digest=""
+        if [[ $ref == *@* ]]; then
+            digest="${ref#*@}"
+        fi
+        name="${ref%%@*}"
+        tag="${name##*:}"
+        # name@digest, a registry port with no tag after it, or a tag like
+        # latest that no release moves
+        if [[ $name != *:* || ! $tag =~ ^v?[0-9]+(\.[0-9]+)*(-.*)?$ ]]; then
+            continue
+        fi
+        name="${name%:*}"
+        if ! tags="$(crane ls "$name" 2>&1 </dev/null)"; then
+            echo "::warning::could not list the tags of $name: $tags" >&2
+            continue
+        fi
+        newer_tags "$tag" <<<"$tags" | while read -r kind new_tag; do
+            if [ -z "$digest" ]; then
+                echo "$kind $ref $name:$new_tag"
+            elif new_digest="$(crane digest "$name:$new_tag" 2>&1 </dev/null)" &&
+                [[ $new_digest =~ ^sha256:[0-9a-f]{64}$ ]]; then
+                echo "$kind $ref $name:$new_tag@$new_digest"
+            else
+                echo "::warning::could not read the digest of $name:$new_tag: $new_digest" >&2
+            fi
+        done
+    done
+}
+
+apply() {
+    local old="$1" new="$2" files=() before
+    while IFS= read -r file; do
+        files+=("$file")
+    done < <(git grep -I -l -F -e "$old" -- "${IGNORED_PATHS[@]}")
+    [ ${#files[@]} -gt 0 ] || die "$old is not pinned anywhere"
+    before="$(cat -- "${files[@]}" | git hash-object --stdin)"
+    OLD="$old" NEW="$new" perl -pi -e \
+        's/(image:[ \t]*"?|_IMAGE=)\Q$ENV{OLD}\E(?=["\s]|$)/$1$ENV{NEW}/g' -- "${files[@]}"
+    if [ "$(cat -- "${files[@]}" | git hash-object --stdin)" = "$before" ]; then
+        die "$old is mentioned, but not pinned, in ${files[*]}"
+    fi
+}
+
+# "redis 8.12.0-alpine" for docker.io/library/redis:8.12.0-alpine@sha256:...
+label() {
+    local ref="${1%%@*}" name
+    name="${ref%:*}"
+    case "${name%%/*}" in
+        *.* | *:* | localhost) [[ $name == */* ]] && name="${name#*/}" ;;
+    esac
+    name="${name#library/}"
+    echo "$name ${ref##*:}"
+}
+
+title() {
+    local kind old new labels=() list
+    while read -r kind old new; do
+        [ -n "$new" ] || continue
+        labels+=("$(label "$new")")
+    done
+    [ ${#labels[@]} -gt 0 ] || die "no updates on stdin"
+    list="${labels[0]}"
+    if [ ${#labels[@]} -gt 1 ]; then
+        list="$(printf '%s, ' "${labels[@]:0:${#labels[@]}-1}")"
+        list="${list%, } and ${labels[${#labels[@]}-1]}"
+    fi
+    if [ ${#list} -gt 200 ]; then
+        list="${#labels[@]} images"
+    fi
+    echo "chore: $list"
+}
+
+case "${1:-}" in
+    plan) plan ;;
+    apply)
+        [ $# -eq 3 ] || die "usage: image_updates.sh apply OLD NEW"
+        apply "$2" "$3"
+        ;;
+    title) title ;;
+    *) die "usage: image_updates.sh plan | apply OLD NEW | title" ;;
+esac
diff --git a/.github/workflows/image-updates.yml b/.github/workflows/image-updates.yml
@@ -0,0 +1,203 @@
+name: Image updates
+
+# Moves the third-party images the panel pins to their newest releases; see
+# .github/scripts/image_updates.sh for what counts as one and which pins only
+# move by hand. Minor and patch releases go into one PR that merges itself
+# once its checks pass. Each major release gets a PR of its own, assigned to
+# REVIEWERS to check that the panel still works on it. Closing a major PR skips
+# that release; a newer major release opens a new PR.
+on:
+  schedule:
+    - cron: "0 13 * * 1"
+  workflow_dispatch:
+
+permissions: {}
+
+concurrency:
+  group: image-updates
+
+env:
+  REVIEWERS: lukepolo
+  CRANE_VERSION: v0.22.1
+  CRANE_SHA256: 0ab7a1d6932a213aed964ce97666c3077fe691c8606413674a8b3e0b9ec4cda0
+  GH_REPO: ${{ github.repository }}
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  update:
+    if: github.repository == '5stackgg/5stack-panel'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    # GITHUB_TOKEN only reads checks and keeps the labels. Branches and PRs
+    # come from the app's token, because ones from GITHUB_TOKEN start no checks
+    # and no release.
+    permissions:
+      contents: read
+      checks: read
+      statuses: read
+      pull-requests: read
+      issues: write
+    steps:
+      - name: Get a token from the 5Stack Ops app
+        id: app
+        uses: actions/create-github-app-token@v3
+        with:
+          client-id: ${{ vars.OPS_APP_CLIENT_ID }}
+          private-key: ${{ secrets.OPS_APP_KEY }}
+          permission-contents: write
+          permission-pull-requests: write
+
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app.outputs.token }}
+          fetch-depth: 0
+
+      - name: Install crane
+        run: |
+          curl -fsSLo "$RUNNER_TEMP/crane.tar.gz" \
+            "https://github.com/google/go-containerregistry/releases/download/$CRANE_VERSION/go-containerregistry_Linux_x86_64.tar.gz"
+          echo "$CRANE_SHA256  $RUNNER_TEMP/crane.tar.gz" | sha256sum -c -
+          tar -xzf "$RUNNER_TEMP/crane.tar.gz" -C "$RUNNER_TEMP" crane
+          echo "$RUNNER_TEMP" >>"$GITHUB_PATH"
+
+      # The steps below switch to branches made from main, so they run a copy
+      # of this commit's script rather than whichever one the branch has.
+      - name: Keep this run's script
+        run: |
+          cp .github/scripts/image_updates.sh "$RUNNER_TEMP/image_updates.sh"
+          echo "SCRIPT=$RUNNER_TEMP/image_updates.sh" >>"$GITHUB_ENV"
+
+      - name: Commit as the app
+        env:
+          GH_TOKEN: ${{ steps.app.outputs.token }}
+          APP_SLUG: ${{ steps.app.outputs.app-slug }}
+        run: |
+          id="$(gh api "/users/$APP_SLUG%5Bbot%5D" --jq .id)"
+          git config user.name "${APP_SLUG}[bot]"
+          git config user.email "$id+${APP_SLUG}[bot]@users.noreply.github.com"
+
+      - name: Keep the labels
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          gh label create image-update --force --color 0e8a16 \
+            --description "Minor and patch image releases, merged once the checks pass"
+          gh label create image-update:major --force --color d93f0b \
+            --description "A major image release to check before merging"
+
+      - name: Merge the minor and patch releases
+        id: minor
+        env:
+          GH_TOKEN: ${{ steps.app.outputs.token }}
+          CHECKS_TOKEN: ${{ github.token }}
+        run: |
+          branch=image-updates/minor
+          "$SCRIPT" plan >"$RUNNER_TEMP/plan"
+          cat "$RUNNER_TEMP/plan"
+          awk '$1 == "minor"' "$RUNNER_TEMP/plan" >"$RUNNER_TEMP/minor"
+          pr="$(gh pr list --head "$branch" --state open --json number --jq '.[0].number // empty')"
+          if [ ! -s "$RUNNER_TEMP/minor" ]; then
+            if [ -n "$pr" ]; then
+              gh pr close "$pr" --delete-branch --comment "main already has these releases."
+            fi
+            exit 0
+          fi
+
+          git switch -q -C "$branch" origin/main
+          while read -r _ old new <&3; do
+            "$SCRIPT" apply "$old" "$new"
+          done 3<"$RUNNER_TEMP/minor"
+          title="$("$SCRIPT" title <"$RUNNER_TEMP/minor")"
+          git commit -q -am "$title"
+          git push -q --force origin "$branch"
+          head="$(git rev-parse HEAD)"
+          if [ -z "$pr" ]; then
+            gh pr create --base main --head "$branch" --title "$title" --body "" --label image-update
+            pr="$(gh pr view "$branch" --json number --jq .number)"
+          else
+            gh pr edit "$pr" --title "$title"
+          fi
+
+          # Let the checks the push started register before waiting on them.
+          sleep 90
+          if ! GH_TOKEN="$CHECKS_TOKEN" gh pr checks "$pr" --watch --fail-fast --interval 30; then
+            gh pr edit "$pr" --add-assignee "$REVIEWERS"
+            echo "::error::#$pr's checks failed, so it was left open for $REVIEWERS"
+            exit 1
+          fi
+          if ! gh pr merge "$pr" --squash --match-head-commit "$head"; then
+            gh pr edit "$pr" --add-assignee "$REVIEWERS"
+            echo "::error::#$pr passed its checks but could not be merged, so it was left open for $REVIEWERS"
+            exit 1
+          fi
+
+      - name: Open a PR for each major release
+        # Also after the minor releases' PR fails, but not when the setup did.
+        if: ${{ !cancelled() && steps.minor.outcome != 'skipped' }}
+        env:
+          GH_TOKEN: ${{ steps.app.outputs.token }}
+        run: |
+          # true when origin's $1 changes main the way HEAD does
+          same_change() {
+            [ "$(git diff "origin/main...origin/$1" | git patch-id --stable)" = \
+              "$(git diff origin/main HEAD | git patch-id --stable)" ]
+          }
+
+          # Planned again on main, which may now have the minor releases.
+          git fetch -q origin main
+          git switch -q --discard-changes --detach origin/main
+          "$SCRIPT" plan >"$RUNNER_TEMP/plan"
+          awk '$1 == "major"' "$RUNNER_TEMP/plan" >"$RUNNER_TEMP/major"
+          cat "$RUNNER_TEMP/major"
+
+          while read -r kind old new <&3; do
+            name="${new%%@*}"
+            tag="${name##*:}"
+            name="${name%:*}"
+            old_tag="${old%%@*}"
+            old_tag="${old_tag##*:}"
+            slug="$(printf '%s' "$name" | tr '/:' '--')"
+            branch="image-updates/major/$slug/$tag"
+            state="$(gh pr list --head "$branch" --state all --json state \
+              --jq 'map(.state) | if any(. == "OPEN") then "open" elif length > 0 then "closed" else "" end')"
+            # Merged, or closed to skip this release.
+            if [ "$state" = closed ]; then
+              continue
+            fi
+
+            git switch -q --discard-changes -C "$branch" origin/main
+            "$SCRIPT" apply "$old" "$new"
+            title="$("$SCRIPT" title <<<"$kind $old $new")"
+            git commit -q -am "$title"
+
+            if [ "$state" = open ]; then
+              # main changed this pin since the PR was opened, as a minor
+              # release of the same image does: rebuild it so it still merges.
+              if ! same_change "$branch"; then
+                git push -q --force origin "$branch"
+              fi
+              continue
+            fi
+
+            git push -q --force origin "$branch"
+            label="${title#chore: }"
+            gh pr create --base main --head "$branch" --title "$title" \
+              --label image-update:major --assignee "$REVIEWERS" --reviewer "$REVIEWERS" \
+              --body "$(printf '%s\n' \
+                "$label is a new major release, up from $old_tag, so it waits for a review instead of merging itself. Before merging, check that the panel still works on it:" \
+                "" \
+                "- [ ] Read its release notes from $old_tag up to $tag for breaking changes and migrations." \
+                "- [ ] Try it on a test panel. This branch is published to the \`branch/$branch\` channel: \`sudo ./service.sh --channel branch/$branch\`." \
+                "" \
+                "Closing this PR skips $tag. A newer major release opens a new PR.")"
+            pr="$(gh pr view "$branch" --json number --jq .number)"
+
+            gh pr list --state open --label image-update:major --json number,headRefName \
+              --jq ".[] | select(.headRefName | startswith(\"image-updates/major/$slug/\")) | select(.headRefName != \"$branch\") | .number" |
+              while read -r superseded; do
+                gh pr close "$superseded" --delete-branch --comment "Superseded by #$pr."
+              done
+          done 3<"$RUNNER_TEMP/major"