From c25265075d59d5541a4c13fbc17c4cc61f547784 Mon Sep 17 00:00:00 2001 From: Derek Date: Wed, 7 Oct 2026 09:21:53 +1100 Subject: [PATCH 1/3] fix: wait 7 days for GitHub releases, drop dive Every tool taken from a GitHub release now installs the newest release at least 7 days old, through one shared role (roles/github_release) instead of a per-tool copy. hyperi_release_min_age_days sets the age, hyperi-io repos skip it, and with no release old enough the installed copy stays and the run warns. Newest means highest version, so a backport published last on an older line is not picked. hyperi-update applies the same rule, with --min-age or HYPERI_RELEASE_MIN_AGE_DAYS to change it, and lists a kept binary in its summary without failing. hyperi-rust-setup does the same for sccache. The Kubernetes binaries in k8s.yml each get their own rescue, so one dead or too-young upstream no longer skips the rest. hadolint on Fedora now takes the upstream binary as Ubuntu does, and the dnf package goes through the orphan-safe purge once the binary is in. dive and gron are no longer installed. Each role removes its own copies on any run: the /usr/local/bin binary where it resolves under /usr/local, gron's apt package on Ubuntu, and the brew formulae on macOS. Both leave the duplicate sweeps and the updater too. kubectl's minor is now infrastructure_kubectl_minor (v1.37). The pkgs.k8s.io repository and the package follow it, up or down, on the next converge, so it is the knob for cluster version skew. The Ubuntu act install unpacks straight into /usr/local/bin, so a second converge no longer reports it changed. --- README.md | 6 +- ansible/molecule/remediation/prepare.yml | 53 ++- ansible/molecule/remediation/verify.yml | 11 +- ansible/playbooks/group_vars/all.yml | 20 +- ansible/roles/astral/tasks/main.yml | 41 +- ansible/roles/contributor/tasks/act.yml | 57 +-- .../roles/contributor/tasks/actionlint.yml | 19 +- ansible/roles/contributor/tasks/git_scrub.yml | 17 +- ansible/roles/contributor/tasks/gitleaks.yml | 17 +- ansible/roles/contributor/tasks/hadolint.yml | 85 +++-- ansible/roles/contributor/tasks/hyperi_ci.yml | 17 +- ansible/roles/developer-go/tasks/go.yml | 17 +- .../developer-rust/files/hyperi-rust-setup | 49 ++- ansible/roles/developer-rust/tasks/rust.yml | 7 +- .../files/update/hyperi-update-linux.sh | 155 +++++--- ansible/roles/developer/tasks/main.yml | 10 + ansible/roles/developer/tasks/nodejs.yml | 17 +- ansible/roles/developer/tasks/removals.yml | 3 - ansible/roles/developer/tasks/retired.yml | 68 ++++ ansible/roles/developer/tasks/utilities.yml | 96 ++--- ansible/roles/github_release/tasks/main.yml | 69 ++++ .../roles/infrastructure/defaults/main.yml | 15 +- ansible/roles/infrastructure/tasks/cloud.yml | 57 +-- ansible/roles/infrastructure/tasks/k8s.yml | 358 +++++++++--------- ansible/roles/infrastructure/tasks/main.yml | 9 + .../roles/infrastructure/tasks/retired.yml | 39 ++ .../soe/templates/hyperi-update.service.j2 | 2 +- ansible/tests/update/test-hyperi-update.sh | 21 + docs/install-matrix.md | 23 +- install.sh | 2 +- 30 files changed, 831 insertions(+), 529 deletions(-) create mode 100644 ansible/roles/developer/tasks/retired.yml create mode 100644 ansible/roles/github_release/tasks/main.yml create mode 100644 ansible/roles/infrastructure/tasks/retired.yml diff --git a/README.md b/README.md index 0878479..4c433fb 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ flowchart TD | `developer` | Generic CLI dev base (the default: git, docker, shell utilities) | | `developer-gui` | VS Code, Ghostty, DBeaver. Privacy + AI-upsell de-nag profile for VSCode/VSCodium/Cursor off unless `-e vscode_privacy_enabled=true` | | `developer-rust` / `-go` / `-python` / `-node` / `-typescript` / `-c` | Language toolchains | -| `infrastructure` | OpenTofu, OpenBao, AWS CLI, helm, terraform-docs, `k8s` (kubectl, kubectx, kubens, k9s, kind, argocd, dive, kustomize, kubeconform, kube-linter), `data` (clickhouse-client, confluent-cli, rpk, valkey-cli, vector), `cloudflare` (flarectl, wrangler) | +| `infrastructure` | OpenTofu, OpenBao, AWS CLI, helm, terraform-docs, `k8s` (kubectl, kubectx, kubens, k9s, kind, argocd, kustomize, kubeconform, kube-linter), `data` (clickhouse-client, confluent-cli, rpk, valkey-cli, vector), `cloudflare` (flarectl, wrangler) | | `packer` | HashiCorp Packer from HashiCorp's repo or tap. Opt-in, in no persona | | `contributor` | hyperi-ci + its check tools (semgrep, alint), gitleaks, trivy, hadolint, pip-audit, yamllint, ansible-lint, pre-commit, act, git-scrub, macbash | | `soe` / `soe-gui` | HyperI org policy (opt-in) | @@ -179,14 +179,14 @@ digest. Read that before changing a role or adding a tool. - Docker (Engine on Linux, CLI-only via Homebrew on macOS, no Docker Desktop, bring your own daemon) - Git, GitHub CLI, Git LFS -- CLI utilities: jq, gron, bat, fzf, ripgrep, fd, sd, git-delta, lazygit, moreutils, miller, tmux, htop, age, ... +- CLI utilities: jq, bat, fzf, ripgrep, fd, sd, git-delta, lazygit, moreutils, miller, tmux, htop, age, ... **Opt-in, via tags:** - `developer-gui`: VS Code, Ghostty (Solarized theme), DBeaver - `vscode-privacy` (off by default): strips the Copilot/AI upsell UI and the telemetry that stock VSCode ships enabled, across VSCode, VSCodium and Cursor. Enable with `-e vscode_privacy_enabled=true`. It merges one marked block into `settings.json` and never touches a comment or a key it does not manage, backs the file up before its first write, and `-e vscode_privacy_uninstall=true` takes only its own keys back out. Where you have set one of those keys yourself further down the file, yours wins and the run tells you which ones -- so it cannot look applied while changing nothing - Languages: Rust, Go, Python, C/C++, Node.js, TypeScript (the Astral suite -- uv, ruff, ty -- ships in the base, as does Node.js: it is core tooling that semantic-release and CI need) -- `infrastructure`: OpenTofu + OpenBao (the OSS forks in place of terraform and vault), AWS CLI v2, checkov, and terraform-docs for generating IaC module reference docs (engine-agnostic -- it reads `.tf` whichever binary runs it, and OpenTofu has no native `tofu docs`). Under `k8s`: kubectl + kubectx + kubens + k9s + kind + argocd + dive + kustomize + kubeconform + kube-linter. helm is NOT in that group -- it sits in `cloud`, so plain `--tags infrastructure` gets it whether or not you select `k8s`. The `data` group: clickhouse-client, confluent-cli, rpk, valkey-cli, vector. The `cloudflare` group: flarectl + wrangler (flarectl builds from source on both platforms -- Cloudflare ships no binary -- so Linux needs `developer-go`) +- `infrastructure`: OpenTofu + OpenBao (the OSS forks in place of terraform and vault), AWS CLI v2, checkov, and terraform-docs for generating IaC module reference docs (engine-agnostic -- it reads `.tf` whichever binary runs it, and OpenTofu has no native `tofu docs`). Under `k8s`: kubectl + kubectx + kubens + k9s + kind + argocd + kustomize + kubeconform + kube-linter. helm is NOT in that group -- it sits in `cloud`, so plain `--tags infrastructure` gets it whether or not you select `k8s`. The `data` group: clickhouse-client, confluent-cli, rpk, valkey-cli, vector. The `cloudflare` group: flarectl + wrangler (flarectl builds from source on both platforms -- Cloudflare ships no binary -- so Linux needs `developer-go`) - `packer` (off by default, and in no persona): HashiCorp Packer, the one HashiCorp tool we install, because it is BUSL with no open-source fork. Ubuntu and Fedora take HashiCorp's repo, with its signing key checked against the fingerprint HashiCorp publishes before it is trusted; macOS takes `hashicorp/tap`. `--tags removals` removes that repo only where Packer is not installed -- see [roles/packer/README.md](ansible/roles/packer/README.md) - `contributor`: hyperi-ci and the tools its checks drive (semgrep, alint), gitleaks, trivy, hadolint, pip-audit, ansible-lint, pre-commit, act, and git-scrub for rewriting AI residue or a leaked secret out of git history -- gitleaks scans FULL history, so a secret removed from HEAD still fails the gate. macbash is here too: it flags the GNU-only bash constructs that break on macOS, and CONTRIBUTING.md asks for it on every shell change - `soe` / `soe-gui`: HyperI org policy: VPN clients, Claude Code, Slack, LibreOffice, RDP client, telemetry-disable, auto-updates, GNOME taskbar diff --git a/ansible/molecule/remediation/prepare.yml b/ansible/molecule/remediation/prepare.yml index 29a7d01..9866f09 100644 --- a/ansible/molecule/remediation/prepare.yml +++ b/ansible/molecule/remediation/prepare.yml @@ -70,6 +70,29 @@ - linear # Linear CLI — removed by decision 2026-05-01 - minikube # superseded by kind + # DERIVED: k8s.yml put dive in /usr/local/bin on both distros, and Fedora took + # gron's release binary there, until both were dropped. + - name: Plant the dropped dive and gron release binaries + ansible.builtin.copy: + content: | + #!/bin/sh + echo "dropped GitHub-release fixture: {{ item }}" + dest: "/usr/local/bin/{{ item }}" + owner: root + group: root + mode: '0755' + loop: + - dive + - gron + + # DERIVED: utilities.yml installed gron from apt on Ubuntu until it was dropped. + - name: Install the dropped gron package (Ubuntu) + ansible.builtin.apt: + name: gron + state: present + update_cache: true + when: ansible_facts['distribution'] == 'Ubuntu' + # Observed on the reference workstation 2026-10-06: helm v4.1.1, unowned, left by the # get.helm.sh download an older revision did. It shadows packaged helm. - name: Plant the stale root-owned helm (observed v4.1.1) @@ -431,7 +454,7 @@ mode: '0755' loop: - kind - - dive + - lazygit - argocd - macbash - git-scrub @@ -456,7 +479,7 @@ mode: '0755' loop: - .local/bin/kind # shadows /usr/local/bin/kind -- must go - - go/bin/dive # shadows /usr/local/bin/dive -- must go + - go/bin/lazygit # shadows /usr/local/bin/lazygit -- must go - .local/bin/kubeconform # no system copy here -- the only one, must stay - .local/bin/tinygo-dev # not a tool the roles manage -- must stay - .local/bin/yq # pip/uv's yq is another program -- must stay @@ -471,7 +494,7 @@ force: true loop: - {name: argocd, src: /usr/local/bin/argocd} - - {name: dive, src: /nonexistent/dive} + - {name: lazygit, src: /nonexistent/lazygit} # CONSTRUCTED: a directory that happens to carry a managed tool's name. - name: Plant a directory named like a managed tool @@ -483,8 +506,8 @@ # Observed on the reference workstation 2026-10-06: the macbash .deb, from no # repository, beside the role's /usr/local/bin/macbash. CONSTRUCTED: a # git-scrub package another package depends on, a kind package that is not - # on the purge list, and a dive package served by a repository -- all three - # must stay. + # on the purge list, and a k9s package served by a repository -- all three + # must stay. Ubuntu offers no k9s of its own. - name: Install hand-built duplicate packages (Ubuntu) when: ansible_facts['distribution'] == 'Ubuntu' block: @@ -498,7 +521,7 @@ path: "/root/dup-fixture/{{ item.0 }}/{{ item.1 }}" state: directory mode: '0755' - loop: "{{ ['macbash', 'git-scrub', 'git-scrub-dependant', 'kind', 'dive'] | product(['DEBIAN', 'usr/bin']) | list }}" + loop: "{{ ['macbash', 'git-scrub', 'git-scrub-dependant', 'kind', 'k9s'] | product(['DEBIAN', 'usr/bin']) | list }}" - name: Write the fixture control files ansible.builtin.copy: @@ -516,7 +539,7 @@ - {name: git-scrub, depends: ''} - {name: git-scrub-dependant, depends: git-scrub} - {name: kind, depends: ''} - - {name: dive, depends: ''} + - {name: k9s, depends: ''} - name: Write the packaged binaries ansible.builtin.copy: @@ -530,7 +553,7 @@ - git-scrub - git-scrub-dependant - kind - - dive + - k9s - name: Build the fixture packages ansible.builtin.command: @@ -541,7 +564,7 @@ - git-scrub - git-scrub-dependant - kind - - dive + - k9s - name: Install the hand-installed fixture packages ansible.builtin.apt: @@ -552,7 +575,7 @@ - git-scrub-dependant - kind - # dive comes from a local repository instead, so a repository offers it. + # k9s comes from a local repository instead, so a repository offers it. # Under /srv, not /root: apt fetches as the _apt user. - name: Create the fixture repository ansible.builtin.file: @@ -560,10 +583,10 @@ state: directory mode: '0755' - - name: Place the dive package in the fixture repository + - name: Place the k9s package in the fixture repository ansible.builtin.copy: - src: /root/dup-fixture/dive.deb - dest: /srv/fixture-repo/dive.deb + src: /root/dup-fixture/k9s.deb + dest: /srv/fixture-repo/k9s.deb remote_src: true mode: '0644' @@ -594,9 +617,9 @@ - APT::Get::List-Cleanup=0 changed_when: false - - name: Install dive from the fixture repository + - name: Install k9s from the fixture repository ansible.builtin.apt: - name: dive + name: k9s state: present # Fedora: the macbash rpm, from no repository, and golangci-lint from the diff --git a/ansible/molecule/remediation/verify.yml b/ansible/molecule/remediation/verify.yml index 564c7f9..fc05f1c 100644 --- a/ansible/molecule/remediation/verify.yml +++ b/ansible/molecule/remediation/verify.yml @@ -25,6 +25,8 @@ - bw # Bitwarden — removed by decision - linear # Linear CLI — removed by decision - minikube # superseded by kind + - dive # dropped, unmaintained since 2025-05 + - gron # dropped, dormant since 2022 register: verify_gone - name: Assert retired artefacts were removed @@ -64,6 +66,7 @@ - tree - httpie - docker-desktop + - gron loop_control: label: "{{ item }}" when: ansible_facts['distribution'] == 'Ubuntu' @@ -291,11 +294,11 @@ follow: false loop: - {path: .local/bin/kind, gone: true} - - {path: go/bin/dive, gone: true} + - {path: go/bin/lazygit, gone: true} - {path: .local/bin/kubeconform, gone: false} - {path: .local/bin/tinygo-dev, gone: false} - {path: .local/bin/argocd, gone: false} - - {path: .local/bin/dive, gone: false} + - {path: .local/bin/lazygit, gone: false} - {path: .local/bin/kubectx, gone: false} - {path: .local/bin/yq, gone: false} loop_control: @@ -321,7 +324,7 @@ path: "{{ item }}" loop: - /usr/local/bin/kind - - /usr/local/bin/dive + - /usr/local/bin/lazygit - /usr/local/bin/argocd - /usr/local/bin/macbash - /usr/local/bin/git-scrub @@ -382,7 +385,7 @@ is offered by a repository, or is the only Go. success_msg: "{{ item }} kept" loop: >- - {{ ((['git-scrub', 'git-scrub-dependant', 'kind', 'dive'] + {{ ((['git-scrub', 'git-scrub-dependant', 'kind', 'k9s'] + (['golang-go'] if inventory_hostname is search('golink|gosrc') else [])) if ansible_facts['distribution'] == 'Ubuntu' else ['golangci-lint', 'golang']) }} diff --git a/ansible/playbooks/group_vars/all.yml b/ansible/playbooks/group_vars/all.yml index cad7c4a..68ff7ab 100644 --- a/ansible/playbooks/group_vars/all.yml +++ b/ansible/playbooks/group_vars/all.yml @@ -2,7 +2,8 @@ # Every tool resolves its version at install time, so a run in four months # installs what is current in four months. # -# Manual binaries query `/releases/latest`. Cargo tools install unversioned. Go +# Manual binaries take the newest GitHub release past the release-age cooldown +# below. Cargo tools install unversioned. Go # and rustup fetch the current version with the checksum published beside it. # # What remains below is a selection rather than a version -- a Node LTS line, @@ -31,6 +32,23 @@ hyperi_github_headers: >- hyperi_github_env: >- {{ {'GITHUB_TOKEN': hyperi_github_token} if hyperi_github_token else {} }} +# Every tool taken from a GitHub release installs the newest release at least +# this many days old, so a compromised or broken release has time to be pulled +# first (roles/github_release). Nothing newer is ever installed in its place: +# with no release old enough, the installed copy stays and the run warns. +# hyperi-update applies the same age, overridable there with --min-age or +# HYPERI_RELEASE_MIN_AGE_DAYS. +hyperi_release_min_age_days: 7 + +# Orgs whose releases ship through our own CI gates, so they skip the wait. +hyperi_release_cooldown_exempt: + - hyperi-io + +# The release-age rule as a process environment, for the scripts that look up +# their own releases. +hyperi_release_env: + HYPERI_RELEASE_MIN_AGE_DAYS: "{{ hyperi_release_min_age_days | string }}" + # ============================================================================ # Core component versions -- the ONE place to bump them # ============================================================================ diff --git a/ansible/roles/astral/tasks/main.yml b/ansible/roles/astral/tasks/main.yml index f964be4..bef2a39 100644 --- a/ansible/roles/astral/tasks/main.yml +++ b/ansible/roles/astral/tasks/main.yml @@ -52,17 +52,18 @@ when: ansible_facts['distribution'] == 'Ubuntu' tags: ['astral', 'uv'] block: - - name: Get latest uv version from GitHub API - ansible.builtin.uri: - url: https://api.github.com/repos/astral-sh/uv/releases/latest - return_content: true - headers: "{{ hyperi_github_headers }}" - register: uv_latest_release - check_mode: false + - name: Resolve the uv release + ansible.builtin.include_role: + name: github_release + # The include is already gated by this file's tags, and its tasks carry none. + apply: + tags: ['always'] + vars: + github_release_repo: astral-sh/uv - name: Set uv version fact ansible.builtin.set_fact: - uv_version: "{{ uv_latest_release.json.tag_name }}" + uv_version: "{{ github_release_doc.tag_name }}" # uv publishes a gnu build for both arches, so the libc token does not move. - name: Determine uv binary architecture @@ -103,9 +104,27 @@ - not ansible_check_mode - ((astral_uv_installed.stdout | default('')).split() + ['', ''])[1] != uv_version -# No rescue: ruff and ty below are installed by `uv tool install`, so warning -# past a missing uv only moves the abort to a task that reports it as a missing -# command. + # A host that already has uv keeps it and warns. One without it still fails + # here: ruff and ty below are installed by `uv tool install`, so warning past + # a missing uv only moves the abort to a task that reports a missing command. + rescue: + - name: Check for an installed uv + ansible.builtin.stat: + path: "{{ user_home }}/.local/bin/uv" + register: astral_uv_present + + - name: Fail without a uv to fall back on + ansible.builtin.fail: + msg: "uv: {{ ansible_failed_result.msg | default('install failed') }}" + when: not (astral_uv_present.stat.executable | default(false)) + + - name: Record that uv was not updated + # noqa: var-naming[no-role-prefix] -- deploy_warnings is the shared + # accumulator reported by playbooks/main.yml post_tasks. + ansible.builtin.set_fact: + deploy_warnings: >- + {{ deploy_warnings | default([]) + + ['uv: ' ~ (ansible_failed_result.msg | default('update failed'))] }} # ============================================================================ # ruff + ty - repo-managed on Fedora/macOS; uv tools on Ubuntu diff --git a/ansible/roles/contributor/tasks/act.yml b/ansible/roles/contributor/tasks/act.yml index 06630a9..1dc5bd3 100644 --- a/ansible/roles/contributor/tasks/act.yml +++ b/ansible/roles/contributor/tasks/act.yml @@ -35,56 +35,37 @@ - name: Install act (Ubuntu) block: - - name: Get latest act version from GitHub API - ansible.builtin.uri: - url: https://api.github.com/repos/nektos/act/releases/latest - return_content: true - headers: "{{ hyperi_github_headers }}" - register: act_release - check_mode: false - - - name: Set act version fact - ansible.builtin.set_fact: - act_version: "{{ act_release.json.tag_name }}" + - name: Resolve the act release + ansible.builtin.include_role: + name: github_release + # The include is already gated by this file's tags, and its tasks carry none. + apply: + tags: ['always'] + vars: + github_release_repo: nektos/act # act does NOT use one naming scheme across arches: amd64 is `x86_64`, # arm64 is `arm64` (not `aarch64`). So neither shared token fits both, and # a substring swap on either would 404. Verified against the real release. - name: Determine the act asset architecture ansible.builtin.set_fact: - act_arch: "{{ 'arm64' if hyperi_arch_deb == 'arm64' else 'x86_64' }}" + contributor_act_arch: "{{ 'arm64' if hyperi_arch_deb == 'arm64' else 'x86_64' }}" - - name: Download act binary tarball - ansible.builtin.get_url: - url: "https://github.com/nektos/act/releases/download/{{ act_version }}/act_Linux_{{ act_arch }}.tar.gz" - dest: /tmp/act.tar.gz - mode: '0644' - - - name: Extract act binary + # Straight from the URL, as gitleaks does: unarchive reports a change only + # when the binary differs, and no root-written file sits at a fixed /tmp path. + - name: Download and extract act ansible.builtin.unarchive: - src: /tmp/act.tar.gz - dest: /tmp + src: >- + https://github.com/nektos/act/releases/download/{{ github_release_doc.tag_name + }}/act_Linux_{{ contributor_act_arch }}.tar.gz + dest: /usr/local/bin remote_src: true - when: not ansible_check_mode - - - name: Install act binary - ansible.builtin.copy: - src: /tmp/act - dest: /usr/local/bin/act + include: act + owner: root + group: root mode: '0755' - remote_src: true when: not ansible_check_mode - - name: Remove act installation files - ansible.builtin.file: - path: "{{ item }}" - state: absent - loop: - - /tmp/act.tar.gz - - /tmp/act - - /tmp/LICENSE - - /tmp/README.md - rescue: - name: Record that act did not install (Ubuntu) ansible.builtin.set_fact: diff --git a/ansible/roles/contributor/tasks/actionlint.yml b/ansible/roles/contributor/tasks/actionlint.yml index 8f5f0a6..2e30fd6 100644 --- a/ansible/roles/contributor/tasks/actionlint.yml +++ b/ansible/roles/contributor/tasks/actionlint.yml @@ -26,18 +26,19 @@ - name: Install actionlint (Linux) when: ansible_facts['distribution'] in ['Fedora', 'Ubuntu'] block: - - name: Get latest actionlint version from GitHub API - ansible.builtin.uri: - url: https://api.github.com/repos/rhysd/actionlint/releases/latest - return_content: true - headers: "{{ hyperi_github_headers }}" - register: contributor_actionlint_release - check_mode: false + - name: Resolve the actionlint release + ansible.builtin.include_role: + name: github_release + # The include is already gated by this file's tags, and its tasks carry none. + apply: + tags: ['always'] + vars: + github_release_repo: rhysd/actionlint # Tags carry a leading `v`; the asset name and `-version` output drop it. - name: Set actionlint version fact ansible.builtin.set_fact: - contributor_actionlint_version: "{{ contributor_actionlint_release.json.tag_name | regex_replace('^v', '') }}" + contributor_actionlint_version: "{{ github_release_doc.tag_name | regex_replace('^v', '') }}" - name: Read the installed actionlint version ansible.builtin.command: @@ -52,7 +53,7 @@ - name: Download and extract actionlint ansible.builtin.unarchive: src: >- - https://github.com/rhysd/actionlint/releases/download/{{ contributor_actionlint_release.json.tag_name + https://github.com/rhysd/actionlint/releases/download/{{ github_release_doc.tag_name }}/actionlint_{{ contributor_actionlint_version }}_linux_{{ hyperi_arch_deb }}.tar.gz dest: /usr/local/bin remote_src: true diff --git a/ansible/roles/contributor/tasks/git_scrub.yml b/ansible/roles/contributor/tasks/git_scrub.yml index ef689fe..29321fb 100644 --- a/ansible/roles/contributor/tasks/git_scrub.yml +++ b/ansible/roles/contributor/tasks/git_scrub.yml @@ -74,17 +74,18 @@ - name: Install git-scrub (re-fetched GitHub release, Tier 3, Linux) when: ansible_facts['distribution'] in ['Ubuntu', 'Fedora'] block: - - name: Get latest git-scrub version from GitHub API - ansible.builtin.uri: - url: https://api.github.com/repos/hyperi-io/git-scrub/releases/latest - return_content: true - headers: "{{ hyperi_github_headers }}" - register: contributor_git_scrub_release - check_mode: false + - name: Resolve the git-scrub release + ansible.builtin.include_role: + name: github_release + # The include is already gated by this file's tags, and its tasks carry none. + apply: + tags: ['always'] + vars: + github_release_repo: hyperi-io/git-scrub - name: Resolve the git-scrub tag ansible.builtin.set_fact: - contributor_git_scrub_ref: "{{ contributor_git_scrub_release.json.tag_name }}" + contributor_git_scrub_ref: "{{ github_release_doc.tag_name }}" # hyperi_arch_deb already spells the architecture the way this release does. - name: Build the git-scrub asset name diff --git a/ansible/roles/contributor/tasks/gitleaks.yml b/ansible/roles/contributor/tasks/gitleaks.yml index 5ac7b13..4865244 100644 --- a/ansible/roles/contributor/tasks/gitleaks.yml +++ b/ansible/roles/contributor/tasks/gitleaks.yml @@ -60,17 +60,18 @@ - name: Install Gitleaks (Ubuntu) block: - - name: Get latest Gitleaks version - ansible.builtin.uri: - url: https://api.github.com/repos/gitleaks/gitleaks/releases/latest - return_content: true - headers: "{{ hyperi_github_headers }}" - register: contributor_gitleaks_latest - check_mode: false + - name: Resolve the Gitleaks release + ansible.builtin.include_role: + name: github_release + # The include is already gated by this file's tags, and its tasks carry none. + apply: + tags: ['always'] + vars: + github_release_repo: gitleaks/gitleaks - name: Resolve the Gitleaks tag ansible.builtin.set_fact: - contributor_gitleaks_ref: "{{ contributor_gitleaks_latest.json.tag_name }}" + contributor_gitleaks_ref: "{{ github_release_doc.tag_name }}" # gitleaks calls amd64 "x64"; arm64 keeps its own name. - name: Map the architecture to the Gitleaks asset token diff --git a/ansible/roles/contributor/tasks/hadolint.yml b/ansible/roles/contributor/tasks/hadolint.yml index bb2a971..614216f 100644 --- a/ansible/roles/contributor/tasks/hadolint.yml +++ b/ansible/roles/contributor/tasks/hadolint.yml @@ -1,9 +1,10 @@ --- # hadolint - Dockerfile linter (blocking gate for `hyperi-ci check`). # -# Fedora ships it in dnf (OS-swept). Ubuntu has no distro/repo/snap channel, so -# it stays a re-fetched GitHub release (Tier 3 -- hyperi-update pulls the latest -# on each run). brew on macOS. +# Linux takes upstream's GitHub release binary (Tier 3 -- hyperi-update +# re-fetches it). Ubuntu has no distro/repo/snap channel, and Fedora's dnf +# package trails upstream by several minors, so both distros share this path. +# brew on macOS. # # hadolint's release assets use x86_64/arm64, NOT amd64/aarch64, so neither # shared arch token fits -- map explicitly like act does. @@ -21,49 +22,27 @@ when: ansible_facts['distribution'] == 'MacOSX' # ============================================================================ -# FEDORA - dnf package +# LINUX - re-fetched GitHub binary (Tier 3) # ============================================================================ -- name: Install hadolint (Fedora) - block: - - name: Install hadolint via dnf (Fedora) - ansible.builtin.dnf: - name: hadolint - state: present - - rescue: - - name: Record that hadolint did not install (Fedora) - # noqa: var-naming[no-role-prefix] -- deploy_warnings is the shared - # accumulator reported by playbooks/main.yml post_tasks. - ansible.builtin.set_fact: - deploy_warnings: >- - {{ deploy_warnings | default([]) - + ['hadolint: ' ~ (ansible_failed_result.msg | default('install failed'))] }} - - when: ansible_facts['distribution'] == 'Fedora' - -# ============================================================================ -# UBUNTU - re-fetched GitHub binary (Tier 3) -# ============================================================================ - -- name: Install hadolint (Ubuntu) +- name: Install hadolint (Linux) block: - name: Determine the hadolint asset architecture ansible.builtin.set_fact: contributor_hadolint_arch: "{{ 'arm64' if hyperi_arch_deb == 'arm64' else 'x86_64' }}" - # Same shape as gitleaks.yml in this role. - - name: Get latest hadolint version - ansible.builtin.uri: - url: https://api.github.com/repos/hadolint/hadolint/releases/latest - return_content: true - headers: "{{ hyperi_github_headers }}" - register: contributor_hadolint_latest - check_mode: false + - name: Resolve the hadolint release + ansible.builtin.include_role: + name: github_release + # The include is already gated by this file's tags, and its tasks carry none. + apply: + tags: ['always'] + vars: + github_release_repo: hadolint/hadolint - name: Resolve the hadolint tag ansible.builtin.set_fact: - contributor_hadolint_ref: "{{ contributor_hadolint_latest.json.tag_name }}" + contributor_hadolint_ref: "{{ github_release_doc.tag_name }}" contributor_hadolint_asset: "hadolint-linux-{{ contributor_hadolint_arch }}" - name: Fetch the hadolint release checksums @@ -84,7 +63,7 @@ # Without a checksum get_url keeps whatever already sits at dest, so a new # release never landed. The digest comes from the same release, so it detects # a change and a truncated download; it is not a provenance check. - - name: Download hadolint binary (Ubuntu) + - name: Download hadolint binary (Linux) ansible.builtin.get_url: url: "https://github.com/hadolint/hadolint/releases/download/{{ contributor_hadolint_ref }}/{{ contributor_hadolint_asset }}" dest: /usr/local/bin/hadolint @@ -93,12 +72,38 @@ group: root mode: '0755' + # The dnf package older revisions installed. Inside the block, so it goes + # only once the binary above is in place to replace it. + - name: Check for the superseded hadolint package (Fedora) + ansible.builtin.command: + argv: [rpm, -q, hadolint] + register: contributor_hadolint_rpm + changed_when: false + failed_when: false + check_mode: false + when: ansible_facts['distribution'] == 'Fedora' + + - name: Remove the superseded hadolint package (Fedora) + ansible.builtin.include_role: + name: system_cleanup + tasks_from: purge_packages.yml + # The include is already gated by this file's tags, and its tasks carry none. + apply: + tags: ['always'] + vars: + system_cleanup_purge_packages: [hadolint] + system_cleanup_purge_reason: the hadolint package superseded by the release binary + when: + - ansible_facts['distribution'] == 'Fedora' + - contributor_hadolint_rpm.rc == 0 + rescue: - - name: Record that hadolint did not install (Ubuntu) - # noqa: var-naming[no-role-prefix] -- shared accumulator, see note above. + - name: Record that hadolint did not install (Linux) + # noqa: var-naming[no-role-prefix] -- deploy_warnings is the shared + # accumulator reported by playbooks/main.yml post_tasks. ansible.builtin.set_fact: deploy_warnings: >- {{ deploy_warnings | default([]) + ['hadolint: ' ~ (ansible_failed_result.msg | default('download failed'))] }} - when: ansible_facts['distribution'] == 'Ubuntu' + when: ansible_facts['distribution'] in ['Fedora', 'Ubuntu'] diff --git a/ansible/roles/contributor/tasks/hyperi_ci.yml b/ansible/roles/contributor/tasks/hyperi_ci.yml index fd9d52d..075fa24 100644 --- a/ansible/roles/contributor/tasks/hyperi_ci.yml +++ b/ansible/roles/contributor/tasks/hyperi_ci.yml @@ -169,17 +169,18 @@ when: ansible_facts['distribution'] in ['Ubuntu', 'Fedora'] block: # Same shape as kubeconform in the infrastructure role. - - name: Get latest osv-scanner version - ansible.builtin.uri: - url: https://api.github.com/repos/google/osv-scanner/releases/latest - return_content: true - headers: "{{ hyperi_github_headers }}" - register: contributor_osv_latest - check_mode: false + - name: Resolve the osv-scanner release + ansible.builtin.include_role: + name: github_release + # The include is already gated by this file's tags, and its tasks carry none. + apply: + tags: ['always'] + vars: + github_release_repo: google/osv-scanner - name: Resolve the osv-scanner tag ansible.builtin.set_fact: - contributor_osv_ref: "{{ contributor_osv_latest.json.tag_name }}" + contributor_osv_ref: "{{ github_release_doc.tag_name }}" contributor_osv_asset: "osv-scanner_linux_{{ hyperi_arch_deb }}" - name: Fetch the osv-scanner release checksums diff --git a/ansible/roles/developer-go/tasks/go.yml b/ansible/roles/developer-go/tasks/go.yml index 799bd51..8182a1b 100644 --- a/ansible/roles/developer-go/tasks/go.yml +++ b/ansible/roles/developer-go/tasks/go.yml @@ -218,17 +218,18 @@ - name: Install golangci-lint (Linux) block: - - name: Get latest golangci-lint version - ansible.builtin.uri: - url: https://api.github.com/repos/golangci/golangci-lint/releases/latest - return_content: true - headers: "{{ hyperi_github_headers }}" - register: golangci_latest - check_mode: false + - name: Resolve the golangci-lint release + ansible.builtin.include_role: + name: github_release + # The include is already gated by this file's tags, and its tasks carry none. + apply: + tags: ['always'] + vars: + github_release_repo: golangci/golangci-lint - name: Resolve the golangci-lint tag ansible.builtin.set_fact: - golangci_ref: "{{ golangci_latest.json.tag_name }}" + golangci_ref: "{{ github_release_doc.tag_name }}" - name: Download and extract golangci-lint ansible.builtin.unarchive: diff --git a/ansible/roles/developer-rust/files/hyperi-rust-setup b/ansible/roles/developer-rust/files/hyperi-rust-setup index b1ce311..688e9e8 100644 --- a/ansible/roles/developer-rust/files/hyperi-rust-setup +++ b/ansible/roles/developer-rust/files/hyperi-rust-setup @@ -110,8 +110,9 @@ MACOS_PACKAGES = ("sccache",) # mold is ELF-only; it cannot link Mach-O. # sccache is NOT in LINUX_PACKAGES: Ubuntu ships 0.13.0 against an upstream on # 0.17.x, and a wrapper four versions behind is what a developer hits when a # hand-run `sccache --show-stats` fails against the server their build started. -# It comes from the project's own release instead, latest on every run, with the -# digest checked against the .sha256 upstream publishes beside every asset. +# It comes from the project's own release instead, the newest at least +# RELEASE_MIN_AGE_DAYS old on every run, with the digest checked against the +# .sha256 upstream publishes beside every asset. # crates.io stays out of the global rustc-wrapper path, which is the objection # that put this on the distro in the first place. # @@ -119,6 +120,11 @@ MACOS_PACKAGES = ("sccache",) # mold is ELF-only; it cannot link Mach-O. SCCACHE_REPO = "mozilla/sccache" SCCACHE_BIN = Path("/usr/local/bin/sccache") +# A release must be this many days old before it is installed, so a compromised +# or broken one has time to be pulled. The playbook passes its own +# hyperi_release_min_age_days; 7 applies when the script is run by hand. +RELEASE_MIN_AGE_DAYS = int(os.environ.get("HYPERI_RELEASE_MIN_AGE_DAYS") or 7) + # The only tables this script owns; every other table in an existing # config.toml is carried across verbatim by split_toml_sections(). MANAGED_TABLES = frozenset({"build", "target"}) @@ -351,6 +357,23 @@ def _github_request(url: str) -> urllib.request.Request: return request +def _newest_release_past_cooldown(releases: list[dict], min_age_days: int) -> str | None: + """The highest-versioned stable release published at least min_age_days ago.""" + cutoff = datetime.now(timezone.utc).timestamp() - min_age_days * 86400 + ready = [ + r["tag_name"] + for r in releases + if not r.get("draft") + and not r.get("prerelease") + and r.get("published_at") + and re.fullmatch(r"v?\d+(\.\d+)+", r.get("tag_name", "")) + and datetime.fromisoformat(r["published_at"].replace("Z", "+00:00")).timestamp() <= cutoff + ] + if not ready: + return None + return max(ready, key=lambda tag: tuple(int(n) for n in re.findall(r"\d+", tag))) + + def _installed_sccache_version() -> str | None: """The version of the sccache this script manages, or None if absent.""" if not SCCACHE_BIN.is_file(): @@ -363,23 +386,29 @@ def _installed_sccache_version() -> str | None: def install_sccache_linux(dry_run: bool, rep: Reporter) -> None: - """Put the latest upstream sccache release at SCCACHE_BIN. + """Put the newest upstream sccache release past the cooldown at SCCACHE_BIN. - Re-runs are a no-op once the installed version matches the latest tag, so - the same call both installs and upgrades. Between converges hyperi-update - refreshes the same binary from the same release assets. + Re-runs are a no-op once the installed version matches that tag, so the + same call both installs and upgrades. Between converges hyperi-update + refreshes the same binary from the same release assets. With no release old + enough, the installed copy is left alone. """ target = _sccache_target() if target is None: rep.warn(f"sccache: no upstream build for {platform.machine()} -- skipping") return - url = f"https://api.github.com/repos/{SCCACHE_REPO}/releases/latest" + url = f"https://api.github.com/repos/{SCCACHE_REPO}/releases?per_page=30" try: with urllib.request.urlopen(_github_request(url), timeout=60) as response: - latest = json.load(response)["tag_name"] - except (OSError, ValueError, KeyError) as exc: - rep.warn(f"sccache: could not read the latest release ({exc}) -- leaving it alone") + latest = _newest_release_past_cooldown(json.load(response), RELEASE_MIN_AGE_DAYS) + except (OSError, ValueError, KeyError, TypeError) as exc: + rep.warn(f"sccache: could not read the releases ({exc}) -- leaving it alone") + return + if latest is None: + rep.warn( + f"sccache: no release is at least {RELEASE_MIN_AGE_DAYS} days old -- leaving it alone" + ) return current = _installed_sccache_version() diff --git a/ansible/roles/developer-rust/tasks/rust.yml b/ansible/roles/developer-rust/tasks/rust.yml index e1a9828..82578b0 100644 --- a/ansible/roles/developer-rust/tasks/rust.yml +++ b/ansible/roles/developer-rust/tasks/rust.yml @@ -802,9 +802,10 @@ {{ '' if rust_cache_central_build_dir else '--no-build-dir' }} become: "{{ ansible_facts['distribution'] != 'MacOSX' }}" become_user: "{{ actual_user if ansible_facts['distribution'] != 'MacOSX' else omit }}" - # The script looks up the latest sccache release itself, so it needs the token - # too -- it is the one GitHub caller the uri module's headers cannot cover. - environment: "{{ cargo_env | combine(hyperi_github_env) }}" + # The script looks up the sccache release itself, so it needs the token and + # the release-age cooldown too -- it is the one GitHub caller the + # github_release role cannot cover. + environment: "{{ cargo_env | combine(hyperi_github_env) | combine(hyperi_release_env) }}" register: developer_rust_setup changed_when: "'CHANGED' in developer_rust_setup.stdout" # An optional build accelerator must never abort the run. diff --git a/ansible/roles/developer/files/update/hyperi-update-linux.sh b/ansible/roles/developer/files/update/hyperi-update-linux.sh index f350dc5..1cc5c1f 100644 --- a/ansible/roles/developer/files/update/hyperi-update-linux.sh +++ b/ansible/roles/developer/files/update/hyperi-update-linux.sh @@ -28,8 +28,9 @@ # the summary without aborting the rest. At the end, if the system needs it, you # get a reboot prompt (default: No). # -# Run with: hyperi-update (confirms, then prompts once for sudo) -# hyperi-update --yes (no confirmation — for scripts/Ansible) +# Run with: hyperi-update (confirms, then prompts once for sudo) +# hyperi-update --yes (no confirmation — for scripts/Ansible) +# hyperi-update --min-age DAYS (release-age cooldown, default 7) # hyperi-update --help set -uo pipefail @@ -68,6 +69,11 @@ export PATH="$HOME/.local/bin:$CARGO_BIN:/usr/local/go/bin:$HOME/go/bin:$PNPM_HO ASSUME_YES=0 +# A GitHub release binary is refreshed to the newest release at least this many +# days old, so a compromised or broken release has time to be pulled first. The +# roles apply the same rule through hyperi_release_min_age_days. +RELEASE_MIN_AGE="${HYPERI_RELEASE_MIN_AGE_DAYS:-7}" + usage() { cat <&2; usage; exit 2 ;; -esac +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) usage; exit 0 ;; + -y|--yes) ASSUME_YES=1 ;; + --min-age) [[ $# -ge 2 ]] || { printf 'hyperi-update: --min-age needs a number of days\n' >&2; exit 2; } + RELEASE_MIN_AGE="$2"; shift ;; + --min-age=*) RELEASE_MIN_AGE="${1#*=}" ;; + *) printf 'hyperi-update: unknown option %q\n' "$1" >&2; usage; exit 2 ;; + esac + shift +done +if [[ ! "$RELEASE_MIN_AGE" =~ ^[0-9]+$ ]]; then + printf 'hyperi-update: the release age must be a whole number of days, not %q\n' "$RELEASE_MIN_AGE" >&2 + exit 2 +fi # --- pretty output --------------------------------------------------------- if [[ -t 1 ]]; then @@ -118,6 +136,11 @@ run() { # not a single command (multi-step fetch-verify-replace sequences). fail() { printf '%s \xe2\x9c\x97 %s%s\n' "$RED" "$1" "$RESET"; FAILURES+=("$1"); } +# warn