diff --git a/.github/actions/ci-inputs/action.yml b/.github/actions/ci-inputs/action.yml new file mode 100644 index 0000000..664d281 --- /dev/null +++ b/.github/actions/ci-inputs/action.yml @@ -0,0 +1,105 @@ +name: CI inputs +description: >- + Hash the files a job depends on (a list in .github/path-filters.yml plus the + workflow running it) and say whether that exact set already passed, so the + job can be skipped. Needs the repository checked out. After the job's work + succeeds, save the key with .github/actions/ci-passed. + +# Two ways to ask: +# exact — has this set of inputs ever passed? Right for checks: the same +# files give the same result, so a CHANGELOG-only push, a re-push +# of the same code, or a docs-only PR (main's run is visible to +# PRs) is skipped. A cancelled or failed run saves nothing, so its +# inputs run again next time. +# latest — did the most recent pass have this set? Right for publishing: +# a revert to older code must publish again, because the `:main` +# image moved on since. +# Markers are tiny Actions caches. A PR can read its own and main's, never +# another PR's. An evicted marker only means the job runs again. + +inputs: + filter: + description: A list name in .github/path-filters.yml. + required: true + match: + description: exact or latest (see above). + required: false + default: exact + +outputs: + key: + description: The cache key to save with ci-passed once the job succeeds. + value: ${{ steps.hash.outputs.key }} + passed: + description: "'true' when these inputs already passed and the job can skip." + value: ${{ steps.decide.outputs.passed }} + +runs: + using: composite + steps: + - name: Hash ${{ inputs.filter }} inputs + id: hash + shell: bash + env: + FILTER: ${{ inputs.filter }} + MATCH: ${{ inputs.match }} + REPOSITORY: ${{ github.repository }} + WORKFLOW_REF: ${{ github.workflow_ref }} + RUN: ${{ github.run_id }}-${{ github.run_attempt }} + run: | + set -euo pipefail + # owner/repo/.github/workflows/ci.yml@refs/... → .github/workflows/ci.yml + workflow="${WORKFLOW_REF#"${REPOSITORY}/"}" + workflow="${workflow%%@*}" + hash="$(bash scripts/ci-inputs.sh "${FILTER}" "${workflow}")" + case "${MATCH}" in + exact) + prefix="ci-passed-${FILTER}-${hash}" + key="${prefix}" + ;; + latest) + prefix="ci-latest-${FILTER}-" + key="${prefix}${RUN}-${hash}" + ;; + *) + echo "::error::match must be exact or latest (got ${MATCH})." + exit 1 + ;; + esac + echo "hash=${hash}" >> "${GITHUB_OUTPUT}" + echo "prefix=${prefix}" >> "${GITHUB_OUTPUT}" + echo "key=${key}" >> "${GITHUB_OUTPUT}" + echo "${FILTER} inputs (with ${workflow}): ${hash}" + + - name: Look up ${{ inputs.filter }} + id: lookup + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .ci-passed + key: ${{ steps.hash.outputs.key }} + restore-keys: ${{ inputs.match == 'latest' && steps.hash.outputs.prefix || '' }} + lookup-only: true + + - name: Decide + id: decide + shell: bash + env: + FILTER: ${{ inputs.filter }} + MATCH: ${{ inputs.match }} + HASH: ${{ steps.hash.outputs.hash }} + HIT: ${{ steps.lookup.outputs.cache-hit }} + MATCHED: ${{ steps.lookup.outputs.cache-matched-key }} + run: | + set -euo pipefail + passed=false + if [ "${MATCH}" = "exact" ] && [ "${HIT}" = "true" ]; then + passed=true + elif [ "${MATCH}" = "latest" ] && [ -n "${MATCHED}" ] && [ "${MATCHED%-"${HASH}"}" != "${MATCHED}" ]; then + passed=true + fi + echo "passed=${passed}" >> "${GITHUB_OUTPUT}" + if [ "${passed}" = "true" ]; then + echo "These ${FILTER} inputs already passed. Skipping." + else + echo "These ${FILTER} inputs have not passed yet. Running." + fi diff --git a/.github/actions/ci-passed/action.yml b/.github/actions/ci-passed/action.yml new file mode 100644 index 0000000..359ef64 --- /dev/null +++ b/.github/actions/ci-passed/action.yml @@ -0,0 +1,27 @@ +name: CI passed +description: >- + Remember that a job's inputs passed (the key from .github/actions/ci-inputs), + so the next run with the same inputs skips it. Put it last in the job, so it + only runs when everything before it succeeded. + +inputs: + key: + description: The key output of ci-inputs. + required: true + +runs: + using: composite + steps: + - name: Write the marker + shell: bash + env: + KEY: ${{ inputs.key }} + run: | + mkdir -p .ci-passed + printf '%s\n' "${KEY}" > .ci-passed/key + + # The path must be the same as in ci-inputs, or the lookup never matches. + - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .ci-passed + key: ${{ inputs.key }} diff --git a/.github/path-filters.yml b/.github/path-filters.yml index fadf9e9..f655a8c 100644 --- a/.github/path-filters.yml +++ b/.github/path-filters.yml @@ -1,27 +1,35 @@ -# Which expensive CI/CD jobs run for a change. -# Used by dorny/paths-filter in ci.yml and cd.yml. +# What each CI/CD job depends on. A job runs only when the files its list +# matches are not a set that already passed (scripts/ci-inputs.sh hashes them, +# .github/actions/ci-inputs looks the hash up). So a push that only adds a +# CHANGELOG entry, or a docs-only PR, does not run the app checks again. # Keep the pipeline diagrams in docs/architecture.md and docs/runbook.md in sync. # -# check → just check + just build -# image → Docker smoke (PR) / GHCR + Sentry (main) -# src/** (including tests) is in both: a test-only change still checks and publishes. +# Patterns are git glob pathspecs: `*` stays inside one directory, `**` +# crosses them, a leading `!` excludes. Every hash also covers this file, +# scripts/ci-inputs.sh, .github/actions/ and the workflow that runs the job, +# so don't list those here. List what the job reads, nothing more. +# `just ci-inputs ` prints what a list matches. +# +# check → just check + just build (ci.yml) +# image → Docker smoke (ci.yml) and GHCR + Sentry (cd.yml) +# docs → Biome on docs/ + the VitePress build (docs.yml) +# terraform → just tf-validate and the env plans (terraform.yml) check: - "src/**" - "scripts/**" - "content/**" - "config/**" - - "package.json" + # vitest.config.ts; Biome lints the rest. + - "*.ts" + # package.json, biome.json, tsconfig*.json + - "*.json" - "pnpm-lock.yaml" - "pnpm-workspace.yaml" - - "tsconfig.json" - - "tsconfig.*.json" - - "biome.json" - - "vitest.config.*" - "justfile" - ".nvmrc" - - ".github/workflows/ci.yml" - - ".github/path-filters.yml" + # Biome skips what git ignores. + - ".gitignore" # scripts/secrets-manifest.test.ts checks these against infra/secrets.manifest.yaml (ADR 0015). - "infra/secrets.manifest.yaml" - "infra/modules/pixel/*.tf" @@ -30,18 +38,42 @@ check: - "docs/runbook.md" - "docs/architecture.md" - "docs/infra.md" + # scripts/ci-inputs.test.ts checks these against the lists below (ci.yml is + # in every check hash already). + - "Dockerfile" + - ".github/workflows/cd.yml" + - ".github/workflows/docs.yml" + - ".github/workflows/terraform.yml" +# What the Dockerfile copies. Tests are copied but never compiled into the image. image: - "src/**" + - "!src/**/*.test.ts" + - "!src/testing/**" - "content/**" - "package.json" - "pnpm-lock.yaml" - "pnpm-workspace.yaml" - "tsconfig.json" - - "tsconfig.*.json" + - "tsconfig.build.json" - "Dockerfile" - ".dockerignore" + +# The site, plus the root package.json (its version is in the nav, and Biome +# comes from the root install). docs.yml's paths trigger must list these too. +docs: + - "docs/**" + - "package.json" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "biome.json" + - ".gitignore" + - ".nvmrc" + - "justfile" + +# Markdown under infra/ does not change the stack. terraform.yml's paths +# trigger must list these too. +terraform: + - "infra/**" + - "!infra/**/*.md" - "justfile" - - ".github/workflows/ci.yml" - - ".github/workflows/cd.yml" - - ".github/path-filters.yml" diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index b8a1aa8..e19b3d3 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,10 +1,12 @@ name: CD -# Merges to `main` publish GHCR and upload Sentry maps when an image path -# changed (dorny/paths-filter + .github/path-filters.yml). They do **not** -# roll pixel-prod (ADR 0013): that is the Release workflow. Docs / changelog / -# infra-only merges skip this pipeline. Azure `dev` stays a local build -# (`just deploy-dev`). Never copy pixel-dev / Mini Discord tokens into prod. +# Merges to `main` publish GHCR and upload Sentry maps when the image inputs +# (the `image` list in .github/path-filters.yml plus this file) differ from +# the last run that published. They do **not** roll pixel-prod (ADR 0013): +# that is the Release workflow. Docs / changelog / test / infra-only merges +# skip this pipeline. A run that didn't finish saves nothing, so the next one +# publishes. Azure `dev` stays a local build (`just deploy-dev`). +# Never copy pixel-dev / Mini Discord tokens into prod. # This workflow does not apply Terraform. on: @@ -17,11 +19,10 @@ on: permissions: contents: read -# Queue, never cancel: cancelling a run mid-publish (say, for a docs-only -# merge right after it) left that commit with no GHCR image and no Sentry -# release. GitHub still keeps only one pending run and replaces it with a -# newer push, so the path filter below compares against the last green run, -# not just the previous push, and a replaced run's changes are not lost. +# Queue, never cancel, so a publish is never cut short halfway. GitHub still +# keeps only one pending run and replaces it with a newer push; that is safe +# because the `latest` lookup below compares against the last run that +# published, not the previous push, so a replaced run's changes are not lost. concurrency: group: cd-${{ github.ref }} cancel-in-progress: false @@ -34,46 +35,24 @@ env: jobs: changes: - name: Path filter + name: What changed runs-on: ubuntu-latest timeout-minutes: 2 - # actions: read lists earlier CD runs to find the last green one. - permissions: - contents: read - actions: read outputs: - image: ${{ steps.filter.outputs.image }} + image: ${{ steps.image.outputs.passed != 'true' }} + image-key: ${{ steps.image.outputs.key }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - fetch-depth: 0 - # A run is green only if it published whatever needed publishing, so - # everything after its commit is still unpublished. Falls back to the - # previous push when there is no green run, or it is not in main's history. - - name: Last green CD commit - id: base - env: - GH_TOKEN: ${{ github.token }} - REPO: ${{ github.repository }} - BEFORE: ${{ github.event.before }} - run: | - set -euo pipefail - last="$(gh api "repos/${REPO}/actions/workflows/cd.yml/runs?branch=main&event=push&status=success&per_page=1" --jq '.workflow_runs[0].head_sha // empty' || true)" - if [ -n "${last}" ] && [ "${last}" != "${GITHUB_SHA}" ] && git merge-base --is-ancestor "${last}" HEAD; then - base="${last}" - else - base="${BEFORE}" - fi - echo "sha=${base}" >> "${GITHUB_OUTPUT}" - echo "Image paths are compared against ${base}." - - - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 - id: filter + # latest, not exact: after a revert the older code must publish again, + # because `:main` has moved on since it was last published. + - uses: ./.github/actions/ci-inputs + id: image with: - base: ${{ steps.base.outputs.sha }} - filters: .github/path-filters.yml + filter: image + match: latest publish: name: Publish GHCR @@ -161,7 +140,7 @@ jobs: sentry-release: name: Sentry release (source maps) - needs: publish + needs: [changes, publish] runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -222,3 +201,8 @@ jobs: set_commits: auto ignore_missing: true disable_telemetry: true + + # Last, so a run counts as published only with its image and its maps. + - uses: ./.github/actions/ci-passed + with: + key: ${{ needs.changes.outputs.image-key }} diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index d712128..5051519 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -20,25 +20,21 @@ jobs: check: name: Require CHANGELOG.md runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false - - uses: pnpm/action-setup@d9184bf108216479bc5a137cc391f4d7b14c870b # v6.1.0 - + # No pnpm install: the gate only uses Node built-ins, and Node runs the + # TypeScript itself. - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version-file: .nvmrc - cache: pnpm - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 - - name: Install dependencies - run: pnpm install --frozen-lockfile - - name: Require a changelog entry unless skip-changelog env: SKIP_CHANGELOG: ${{ contains(github.event.pull_request.labels.*.name, 'skip-changelog') }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9553c94..9f2b1ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,18 +1,19 @@ name: CI -# just check and the Docker smoke build each run only when their inputs -# changed (dorny/paths-filter + .github/path-filters.yml). Jobs always start -# so required checks stay green when that job is not needed. +# just check and the Docker smoke build each run only when their inputs (a +# list in .github/path-filters.yml plus this file) are not a set that already +# passed. A push that only adds a CHANGELOG entry, a re-push of the same code, +# or a docs-only PR skips them: GitHub shows a skipped job as passed. Pushes +# to main run them too, so PRs can reuse main's results. on: pull_request: push: branches: [main] -# Least privilege: CI only needs to read the repo and (on PRs) the changed-files API. +# Least privilege: CI only reads the repo (the input markers are Actions caches). permissions: contents: read - pull-requests: read concurrency: group: ci-${{ github.ref }} @@ -23,92 +24,84 @@ concurrency: jobs: changes: - name: Path filter + name: What changed runs-on: ubuntu-latest timeout-minutes: 2 outputs: - check: ${{ steps.filter.outputs.check }} - image: ${{ steps.filter.outputs.image }} + check: ${{ steps.check.outputs.passed != 'true' }} + check-key: ${{ steps.check.outputs.key }} + image: ${{ steps.image.outputs.passed != 'true' }} + image-key: ${{ steps.image.outputs.key }} steps: - # Needed on push to main (git against the previous commit). Harmless on PRs - # where dorny uses the pull-request files API. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - fetch-depth: 0 - - uses: dorny/paths-filter@ceb8a2b8f2d89434be7ff52d3de7ec3738c5cc9d # v4.0.3 - id: filter + - uses: ./.github/actions/ci-inputs + id: check with: - filters: .github/path-filters.yml + filter: check + + - uses: ./.github/actions/ci-inputs + id: image + with: + filter: image check: name: Lint, typecheck, test, build needs: changes + if: needs.changes.outputs.check == 'true' runs-on: ubuntu-latest timeout-minutes: 10 steps: - - name: Skip just check - if: needs.changes.outputs.check != 'true' - run: echo "Skipping just check; no check path changed." - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - if: needs.changes.outputs.check == 'true' with: persist-credentials: false - uses: pnpm/action-setup@d9184bf108216479bc5a137cc391f4d7b14c870b # v6.1.0 - if: needs.changes.outputs.check == 'true' - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 - if: needs.changes.outputs.check == 'true' with: node-version-file: .nvmrc cache: pnpm - uses: extractions/setup-just@53165ef7e734c5c07cb06b3c8e7b647c5aa16db3 # v4.0.0 - if: needs.changes.outputs.check == 'true' - name: Install dependencies - if: needs.changes.outputs.check == 'true' run: pnpm install --frozen-lockfile - name: Lint, typecheck, test with coverage thresholds - if: needs.changes.outputs.check == 'true' run: just check - name: Build - if: needs.changes.outputs.check == 'true' run: just build - name: Upload coverage report - if: ${{ !cancelled() && needs.changes.outputs.check == 'true' }} + if: ${{ !cancelled() }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: coverage path: coverage/ retention-days: 14 + - uses: ./.github/actions/ci-passed + with: + key: ${{ needs.changes.outputs.check-key }} + docker: name: Docker image builds needs: changes + if: needs.changes.outputs.image == 'true' runs-on: ubuntu-latest timeout-minutes: 15 steps: - - name: Skip Docker image - if: needs.changes.outputs.image != 'true' - run: echo "Skipping Docker; no image path changed." - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - if: needs.changes.outputs.image == 'true' with: persist-credentials: false - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 - if: needs.changes.outputs.image == 'true' - name: Build image (not pushed) - if: needs.changes.outputs.image == 'true' uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 with: context: . @@ -120,3 +113,7 @@ jobs: PIXEL_GIT_BRANCH=${{ github.head_ref || github.ref_name }} cache-from: type=gha cache-to: type=gha,mode=max + + - uses: ./.github/actions/ci-passed + with: + key: ${{ needs.changes.outputs.image-key }} diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 01443b2..f651f72 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -4,13 +4,27 @@ name: Docs site # build it. Publishing to GitHub Pages is the Release workflow (ADR 0013). This # workflow can still publish by hand (workflow_dispatch) if a maintainer needs # to republish without cutting a version. App CI does not run for docs-only -# changes; lint lives here instead. +# changes; lint lives here instead. On a PR the build is skipped when the +# `docs` inputs (.github/path-filters.yml) already passed, such as after a +# push that only adds a CHANGELOG entry. on: pull_request: + # The `docs` list in .github/path-filters.yml, plus what every input hash + # covers (scripts/ci-inputs.test.ts keeps these in step). paths: - "docs/**" + - "package.json" + - "pnpm-lock.yaml" + - "pnpm-workspace.yaml" + - "biome.json" + - ".gitignore" + - ".nvmrc" + - "justfile" - ".github/workflows/docs.yml" + - ".github/path-filters.yml" + - ".github/actions/**" + - "scripts/ci-inputs.sh" workflow_dispatch: # Least privilege: building only needs to read the repo. The deploy job asks for @@ -27,8 +41,28 @@ concurrency: # moved or compromised tag can't change what runs here. jobs: + changes: + name: What changed + runs-on: ubuntu-latest + timeout-minutes: 2 + outputs: + docs: ${{ steps.docs.outputs.passed != 'true' }} + docs-key: ${{ steps.docs.outputs.key }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/ci-inputs + id: docs + with: + filter: docs + build: name: Build the site + needs: changes + # A publish always builds. + if: github.event_name != 'pull_request' || needs.changes.outputs.docs == 'true' runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -76,6 +110,10 @@ jobs: with: path: docs/.vitepress/dist + - uses: ./.github/actions/ci-passed + with: + key: ${{ needs.changes.outputs.docs-key }} + deploy: name: Publish to GitHub Pages needs: build diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index daec487..69e078d 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -1,19 +1,31 @@ name: Terraform # Only when the stack, this workflow, or the just recipe could change the result. -# A feature PR that never touches those should not download Terraform. +# A feature PR that never touches those should not download Terraform. Inside +# a run, everything is skipped when the `terraform` inputs +# (.github/path-filters.yml) already passed, and the Azure plans run only when +# Terraform files under infra/ changed (markdown does not count). Skipped plan +# jobs never start, so they never wait on an Environment. on: pull_request: + # The `terraform` list in .github/path-filters.yml, plus what every input + # hash covers (scripts/ci-inputs.test.ts keeps these in step). paths: - "infra/**" - "justfile" - ".github/workflows/terraform.yml" + - ".github/path-filters.yml" + - ".github/actions/**" + - "scripts/ci-inputs.sh" push: branches: [main] paths: - "infra/**" - "justfile" - ".github/workflows/terraform.yml" + - ".github/path-filters.yml" + - ".github/actions/**" + - "scripts/ci-inputs.sh" permissions: contents: read @@ -23,8 +35,48 @@ concurrency: cancel-in-progress: true jobs: + changes: + name: What changed + runs-on: ubuntu-latest + timeout-minutes: 2 + outputs: + terraform: ${{ steps.terraform.outputs.passed != 'true' }} + terraform-key: ${{ steps.terraform.outputs.key }} + stack: ${{ steps.stack.outputs.changed }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + fetch-depth: 0 + + - uses: ./.github/actions/ci-inputs + id: terraform + with: + filter: terraform + + - name: Did Terraform files change? + id: stack + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + run: | + set -euo pipefail + if [ -z "${BASE_SHA}" ] || [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + exit 0 + fi + # Markdown under infra/ does not change the stack. After prod apply, + # GitHub is not Key Vault Secrets User (ADR 0012); a secret refresh 403s. + if git diff --name-only "${BASE_SHA}" HEAD | grep -E '^infra/' | grep -vE '\.md$' | grep -q .; then + echo "changed=true" >> "${GITHUB_OUTPUT}" + else + echo "changed=false" >> "${GITHUB_OUTPUT}" + echo "No Terraform files in infra/. Skipping the Azure plans (they never apply)." + fi + validate: name: Validate Terraform + needs: changes + if: needs.changes.outputs.terraform == 'true' runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -44,7 +96,8 @@ jobs: plan-dev: name: Plan envs/dev - needs: validate + needs: [changes, validate] + if: needs.changes.outputs.stack == 'true' runs-on: ubuntu-latest timeout-minutes: 10 environment: dev @@ -55,35 +108,13 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - fetch-depth: 0 - - - name: Skip Azure plan unless Terraform files changed - id: stack - env: - BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - run: | - set -euo pipefail - if [ -z "${BASE_SHA}" ] || [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then - echo "changed=true" >> "${GITHUB_OUTPUT}" - exit 0 - fi - # Markdown under infra/ does not change the stack. After prod apply, - # GitHub is not Key Vault Secrets User (ADR 0012); a secret refresh 403s. - if git diff --name-only "${BASE_SHA}" HEAD | grep -E '^infra/' | grep -vE '\.md$' | grep -q .; then - echo "changed=true" >> "${GITHUB_OUTPUT}" - else - echo "changed=false" >> "${GITHUB_OUTPUT}" - echo "No Terraform files in infra/. Skipping Azure plan (does not apply)." - fi - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - if: steps.stack.outputs.changed == 'true' with: terraform_version: 1.16.5 terraform_wrapper: false - name: Require ARM_* on GitHub Environment dev - if: steps.stack.outputs.changed == 'true' env: ARM_CLIENT_ID: ${{ vars.ARM_CLIENT_ID }} ARM_TENANT_ID: ${{ vars.ARM_TENANT_ID }} @@ -108,14 +139,12 @@ jobs: fi - uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0 - if: steps.stack.outputs.changed == 'true' with: client-id: ${{ vars.ARM_CLIENT_ID }} tenant-id: ${{ vars.ARM_TENANT_ID }} subscription-id: ${{ vars.ARM_SUBSCRIPTION_ID }} - name: Plan infra/envs/dev - if: steps.stack.outputs.changed == 'true' working-directory: infra/envs/dev run: | terraform init -backend-config=backend.azurerm.example.hcl -input=false @@ -126,7 +155,8 @@ jobs: # Missing ARM_* fails closed before azure/login. This job does not apply. plan-prod: name: Plan envs/prod - needs: validate + needs: [changes, validate] + if: needs.changes.outputs.stack == 'true' runs-on: ubuntu-latest timeout-minutes: 10 environment: prod @@ -137,35 +167,13 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - fetch-depth: 0 - - - name: Skip Azure plan unless Terraform files changed - id: stack - env: - BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - run: | - set -euo pipefail - if [ -z "${BASE_SHA}" ] || [ "${BASE_SHA}" = "0000000000000000000000000000000000000000" ]; then - echo "changed=true" >> "${GITHUB_OUTPUT}" - exit 0 - fi - # Markdown under infra/ does not change the stack. The plan below skips - # refreshing (GitHub can't read prod's Key Vault secrets, ADR 0012). - if git diff --name-only "${BASE_SHA}" HEAD | grep -E '^infra/' | grep -vE '\.md$' | grep -q .; then - echo "changed=true" >> "${GITHUB_OUTPUT}" - else - echo "changed=false" >> "${GITHUB_OUTPUT}" - echo "No Terraform files in infra/. Skipping Azure plan (does not apply)." - fi - uses: hashicorp/setup-terraform@dfe3c3f87815947d99a8997f908cb6525fc44e9e # v4.0.1 - if: steps.stack.outputs.changed == 'true' with: terraform_version: 1.16.5 terraform_wrapper: false - name: Require ARM_* on GitHub Environment prod - if: steps.stack.outputs.changed == 'true' env: ARM_CLIENT_ID: ${{ vars.ARM_CLIENT_ID }} ARM_TENANT_ID: ${{ vars.ARM_TENANT_ID }} @@ -190,7 +198,6 @@ jobs: fi - uses: azure/login@a641126d1b8aa4d1fa005f4f92df94a3a4c4c906 # v3.1.0 - if: steps.stack.outputs.changed == 'true' with: client-id: ${{ vars.ARM_CLIENT_ID }} tenant-id: ${{ vars.ARM_TENANT_ID }} @@ -202,8 +209,27 @@ jobs: # shows those secrets being removed; it never applies. For the real picture, a # maintainer runs `just tf-plan prod`. - name: Plan infra/envs/prod - if: steps.stack.outputs.changed == 'true' working-directory: infra/envs/prod run: | terraform init -backend-config=backend.azurerm.example.hcl -input=false terraform plan -input=false -refresh=false -var-file=terraform.example.tfvars + + # Remembers these inputs only when validate and every plan that had to run + # passed. A skipped plan (no Terraform change) is fine. + passed: + name: Remember Terraform passed + needs: [changes, validate, plan-dev, plan-prod] + if: >- + ${{ !cancelled() && needs.validate.result == 'success' && + contains(fromJSON('["success","skipped"]'), needs.plan-dev.result) && + contains(fromJSON('["success","skipped"]'), needs.plan-prod.result) }} + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/ci-passed + with: + key: ${{ needs.changes.outputs.terraform-key }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b334fa..bfad26b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Until 1.0.0, releases stay on 0.x: `Added` bumps minor, `Fixed`/`Changed` (and t ## [Unreleased] +### Changed + +- CI jobs run only when the files they depend on have not already passed. Each workflow's first job hashes the job's inputs (a list in `.github/path-filters.yml`, plus the workflow running it) and looks for a marker that an earlier run with exactly those files succeeded. So pushing only a CHANGELOG entry, re-pushing the same code, or opening a docs-only PR no longer runs `just check`, the Docker build, the docs site build or Terraform validate again. Skipped jobs no longer start a runner. Test-only changes no longer build or publish an image. CD publishes when the image inputs differ from the last publish, so a CD run cancelled by a newer merge no longer leaves code without a GHCR image. Terraform plan jobs that aren't needed never start, so they never wait on an Environment. The changelog gate no longer runs `pnpm install`. `just ci-inputs ` shows what a list matches; the runbook's "CI and CD" says how to force a rerun. + ### Fixed - `/spacestate`: the Sentry context for a failed call now says why the network failed (`errorCause`, such as `ENOTFOUND`, `ECONNREFUSED` or `UNEXPECTED_REDIRECT`), and shows the address without its query. Sentry was replacing the old redacted address with `[Filtered]`. @@ -21,7 +25,6 @@ Until 1.0.0, releases stay on 0.x: `Added` bumps minor, `Fixed`/`Changed` (and t - Discord: when Discord refused a slow command's acknowledgement (its deferral), the failure is no longer also logged and sent to Sentry as an unhandled rejection with no user or command; it is handled once, by the adapter. A command whose "Checking…" placeholder couldn't be posted (`/status`, `/ha` and others) is no longer reported as a failed command: the handler still doesn't run, and the adapter logs it like any other refused acknowledgement. - Discord: only "already acknowledged" (40060) is logged as `discord.duplicate_handler` (a second Pixel on the token). "Unknown interaction" (10062) means Pixel didn't acknowledge within Discord's 3 seconds, so it is now logged as `discord.slow_ack` with the command and who ran it, and reported to Sentry as a `SlowAcknowledgementError` naming the command. - Sentry: traces no longer carry Discord interaction tokens. Every reply to a command goes to a Discord URL with the interaction token in its path, and the request's span (`url.full`, `url.path`) and `http` breadcrumb kept it, so anyone with access to the Sentry project could post as the bot for 15 minutes after each command. Spans now go through the same scrubbing as errors, breadcrumbs and logs, and the token in `/interactions//` and `/webhooks//` paths is shown as `[redacted]`. The scrubbing also no longer skips an object that appears twice in what is sent, and keeps an error's message and stack (scrubbed) instead of dropping them. -- CD on `main` no longer loses an image: runs queue instead of cancelling each other, and each one publishes when an image path changed since the last green CD run, not just since the previous push. Before, a docs-only merge right after a code merge cancelled that merge's publish and then skipped its own, so the code merge never got a GHCR image or a Sentry release. - The Release workflow runs `just check` on the `main` commit it is about to release, and only tags, publishes and deploys to prod if it passes. CI on `main` can be cancelled or red and never runs on the bump commit, so before this nothing tested what reached prod. - Sentry errors from the Docker image now show the TypeScript source. The image ran Node with `--enable-source-maps`, which rewrote stack frames to `src/*.ts` before Sentry saw them, so the source maps uploaded for `/app/dist` never matched and issues had no source context. Node now runs without it and Sentry applies the uploaded maps. Stack traces in the container logs and log files now show the compiled `dist/*.js` lines. - `/admin doors on|off`: the `home.kind_switched` log now names the admin like every other action (Discord ID, name and handle) and carries the command, so moderators can recognise who switched door control. Before, it had only the Discord ID. It is still logged once per switch. @@ -29,6 +32,7 @@ Until 1.0.0, releases stay on 0.x: `Added` bumps minor, `Fixed`/`Changed` (and t - Mastodon: a response from the instance is now read only up to the 64 KiB limit. A larger body (declared or streamed without a length) is cancelled instead of buffered in full or left open. - Commands with a form (`/schedule`, closing time): opening the form now takes a token from the per-person command rate limit (refused and logged as `command.rate_limited` like any command), and errors reported while the form is being prepared are tied to the person's Discord ID in Sentry, as they are while a command runs. Submitting the form still takes its own token. - `/schedule`: `when` no longer guesses. Four digits after a date are read the same with or without a month name: a time (`14 nov 1900` is 19:00, `14 nov 2030` is 20:30, as `14-11 2030` already was), and a year only beside another time (`14 nov 2030 19:00`) or when no clock reads them (`14 nov 2099`). Months must be English names or their short forms (`sep`, `sept`, `september`), so words like `junk`, `decent` or `mars` are refused instead of read as June, December or March. +- CD on `main` no longer loses an image: runs queue instead of cancelling each other, and each one publishes when the image inputs differ from the last run that published, not just from the previous push. Before, a docs-only merge right after a code merge cancelled that merge's publish and then skipped its own, so the code merge never got a GHCR image or a Sentry release. ## [0.17.0] - 2026-10-15 diff --git a/docs/architecture.md b/docs/architecture.md index cb71e04..bedb8ee 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -481,34 +481,38 @@ There is **no database**. Access, schedules and (planned) account linking are fi - **Terraform layout:** `infra/bootstrap` (state storage, GitHub OIDC, applied) then `infra/modules/pixel`, `infra/envs/dev` (#9) and `infra/envs/prod` (#12). Secret values never go into Terraform state; apply can take them as ephemeral write-only inputs. Subscription `d150e252-e2f0-47fb-8a4a-c3f29e9aebd4`, West Europe. See [ADR 0008](adr/0008-terraform-bootstrap.md), [ADR 0009](adr/0009-container-apps-dev.md) and [ADR 0012](adr/0012-prod-environment.md). CD does **not** apply Terraform. After first create, **dev** apply ignores the Pixel container image; `just deploy-dev` / `az` updates it. Prod apply sets `container_image`. Coding agents must not apply prod (`PIXEL_APPLY_PROD=1` required). - **Home Assistant from Azure:** Tailscale sidecar into the space network is the preferred path (`tailscale_enabled`); `HOME_ASSISTANT_URL` can still be a Nabu Casa URL. Both URL and token or neither. Sidecar is userspace (no TUN). LAN MagicDNS through it is remaining work for #43. - **Cost (rough):** always-on `dev` is about €40–55/month West Europe without Tailscale, plus about €15 with the sidecar. Not a quote. -- **CI (built):** [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) runs on every PR and on pushes to `main`. [`dorny/paths-filter`](https://github.com/dorny/paths-filter) with allowlists in [`.github/path-filters.yml`](../.github/path-filters.yml) decides which expensive jobs run: **check** (`just check` + `just build`) when app/scripts/config/tooling paths change; **Docker smoke** when image inputs change (`src/` including tests, `content/`, Dockerfile, lockfile, …). Jobs always start so required GitHub checks stay green when that job is not needed. [`changelog.yml`](../.github/workflows/changelog.yml) still runs on every PR. [`docs.yml`](../.github/workflows/docs.yml) Biome-lints and builds the site when `docs/` changes. [`terraform.yml`](../.github/workflows/terraform.yml) runs `just tf-validate` only when `infra/`, that workflow, or the `justfile` change, then `terraform plan` on each env via GitHub Environment `dev` / `prod` OIDC **only if `infra/` Terraform files changed** (markdown does not). Missing `ARM_*` **fails closed**. No apply on merge. GitHub is not Key Vault Secrets User, so a secret refresh would 403 (ADR 0012). Actions are pinned to commit SHAs. -- **CD:** `just deploy-dev` for Azure `dev` (local GHCR push + `az` + `just register dev` + Sentry release). Production slash commands: the Release's `register-prod` job (fallback `just register prod`). [`.github/workflows/cd.yml`](../.github/workflows/cd.yml) on `main` only, and only when an **image** path changed since the last green CD run (not just since the previous push): publish with `GITHUB_TOKEN` (`packages: write` on that job) and upload Sentry source maps (fails closed if org/token are missing). CD runs queue and are never cancelled, so a quick docs-only merge cannot cut short the publish before it. **No Azure login and no `pixel-prod` update** on merge. [`.github/workflows/release.yml`](../.github/workflows/release.yml) (`workflow_dispatch` on `main`) runs `just check` on the `main` commit being released (CI on `main` may have been cancelled or red), cuts the version only if that passes, publishes `:vX.Y.Z`, Environment `prod` approval rolls `pixel-prod` (`az`, min=max=1, no Terraform apply), a second approval runs `register-prod` (reads only `discord-token` from `pixel-prod-kv`), and publishes GitHub Pages. See [ADR 0010](adr/0010-ghcr-cd.md), [ADR 0012](adr/0012-prod-environment.md) and [ADR 0013](adr/0013-release-workflow.md). +- **CI (built):** [`.github/workflows/ci.yml`](../.github/workflows/ci.yml) runs on every PR and on pushes to `main`. Each expensive job depends on a list of files in [`.github/path-filters.yml`](../.github/path-filters.yml) (plus the workflow running it): **check** (`just check` + `just build`) on app/scripts/config/tooling files; **Docker smoke** on what the Dockerfile copies (`src/` without tests, `content/`, Dockerfile, lockfile, tsconfigs). A cheap first job hashes those files ([`scripts/ci-inputs.sh`](../scripts/ci-inputs.sh)) and looks the hash up in a tiny Actions cache marker ([`.github/actions/ci-inputs`](../.github/actions/ci-inputs/action.yml)); the job runs only if that exact set of files has not passed before, and saves the marker when it succeeds ([`ci-passed`](../.github/actions/ci-passed/action.yml)). So a push that only adds a CHANGELOG entry, a re-push of the same code, or a docs-only PR (main's markers are visible to PRs) skips them, and a cancelled or failed run saves nothing and runs again. Skipped jobs never start a runner; GitHub shows them as passed. [`changelog.yml`](../.github/workflows/changelog.yml) still runs on every PR, without a `pnpm install` (Node runs the gate's TypeScript itself). [`docs.yml`](../.github/workflows/docs.yml) Biome-lints and builds the site when its `docs` inputs change and have not passed. [`terraform.yml`](../.github/workflows/terraform.yml) starts only when `infra/`, that workflow, the `justfile` or the gate change, skips everything if its `terraform` inputs already passed, and runs `terraform plan` on each env via GitHub Environment `dev` / `prod` OIDC **only if `infra/` Terraform files changed** (markdown does not); plan jobs that aren't needed never start, so they never wait on an Environment. Missing `ARM_*` **fails closed**. No apply on merge. GitHub is not Key Vault Secrets User, so a secret refresh would 403 (ADR 0012). Actions are pinned to commit SHAs. `just ci-inputs ` shows what a list matches. +- **CD:** `just deploy-dev` for Azure `dev` (local GHCR push + `az` + `just register dev` + Sentry release). Production slash commands: the Release's `register-prod` job (fallback `just register prod`). [`.github/workflows/cd.yml`](../.github/workflows/cd.yml) on `main` only, and only when the **image** inputs differ from the last run that published (so a revert publishes again, and a run cancelled by a newer merge leaves the newer one to publish): publish with `GITHUB_TOKEN` (`packages: write` on that job) and upload Sentry source maps (fails closed if org/token are missing). CD runs queue and are never cancelled, so a quick docs-only merge cannot cut short the publish before it. **No Azure login and no `pixel-prod` update** on merge. [`.github/workflows/release.yml`](../.github/workflows/release.yml) (`workflow_dispatch` on `main`) runs `just check` on the `main` commit being released (CI on `main` may have been cancelled or red), cuts the version only if that passes, publishes `:vX.Y.Z`, Environment `prod` approval rolls `pixel-prod` (`az`, min=max=1, no Terraform apply), a second approval runs `register-prod` (reads only `discord-token` from `pixel-prod-kv`), and publishes GitHub Pages. See [ADR 0010](adr/0010-ghcr-cd.md), [ADR 0012](adr/0012-prod-environment.md) and [ADR 0013](adr/0013-release-workflow.md). ### CI and CD pipelines -How a change moves through GitHub. Operator steps: [`runbook.md`](runbook.md). Hosting picture: [`infra.md`](infra.md). Path allowlists: [`.github/path-filters.yml`](../.github/path-filters.yml). +How a change moves through GitHub. Operator steps: [`runbook.md`](runbook.md). Hosting picture: [`infra.md`](infra.md). Job inputs: [`.github/path-filters.yml`](../.github/path-filters.yml). Pull request: ```mermaid flowchart TD - pr[Pull request] - pr --> clog[Changelog gate always] - pr --> docs{"docs/ changed?"} - docs -->|yes| site[Biome + docs site build] - docs -->|no| nosite[Docs site off] + pr[Pull request push] + pr --> clog[Changelog gate always, no install] + pr --> docs{"docs inputs in this push?"} + docs -->|yes| docsp{"docs inputs already passed?"} + docsp -->|no| site[Biome + docs site build] + docsp -->|yes| nosite[Skip site build] + docs -->|no| nodocs[Docs workflow off] pr --> tf{"infra/, justfile or terraform.yml?"} - tf -->|yes| tfval[just tf-validate] + tf -->|yes| tfp{"terraform inputs already passed?"} + tfp -->|no| tfval[just tf-validate] tfval --> tffiles{"infra Terraform files?"} tffiles -->|yes| azplan[Azure plan via Environment OIDC] - tffiles -->|no| noaz[Skip Azure plan] + tffiles -->|no| noaz[Plan jobs never start] + tfp -->|yes| notfrun[Skip validate and plans] tf -->|no| notf[Terraform workflow off] - pr --> checkp{"check path?"} - checkp -->|yes| check["just check + just build"] - checkp -->|no| nocheck[Skip just check] - pr --> imagep{"image path?"} - imagep -->|yes| docker[Docker image smoke] - imagep -->|no| nodocker[Skip Docker] + pr --> checkp{"check inputs already passed?"} + checkp -->|no| check["just check + just build"] + checkp -->|yes| nocheck[Skip just check] + pr --> imagep{"image inputs already passed?"} + imagep -->|no| docker[Docker image smoke] + imagep -->|yes| nodocker[Skip Docker] ``` Merge to `main`: @@ -516,12 +520,13 @@ Merge to `main`: ```mermaid flowchart TD main[Push to main] - main --> ci[Same CI path filter as PRs] + main --> ci["Same CI gates as PRs; what passes here PRs reuse"] main --> queue[CD runs queue, never cancel] - queue --> imagep{"image path since last green CD run?"} - imagep -->|yes| pub["CD: GHCR SHA and :main"] + queue --> imagep{"image inputs same as last publish?"} + imagep -->|no| pub["CD: GHCR SHA and :main"] pub --> maps[Sentry source maps] - imagep -->|no| nocd[Skip GHCR and Sentry] + maps --> mark[Remember these inputs as published] + imagep -->|yes| nocd[Skip GHCR and Sentry] main --> note[Does not roll pixel-prod] ``` diff --git a/docs/infra.md b/docs/infra.md index 789aec2..88b6421 100644 --- a/docs/infra.md +++ b/docs/infra.md @@ -35,8 +35,8 @@ flowchart LR subgraph github["GitHub pixelbar/pixel"] SRC[git + content/info] GHCR["GHCR public
ghcr.io/pixelbar/pixel"] - CI["CI: check and/or image smoke
by path allowlist"] - CD["CD on main: GHCR when image paths
change; Sentry maps"] + CI["CI: check and/or image smoke
unless those inputs already passed"] + CD["CD on main: GHCR when image inputs
differ from last publish; Sentry maps"] REL["Release: bump, GHCR :vX.Y.Z,
Pages, approved pixel-prod,
register prod commands"] TFW["terraform.yml: plan envs/dev + prod"] ENVDEV["Environment dev
ARM_* for OIDC plan"] diff --git a/docs/runbook.md b/docs/runbook.md index 8aec9d8..f5301c0 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -2,7 +2,7 @@ How to run Pixel day to day and what to do when something goes wrong. It is written for volunteers: anyone with the right access should be able to follow it without the original author. -**This is a living document.** Bootstrap (#8) is applied. Container App `pixel-dev` is live (#9). `infra/envs/prod` is in repo (#12). CD: `just deploy-dev` rolls Azure `dev`; merges to `main` publish GHCR when an image path changed. Prod and the docs site ship from the Release workflow. Every procedure has two parts: +**This is a living document.** Bootstrap (#8) is applied. Container App `pixel-dev` is live (#9). `infra/envs/prod` is in repo (#12). CD: `just deploy-dev` rolls Azure `dev`; merges to `main` publish GHCR when the image inputs differ from the last publish. Prod and the docs site ship from the Release workflow. Every procedure has two parts: - **Today:** what works now, with Pixel run locally or in a container you start yourself. Everything here has been checked against the code. - **🚧 Azure:** a marker for the steps that still depend on hosting that is not applied, or on #12. Search for `🚧` to find what's left. @@ -50,7 +50,7 @@ Before you change anything on **prod**, read "Rules of thumb" below. | **Environments** | `dev` (Pixel Dev bot, test guild) and `prod` (production Pixel bot, production guild). Separate bots, tokens and secrets. Never copy the Dev token into prod. | | **Who is on the hook when the bot is down** | 🚧 Not decided. Put a named contact (and a backup) here, and where to reach them. | | **Where Pixel runs** | Azure `pixel-dev` (one replica, no public ingress) plus `pixel-prod` after a maintainer apply, plus any laptop `just dev`. Subscription `d150e252-e2f0-47fb-8a4a-c3f29e9aebd4`, West Europe. Resource groups `pixel-bootstrap`, `pixel-dev`, `pixel-prod`. `just deploy-dev` rolls `pixel-dev`. **Do not** also run `just dev` on the Pixel Dev token while that app is up. A Release rolls `pixel-prod` after Environment `prod` approval. Diagrams: [`infra.md`](infra.md). | -| **Where the code and CI are** | GitHub, `pixelbar/pixel`. CI runs `just check` and/or a Docker smoke build only when those jobs' path allowlists match. Docs PRs run Biome + the site build. CD on `main` publishes `ghcr.io/pixelbar/pixel:` when an image path changed. | +| **Where the code and CI are** | GitHub, `pixelbar/pixel`. CI runs `just check` and/or a Docker smoke build only when the files those jobs depend on have not already passed. Docs PRs run Biome + the site build. CD on `main` publishes `ghcr.io/pixelbar/pixel:` when the image inputs differ from the last publish. | | **Access you may need** | The Discord Developer Portal (bot token), a Discord role that can manage the server, the Sentry project, the GitHub repo, the Home Assistant admin account, Owner on the Pixel Azure subscription (to apply bootstrap), and the host. | ## Quick reference @@ -434,7 +434,7 @@ All of them are gitignored. The share files are **read-only mirrors**: edit the **Azure CD ([ADR 0010](adr/0010-ghcr-cd.md), [ADR 0013](adr/0013-release-workflow.md)):** - **`dev`:** from this tree, `just deploy-dev`. Builds **linux/amd64** (Azure cannot run Mini arm64 images), pushes `ghcr.io/pixelbar/pixel:` (or `dev-dirty-*` if the tree is dirty) and `:dev`, then `az containerapp update` on `pixel-dev` only (min=max=1). Then `just register dev` against the Pixel Dev guild from `.env.dev` (or `.env`), then `just sentry-release` (skipped without org/token; a Sentry failure still fails the recipe but after commands are registered). Needs `gh auth refresh --scopes write:packages,read:packages,repo`, Docker, `az` on subscription `d150e252-e2f0-47fb-8a4a-c3f29e9aebd4`, and `.env` for register. Does not start Pixel. CI does not deploy pull requests (so a local SHA is not overwritten by another CI head). -- **`prod`:** not on merge. A maintainer runs **Actions → Release → Run workflow** on `main` ([`.github/workflows/release.yml`](../.github/workflows/release.yml)). That runs `just check` on the `main` commit (if it fails, nothing is tagged, published or deployed), cuts Unreleased, tags `vX.Y.Z`, opens a GitHub Release, publishes `:` / `:vX.Y.Z` / `:main`, uploads Sentry source maps, publishes the docs site, then GitHub Environment `prod` (reviewers, branch `main`) rolls `pixel-prod` to **that same digest**. No Terraform apply. Fails closed without `ARM_*` or if `pixel-prod` does not exist yet. Never copy `pixel-dev-kv` or the Mini local Discord token into prod. After the roll, a second Environment `prod` approval runs **Register prod slash commands**: it reads the app ID, guild ID and `discord-token` reference from `pixel-prod` itself, refuses unless `PIXEL_ENV=prod`, and runs `just register-ci prod` (example access lists, throwaway data dir). It fails closed until `github_register_principal_id` is applied in `infra/envs/prod` ([ADR 0014](adr/0014-release-registers-commands.md)); then **Re-run failed jobs**, or `just register prod` with `.env.prod`. Merges to `main` ([`.github/workflows/cd.yml`](../.github/workflows/cd.yml)) publish the merge SHA and `:main` and upload Sentry maps when an image path changed since the last green CD run; they do not log in to Azure. CD runs queue rather than cancel each other, so a merge that failed or was skipped is picked up by the next one: if a SHA is missing from GHCR, fix the cause and re-run or merge again. +- **`prod`:** not on merge. A maintainer runs **Actions → Release → Run workflow** on `main` ([`.github/workflows/release.yml`](../.github/workflows/release.yml)). That runs `just check` on the `main` commit (if it fails, nothing is tagged, published or deployed), cuts Unreleased, tags `vX.Y.Z`, opens a GitHub Release, publishes `:` / `:vX.Y.Z` / `:main`, uploads Sentry source maps, publishes the docs site, then GitHub Environment `prod` (reviewers, branch `main`) rolls `pixel-prod` to **that same digest**. No Terraform apply. Fails closed without `ARM_*` or if `pixel-prod` does not exist yet. Never copy `pixel-dev-kv` or the Mini local Discord token into prod. After the roll, a second Environment `prod` approval runs **Register prod slash commands**: it reads the app ID, guild ID and `discord-token` reference from `pixel-prod` itself, refuses unless `PIXEL_ENV=prod`, and runs `just register-ci prod` (example access lists, throwaway data dir). It fails closed until `github_register_principal_id` is applied in `infra/envs/prod` ([ADR 0014](adr/0014-release-registers-commands.md)); then **Re-run failed jobs**, or `just register prod` with `.env.prod`. Merges to `main` ([`.github/workflows/cd.yml`](../.github/workflows/cd.yml)) publish the merge SHA and `:main` and upload Sentry maps when the image inputs differ from the last publish; they do not log in to Azure. CD runs queue rather than cancel each other, and a merge that failed or was skipped is picked up by the next one: if a SHA is missing from GHCR, fix the cause and re-run or merge again. - GHCR: CI uses `GITHUB_TOKEN` (`packages: write` on the publish job only). Local uses `gh`. The package is **public** (no secrets in the image) so Azure can pull. - Prod Azure login is GitHub Environment OIDC (`ARM_*` variables on Environment `prod`). `dev` uses local `az`. Terraform plan on PRs uses Environment `dev` / `prod` `ARM_*` and fails closed if they are missing. - Neither path applies Terraform. After first create, **dev** apply **ignores** the Pixel container image, so `just tf-apply dev` cannot silently replace the SHA `just deploy-dev` pinned. Prod apply sets `container_image`; a later Release roll moves it again. Dev rollback is `az`, not a tfvars bump. @@ -444,37 +444,45 @@ All of them are gitignored. The share files are **read-only mirrors**: edit the ## CI and CD -What GitHub runs. The same diagrams live in [`architecture.md`](architecture.md#ci-and-cd-pipelines). Allowlists: [`.github/path-filters.yml`](../.github/path-filters.yml) ([`dorny/paths-filter`](https://github.com/dorny/paths-filter)). +What GitHub runs. The same diagrams live in [`architecture.md`](architecture.md#ci-and-cd-pipelines). Job inputs: [`.github/path-filters.yml`](../.github/path-filters.yml). -**Check** (`just check` + `just build`): `src/`, `scripts/`, `content/`, `config/`, lockfile / `package.json`, tsconfigs, `biome.json`, `vitest.config.*`, `.nvmrc`, `justfile`, `.github/workflows/ci.yml`, `.github/path-filters.yml`, plus the files the secrets manifest test reads: `infra/secrets.manifest.yaml`, `infra/modules/pixel/*.tf`, `infra/envs/*/*.tf`, `.env.example`, `docs/runbook.md`, `docs/architecture.md`, `docs/infra.md`. +Each expensive job runs only when the files it depends on are not a set that already passed. A first "What changed" job hashes them ([`scripts/ci-inputs.sh`](../scripts/ci-inputs.sh)) and looks for a marker in the Actions cache; the job saves one when it succeeds. So pushing only a CHANGELOG entry, re-pushing the same code, or a docs-only PR (PRs can see `main`'s markers) skips them. A skipped job never starts and GitHub shows it as passed. Every hash also covers the workflow running the job, `.github/path-filters.yml`, `scripts/ci-inputs.sh` and `.github/actions/`. To see what a list matches: `just ci-inputs check`. -**Image** (Docker smoke on PRs; GHCR + Sentry on `main`): `src/` (including tests), `content/`, Dockerfile, `.dockerignore`, lockfile / `package.json`, tsconfigs, `justfile`, `.github/workflows/ci.yml` and `cd.yml`, `.github/path-filters.yml`. +**Check** (`just check` + `just build`): `src/`, `scripts/`, `content/`, `config/`, root `*.ts` and `*.json` (`package.json`, tsconfigs, `biome.json`, `vitest.config.ts`), lockfile, `.nvmrc`, `.gitignore`, `justfile`, plus the files the secrets manifest test reads: `infra/secrets.manifest.yaml`, `infra/modules/pixel/*.tf`, `infra/envs/*/*.tf`, `.env.example`, `docs/runbook.md`, `docs/architecture.md`, `docs/infra.md`, and the files `scripts/ci-inputs.test.ts` checks the lists against: `Dockerfile`, `cd.yml`, `docs.yml`, `terraform.yml`. -**Docs** (`docs/`): Biome lint + VitePress build. Not app CI. +**Image** (Docker smoke on PRs and `main`; GHCR + Sentry on `main`): what the Dockerfile copies: `src/` without tests (`*.test.ts`, `src/testing/`), `content/`, lockfile / `package.json`, `tsconfig.json`, `tsconfig.build.json`, Dockerfile, `.dockerignore`. CD publishes when these (and `cd.yml`) differ from the last run that published, so a revert publishes again. If a CD run was cancelled or failed, the next merge publishes. -**Always on PRs:** the changelog gate. **Terraform:** workflow when `infra/`, that workflow, or the `justfile` change; Azure plan only when Terraform files under `infra/` changed (markdown does not). +**Docs** (`docs/`, root `package.json` / lockfile, `biome.json`, `.gitignore`, `.nvmrc`, `justfile`): Biome lint + VitePress build. Not app CI. + +**Always on PRs:** the changelog gate (no `pnpm install`). **Terraform:** the workflow starts when `infra/`, that workflow, the `justfile` or the gate change; it skips if its inputs (`infra/` without markdown, `justfile`) already passed; Azure plans run only when Terraform files under `infra/` changed, and otherwise never start (so never wait on an Environment). + +**Forcing a rerun:** re-run the workflow after deleting the `ci-passed-…` cache under Actions → Caches, or change a file in the list. Markers are evicted after 7 days unused; then the job just runs again. Pull request: ```mermaid flowchart TD - pr[Pull request] - pr --> clog[Changelog gate always] - pr --> docs{"docs/ changed?"} - docs -->|yes| site[Biome + docs site build] - docs -->|no| nosite[Docs site off] + pr[Pull request push] + pr --> clog[Changelog gate always, no install] + pr --> docs{"docs inputs in this push?"} + docs -->|yes| docsp{"docs inputs already passed?"} + docsp -->|no| site[Biome + docs site build] + docsp -->|yes| nosite[Skip site build] + docs -->|no| nodocs[Docs workflow off] pr --> tf{"infra/, justfile or terraform.yml?"} - tf -->|yes| tfval[just tf-validate] + tf -->|yes| tfp{"terraform inputs already passed?"} + tfp -->|no| tfval[just tf-validate] tfval --> tffiles{"infra Terraform files?"} tffiles -->|yes| azplan[Azure plan via Environment OIDC] - tffiles -->|no| noaz[Skip Azure plan] + tffiles -->|no| noaz[Plan jobs never start] + tfp -->|yes| notfrun[Skip validate and plans] tf -->|no| notf[Terraform workflow off] - pr --> checkp{"check path?"} - checkp -->|yes| check["just check + just build"] - checkp -->|no| nocheck[Skip just check] - pr --> imagep{"image path?"} - imagep -->|yes| docker[Docker image smoke] - imagep -->|no| nodocker[Skip Docker] + pr --> checkp{"check inputs already passed?"} + checkp -->|no| check["just check + just build"] + checkp -->|yes| nocheck[Skip just check] + pr --> imagep{"image inputs already passed?"} + imagep -->|no| docker[Docker image smoke] + imagep -->|yes| nodocker[Skip Docker] ``` Merge to `main`: @@ -482,12 +490,13 @@ Merge to `main`: ```mermaid flowchart TD main[Push to main] - main --> ci[Same CI path filter as PRs] + main --> ci["Same CI gates as PRs; what passes here PRs reuse"] main --> queue[CD runs queue, never cancel] - queue --> imagep{"image path since last green CD run?"} - imagep -->|yes| pub["CD: GHCR SHA and :main"] + queue --> imagep{"image inputs same as last publish?"} + imagep -->|no| pub["CD: GHCR SHA and :main"] pub --> maps[Sentry source maps] - imagep -->|no| nocd[Skip GHCR and Sentry] + maps --> mark[Remember these inputs as published] + imagep -->|yes| nocd[Skip GHCR and Sentry] main --> note[Does not roll pixel-prod] ``` diff --git a/justfile b/justfile index f29e862..69734ca 100644 --- a/justfile +++ b/justfile @@ -43,13 +43,19 @@ typecheck: pnpm exec tsc --noEmit # Everything CI runs for the Node app: lint, types, tests with coverage thresholds. -# GitHub runs this only when a check-path matches (.github/path-filters.yml). +# GitHub skips it when the `check` inputs already passed (.github/path-filters.yml). check: lint typecheck coverage +# The files a CI job depends on (a list in .github/path-filters.yml), and the +# hash CI uses to skip the job when they already passed: just ci-inputs check +ci-inputs filter: + bash scripts/ci-inputs.sh --list {{filter}} + @echo "hash (CI also adds the workflow file): $(bash scripts/ci-inputs.sh {{filter}})" + # Fail unless this branch updates CHANGELOG.md vs the base (default origin/main). # CI sets SKIP_CHANGELOG when the PR has the skip-changelog label. check-changelog *args: - pnpm exec tsx scripts/check-changelog.ts {{args}} + node scripts/check-changelog.ts {{args}} # Cut [Unreleased] into a dated version and bump package.json. # Added → minor; Fixed/Changed → patch. Never major before 1.0. diff --git a/scripts/ci-inputs.sh b/scripts/ci-inputs.sh new file mode 100755 index 0000000..022b076 --- /dev/null +++ b/scripts/ci-inputs.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Hash the files a CI job depends on, so the job can skip when that exact set +# of files already passed (.github/actions/ci-inputs looks the hash up). +# +# Usage: scripts/ci-inputs.sh [--list] [extra-file...] +# a list in .github/path-filters.yml +# extra-file more files to include (the workflow that runs the job) +# --list print the matched files instead of the hash +# +# Patterns are git glob pathspecs: `*` stays inside one directory, `**` +# crosses them, and a leading `!` excludes. The hash covers each file's path, +# mode and committed content (what git has, not unstaged edits), plus the +# filter file, this script and the CI actions, so changing any of them reruns +# every job. +set -euo pipefail + +filters=".github/path-filters.yml" +list=false +if [ "${1:-}" = "--list" ]; then + list=true + shift +fi +filter="${1:?usage: ci-inputs.sh [--list] [extra-file...]}" +shift + +# The filter file is plain `name:` keys with `- "pattern"` lists under them. +patterns="$(awk -v want="${filter}" ' + /^[A-Za-z0-9_-]+:[[:space:]]*(#.*)?$/ { current = substr($0, 1, index($0, ":") - 1); next } + /^[^[:space:]#]/ { current = "" } + current == want && /^[[:space:]]+-[[:space:]]/ { + line = $0 + sub(/^[[:space:]]+-[[:space:]]+/, "", line) + sub(/[[:space:]]+#.*$/, "", line) + gsub(/^"|"$/, "", line) + print line + } +' "${filters}")" + +if [ -z "${patterns}" ]; then + echo "ci-inputs: no filter named ${filter} in ${filters}" >&2 + exit 1 +fi + +pathspecs=(":(literal)${filters}" ":(literal)scripts/ci-inputs.sh" ":(glob).github/actions/**") +while IFS= read -r pattern; do + case "${pattern}" in + "!"*) pathspecs+=(":(glob,exclude)${pattern#!}") ;; + *) pathspecs+=(":(glob)${pattern}") ;; + esac +done <<<"${patterns}" +for extra in "$@"; do + pathspecs+=(":(literal)${extra}") +done + +if "${list}"; then + git ls-files -- "${pathspecs[@]}" +else + { + echo "${filter}" + git ls-files -s -- "${pathspecs[@]}" + } | git hash-object --stdin +fi diff --git a/scripts/ci-inputs.test.ts b/scripts/ci-inputs.test.ts new file mode 100644 index 0000000..5732cac --- /dev/null +++ b/scripts/ci-inputs.test.ts @@ -0,0 +1,180 @@ +import { execFileSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +const SCRIPT = resolve("scripts/ci-inputs.sh"); +const FILTERS = ".github/path-filters.yml"; +/** What every input hash covers on top of its list (see scripts/ci-inputs.sh). */ +const ALWAYS = [FILTERS, ".github/actions/**", "scripts/ci-inputs.sh"]; + +const read = (file: string) => readFileSync(file, "utf8"); + +function run(cwd: string, ...args: string[]): string { + return execFileSync("bash", [SCRIPT, ...args], { cwd, encoding: "utf8", stdio: "pipe" }).trim(); +} + +const list = (cwd: string, filter: string) => run(cwd, "--list", filter).split("\n"); + +/** The lists in the real filter file. */ +function filters(): Record { + return parse(read(FILTERS)) as Record; +} + +describe("ci-inputs.sh", () => { + let dir: string; + + const write = (file: string, text: string) => { + mkdirSync(dirname(join(dir, file)), { recursive: true }); + writeFileSync(join(dir, file), text); + }; + const git = (...args: string[]) => execFileSync("git", args, { cwd: dir, stdio: "pipe" }); + const commit = () => { + git("add", "-A"); + git("-c", "user.name=t", "-c", "user.email=t@example.invalid", "commit", "-qm", "x"); + }; + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "pixel-ci-inputs-")); + git("init", "-q"); + write( + FILTERS, + [ + "# a comment", + "app:", + ' - "src/**"', + ' - "!src/**/*.test.ts" # tests do not ship', + " # an indented comment", + ' - "*.json"', + "site:", + ' - "docs/**"', + "", + ].join("\n"), + ); + write("src/a.ts", "a"); + write("src/deep/b.ts", "b"); + write("src/deep/b.test.ts", "test"); + write("package.json", "{}"); + write("config/x.json", "{}"); + write("docs/index.md", "docs"); + write("CHANGELOG.md", "changes"); + commit(); + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + it("lists a filter's files, its exclusions applied, plus the filter file", () => { + expect(list(dir, "app").sort()).toEqual( + [FILTERS, "package.json", "src/a.ts", "src/deep/b.ts"].sort(), + ); + expect(list(dir, "site").sort()).toEqual([FILTERS, "docs/index.md"].sort()); + }); + + it("keeps the hash when only files outside the filter change", () => { + const before = run(dir, "app"); + write("CHANGELOG.md", "more changes"); + write("src/deep/b.test.ts", "another test"); + write("config/x.json", '{"a":1}'); + commit(); + expect(run(dir, "app")).toBe(before); + }); + + it("changes the hash when a matched file changes or moves", () => { + const before = run(dir, "app"); + write("src/a.ts", "a2"); + commit(); + const edited = run(dir, "app"); + expect(edited).not.toBe(before); + git("mv", "src/a.ts", "src/c.ts"); + commit(); + expect(run(dir, "app")).not.toBe(edited); + }); + + it("gives different filters with the same files different hashes", () => { + write(FILTERS, `${read(join(dir, FILTERS))}same:\n - "docs/**"\n`); + commit(); + expect(run(dir, "same")).not.toBe(run(dir, "site")); + }); + + it("adds extra files, such as the workflow running the job", () => { + write(".github/workflows/ci.yml", "on: push"); + commit(); + expect(run(dir, "--list", "site", ".github/workflows/ci.yml")).toContain( + ".github/workflows/ci.yml", + ); + expect(run(dir, "site", ".github/workflows/ci.yml")).not.toBe(run(dir, "site")); + }); + + it("fails on a filter that does not exist", () => { + expect(() => run(dir, "nope")).toThrow(); + }); +}); + +describe(FILTERS, () => { + it("is plain lists, so scripts/ci-inputs.sh can read it", () => { + for (const [name, patterns] of Object.entries(filters())) { + expect(name).toMatch(/^[A-Za-z0-9_-]+$/); + expect(Array.isArray(patterns), name).toBe(true); + expect(run(".", "--list", name).split("\n").length).toBeGreaterThan(1); + } + }); + + // These tests read them, so `just check` must run when they change. + it("puts the files this test reads in `check`", () => { + const check = new Set(list(".", "check")); + for (const file of [ + "Dockerfile", + FILTERS, + ".github/workflows/cd.yml", + ".github/workflows/docs.yml", + ".github/workflows/terraform.yml", + ]) { + expect(check.has(file), `${file} is read by this test but not in the check list`).toBe(true); + } + }); + + it("puts everything the Dockerfile copies in from the repository in `image`", () => { + const image = new Set(list(".", "image")); + const copies = read("Dockerfile") + .split("\n") + .filter((line) => /^COPY\s/.test(line) && !line.includes("--from=")) + .flatMap((line) => line.trim().split(/\s+/).slice(1, -1)); + expect(copies.length).toBeGreaterThan(0); + for (const source of copies) { + const shipped = [...image].some((file) => file === source || file.startsWith(`${source}/`)); + expect(shipped, `${source} is copied into the image but not in the image list`).toBe(true); + } + }); + + it("names only lists that exist in the workflows", () => { + const known = new Set(Object.keys(filters())); + for (const workflow of ["ci", "cd", "docs", "terraform"]) { + const used = [...read(`.github/workflows/${workflow}.yml`).matchAll(/filter: (\S+)/g)]; + expect(used.length, workflow).toBeGreaterThan(0); + for (const [, name] of used) expect(known.has(name ?? ""), `${workflow}: ${name}`).toBe(true); + } + }); + + // A paths trigger that missed one of the list's files would never start the + // workflow for a change to it. + it.each([ + ["docs", "docs"], + ["terraform", "terraform"], + ])("%s.yml starts for every file its `%s` list covers", (workflow, filter) => { + const file = `.github/workflows/${workflow}.yml`; + const on = (parse(read(file)) as { on: Record }).on; + const wanted = [ + ...(filters()[filter] ?? []).filter((pattern) => !pattern.startsWith("!")), + ...ALWAYS, + file, + ]; + for (const event of Object.values(on)) { + if (!event?.paths) continue; + expect([...event.paths].sort()).toEqual([...new Set(wanted)].sort()); + } + }); +});