From dbb107556f325d4a66530fad40100ab92e42e168 Mon Sep 17 00:00:00 2001 From: NinjaLikesCheez Date: Thu, 15 Oct 2026 12:12:37 +0200 Subject: [PATCH] Make just secrets-push the only writer of app secrets (ADR 0017) Terraform's write_secrets now writes only the manifest's infra group (tailscale-auth-key, ghcr-pull-token) as azurerm_key_vault_secret.infra. moved blocks carry those keys over from .managed; a removed block with destroy = false drops the app secrets from state without deleting them from Key Vault. The app-secret Terraform inputs are gone from the module and both stacks. The Container App still references every enabled secret. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- CHANGELOG.md | 1 + docs/.vitepress/config.ts | 4 ++ docs/adr/0015-secrets-manifest.md | 2 +- docs/adr/0016-secrets-sync.md | 4 +- docs/adr/0017-one-writer-for-app-secrets.md | 24 +++++++ docs/architecture.md | 3 +- docs/infra.md | 4 +- docs/maintainers/index.md | 1 + docs/runbook.md | 20 +++--- infra/README.md | 2 +- infra/envs/dev/README.md | 25 ++++--- infra/envs/dev/main.tf | 11 +--- infra/envs/dev/outputs.tf | 2 +- infra/envs/dev/terraform.example.tfvars | 24 +++---- infra/envs/dev/variables.tf | 72 +------------------- infra/envs/prod/README.md | 20 +++--- infra/envs/prod/main.tf | 11 +--- infra/envs/prod/outputs.tf | 2 +- infra/envs/prod/terraform.example.tfvars | 16 ++--- infra/envs/prod/variables.tf | 72 +------------------- infra/modules/pixel/README.md | 2 +- infra/modules/pixel/container_app.tf | 4 +- infra/modules/pixel/identity.tf | 5 +- infra/modules/pixel/locals.tf | 4 ++ infra/modules/pixel/outputs.tf | 4 +- infra/modules/pixel/secrets.tf | 73 ++++++++++++--------- infra/modules/pixel/variables.tf | 72 +------------------- infra/secrets.manifest.yaml | 12 ++-- scripts/secrets-manifest.test.ts | 22 ++++++- scripts/secrets-manifest.ts | 8 +++ scripts/secrets-sync-lib.test.ts | 6 +- scripts/secrets-sync.ts | 4 -- 33 files changed, 190 insertions(+), 348 deletions(-) create mode 100644 docs/adr/0017-one-writer-for-app-secrets.md diff --git a/AGENTS.md b/AGENTS.md index e626423..381228e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -111,7 +111,7 @@ infra/ # Terraform. bootstrap/ is #8; modules/pixel and envs/de - **Calendar:** features read events from the core `Calendar` (neutral `CalendarEvent`s) and never touch a platform. An adapter plugs in a `CalendarSource` when it's ready. **Text written by other people** (event titles, locations and so on) must go through `escapeMarkdown` before it's put in a reply. - **Content:** `content/info/*.md` is committed to a **public** repository, so never put secrets (wifi passwords, door codes, tokens) or personal data in it, and don't add member-only topics. Keep topics short and link to the canonical page on pixelbar.nl rather than copying it. Don't invent facts: every claim should come from Pixelbar's own pages or from the board. Adding, removing or renaming a topic needs the commands re-registered (`just register dev` or `just register prod`), and CI loads the real content, so keep it valid. - **Background work:** a feature that needs it declares `start()`, which returns a stop function. Don't start timers or listeners at import time or in a factory. Anything that runs from a timer must catch its own failures so none escapes as an unhandled rejection. -- **Secrets manifest:** [`infra/secrets.manifest.yaml`](infra/secrets.manifest.yaml) is the one list of Key Vault secrets (names and metadata only, never a value; [ADR 0015](docs/adr/0015-secrets-manifest.md)). A new secret is an entry there plus its Terraform variable and write-only entry, the setting in `config.ts`, `.env.example`, the architecture table and the runbook; `scripts/secrets-manifest.test.ts` names what you missed. Don't hard-code a secret's vault name in `locals.tf`. Maintainers sync the `app` group with `just secrets-pull` / `just secrets-push` ([ADR 0016](docs/adr/0016-secrets-sync.md)); the sync never prints a value, so keep it that way. Coding agents never run `just secrets-push prod` or `just push prod`. `terraform.tfvars` is shared through Key Vault (`terraform-tfvars`, the `tooling` group, which Terraform must keep skipping) and holds settings only: secret Terraform inputs go in gitignored `secrets.auto.tfvars`. +- **Secrets manifest:** [`infra/secrets.manifest.yaml`](infra/secrets.manifest.yaml) is the one list of Key Vault secrets (names and metadata only, never a value; [ADR 0015](docs/adr/0015-secrets-manifest.md)). A new secret is an entry there plus the setting in `config.ts`, `.env.example`, the architecture table and the runbook (an `infra`-group secret also gets its Terraform variable and write-only entry); `scripts/secrets-manifest.test.ts` names what you missed. **`just secrets-push` / `just push` are the only writers of the `app` group** ([ADR 0017](docs/adr/0017-one-writer-for-app-secrets.md)): Terraform references those secrets but never writes them, so don't add an app secret to `secrets.tf` or give it a Terraform input; `write_secrets` covers the `infra` group only. Don't hard-code a secret's vault name in `locals.tf`. Maintainers sync the `app` group with `just secrets-pull` / `just secrets-push` ([ADR 0016](docs/adr/0016-secrets-sync.md)); the sync never prints a value, so keep it that way. Coding agents never run `just secrets-push prod` or `just push prod`. `terraform.tfvars` is shared through Key Vault (`terraform-tfvars`, the `tooling` group, which Terraform must keep skipping) and holds settings only: secret Terraform inputs go in gitignored `secrets.auto.tfvars`. - **Config:** a new environment variable must be added to `config.ts`, `.env.example` and the architecture doc's config table. - **Errors:** throw `UserFacingError` for problems the user should see. Anything else is reported to Sentry, and the user gets a generic reply. - **Types:** `strict`, no `any` (use `unknown` and narrow it). Prefer `type` over `interface`. diff --git a/CHANGELOG.md b/CHANGELOG.md index 423cd5b..76e783a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ Until 1.0.0, releases stay on 0.x: `Added` bumps minor, `Fixed`/`Changed` (and t ### Changed - CI jobs run only when the files they depend on have not already passed. Each workflow's first job hashes the job's inputs (a list in `.github/path-filters.yml`, plus the workflow running it) and looks for a marker that an earlier run with exactly those files succeeded. So pushing only a CHANGELOG entry, re-pushing the same code, or opening a docs-only PR no longer runs `just check`, the Docker build, the docs site build or Terraform validate again. Skipped jobs no longer start a runner. Test-only changes no longer build or publish an image. CD publishes when the image inputs differ from the last publish, so a CD run cancelled by a newer merge no longer leaves code without a GHCR image. Terraform plan jobs that aren't needed never start, so they never wait on an Environment. The changelog gate no longer runs `pnpm install`. `just ci-inputs ` shows what a list matches; the runbook's "CI and CD" says how to force a rerun. +- `just secrets-push` / `just push` are now the only writers of the app secrets in Key Vault (Discord token, `admins.yaml`, Home Assistant, Mastodon, space state, Sentry). Terraform's `write_secrets` writes only `tailscale-auth-key` and `ghcr-pull-token`, and the app-secret Terraform inputs (`discord_token`, `admins_yaml_file`, `sentry_dsn`, `spacestate_user` and the rest) are gone, so turning on a feature no longer makes an apply demand or collide with secrets the sync tool set. Push an app secret before the apply that turns its feature on. The next `just tf-plan dev|prod` drops the app secrets Terraform had written from state ("will no longer be managed by Terraform, but will not be destroyed"); nothing leaves Key Vault. Delete the app-secret lines from any `secrets.auto.tfvars`. See [ADR 0017](docs/adr/0017-one-writer-for-app-secrets.md). ### Fixed diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 1771bb6..324069a 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -167,6 +167,10 @@ export default defineConfig({ }, { text: "0015 · Secrets manifest", link: "/maintainers/adr/0015-secrets-manifest" }, { text: "0016 · Secrets sync", link: "/maintainers/adr/0016-secrets-sync" }, + { + text: "0017 · One writer for app secrets", + link: "/maintainers/adr/0017-one-writer-for-app-secrets", + }, ], }, ], diff --git a/docs/adr/0015-secrets-manifest.md b/docs/adr/0015-secrets-manifest.md index 0f0d408..bd06e88 100644 --- a/docs/adr/0015-secrets-manifest.md +++ b/docs/adr/0015-secrets-manifest.md @@ -1,6 +1,6 @@ # 0015. A secrets manifest is the one list of Key Vault secrets -* Status: accepted (the sync tool is [ADR 0016](0016-secrets-sync.md)) +* Status: accepted (the sync tool is [ADR 0016](0016-secrets-sync.md); [ADR 0017](0017-one-writer-for-app-secrets.md) makes it the only writer of the `app` group) * Date: 2026-10-15 ## Context diff --git a/docs/adr/0016-secrets-sync.md b/docs/adr/0016-secrets-sync.md index d5af075..b4f65b5 100644 --- a/docs/adr/0016-secrets-sync.md +++ b/docs/adr/0016-secrets-sync.md @@ -1,6 +1,6 @@ # 0016. A laptop tool syncs app secrets with Key Vault -* Status: accepted +* Status: accepted (Terraform's part narrowed by [ADR 0017](0017-one-writer-for-app-secrets.md)) * Date: 2026-10-15 ## Context @@ -18,7 +18,7 @@ Maintainers set Pixel's secrets in Key Vault by pasting values into gitignored ` ## Consequences -- Terraform's `write_secrets` path still works and still owns the `infra` secrets. If an environment keeps `write_secrets = true` with app values in its tfvars (now `secrets.auto.tfvars`, see the addendum), a later apply that bumps `secrets_version` puts those values back, so keep them up to date or turn `write_secrets` off once the vault is seeded. +- Terraform's `write_secrets` path owns only the `infra` secrets. It used to write app values too, which collided with this tool; since [ADR 0017](0017-one-writer-for-app-secrets.md) this tool is the only writer of the `app` group. - A maintainer needs Key Vault Secrets Officer on the vault (Secrets User is enough to pull), and the same storage access as `just ha-devices` for the `members.yaml` check. - The local files hold real secrets. They are gitignored and written with mode `0600`; don't copy them anywhere else. - Coding agents don't run `just secrets-push prod` or `just push prod` (AGENTS.md: no prod secrets). diff --git a/docs/adr/0017-one-writer-for-app-secrets.md b/docs/adr/0017-one-writer-for-app-secrets.md new file mode 100644 index 0000000..8bc2311 --- /dev/null +++ b/docs/adr/0017-one-writer-for-app-secrets.md @@ -0,0 +1,24 @@ +# 0017. Only the sync tool writes app secrets + +* Status: accepted (narrows [ADR 0016](0016-secrets-sync.md)) +* Date: 2026-10-15 + +## Context + +After [ADR 0016](0016-secrets-sync.md), two things wrote the manifest's `app` secrets into Key Vault: `just secrets-push` / `just push`, and Terraform's `write_secrets` path, which created `azurerm_key_vault_secret.managed` for every enabled secret except the `tooling` group. They collided. Turning on a feature (`spacestate_api_url`) made Terraform demand the new values as inputs and fail with "Resource precondition failed" when they lived only in `.env.prod`; when the tool had already created them, Terraform failed with "already exists, needs to be imported". A prod apply failed this way on 2026-10-15 over `spacestate-user` and `spacestate-token`. + +## Decision + +- **`just secrets-push` / `just push` are the only writers of the `app` group** (`discord-token`, `admins-yaml`, `home-assistant-token`, `mastodon-access-token`, `spacestate-user`, `spacestate-token`, `spacestate-additional-headers`, `sentry-dsn`). +- **Terraform's `write_secrets` writes only the `infra` group** (`tailscale-auth-key`, `ghcr-pull-token`): `azurerm_key_vault_secret.infra`, whose `for_each` (`kv_infra_secret_names` in `locals.tf`) is derived from the manifest. +- **The Container App still references every enabled secret** (`key_vault_secret_ids`), app group included. Only the writing moved. +- **The app-group Terraform inputs are gone** (`discord_token`, `admins_yaml`, `admins_yaml_file`, `home_assistant_token`, `mastodon_access_token`, `spacestate_user`, `spacestate_token`, `spacestate_additional_headers`, `sentry_dsn`), from the module and both stacks, so there is no way to set them through Terraform. An old `secrets.auto.tfvars` that still sets them only gets an "undeclared variable" warning; delete those lines. +- **Moving state, deleting nothing.** `moved` blocks carry `managed["tailscale-auth-key"]` and `managed["ghcr-pull-token"]` to `infra[...]`. A `removed` block with `lifecycle { destroy = false }` drops whatever is left under `azurerm_key_vault_secret.managed` (the app secrets) from state and leaves them in Key Vault. `removed` can't name single instances, so the infra keys are moved out first. Keep both until every stack has applied them. + +## Consequences + +- The next `just tf-plan dev|prod` shows each app secret Terraform had written as "will no longer be managed by Terraform, but will not be destroyed", any infra secret it had written as "has moved to", and no `destroy` for a Key Vault secret. Anything else in that plan is unrelated to this change. +- Turning on a feature that needs an app secret is: `just secrets-push ` first, then set the feature in the tfvars and apply. The apply only references the secret; the Container App won't start a revision until it exists. +- `write_secrets = true` still needs the infra inputs (`tailscale_auth_key`, `container_registry_password`) for the infra features that are on, and turning it off later removes the infra secrets Terraform wrote from Key Vault, as before. Keep it on while Terraform owns them, or set them with `az` and leave it off from the start. +- The Release workflow's identity may read `discord-token` (ADR 0014). That role assignment is scoped to the secret, so push `discord-token` before the first apply that sets `github_register_principal_id`. +- `scripts/secrets-manifest.test.ts` fails if Terraform's write map names an app secret, or if the `removed` block goes missing. diff --git a/docs/architecture.md b/docs/architecture.md index 10287d6..305c895 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -474,7 +474,7 @@ There is **no database**. Access, schedules and (planned) account linking are fi - **Platform:** Azure Container Apps, **exactly one replica**, no ingress, with a managed identity. A Discord gateway connection needs an always-on process. Two replicas would both connect and answer every command twice. Max replicas = 1 (not a variable). Deploys **stop the old instance before starting the new one** (#11). This stack only pins min=max=1 and `revision_mode = Single`. A database lock is not a plan; there is no database. - **Images:** local `just deploy-dev` pushes `ghcr.io/pixelbar/pixel:` (or a `dev-dirty-*` tag) and moving `:dev` for Azure `dev`. Merges to `main` push the SHA and moving `:main` via [`.github/workflows/cd.yml`](../.github/workflows/cd.yml). A Release rebuilds from the bump commit and also tags `:vX.Y.Z`. Builds are **linux/amd64** (Azure Container Apps); the Mini is arm64, so `just deploy-dev` cross-builds. The images contain no secrets and no access lists. The GHCR package is **public** so Container Apps can pull without a registry password. If it is ever private again, `dev` can set `container_registry_server` / `container_registry_username` and Key Vault `ghcr-pull-token`. The running app pins an explicit tag, never `latest` alone. After first create, **dev** Terraform ignores that pin so apply cannot roll it back (#73). Prod apply will set `container_image`. CI still builds with `push: false` as a smoke check. -- **Secrets:** Key Vault. Discord token, Home Assistant token, Sentry DSN, Tailscale auth key, `admins.yaml`, and optional `ghcr-pull-token` (private GHCR only) reach the container as secrets, never as image layers or Terraform state. Terraform may write them at apply from ephemeral write-only inputs (`write_secrets`); otherwise set the app ones with `just secrets-push dev|prod` ([ADR 0016](adr/0016-secrets-sync.md)), `az` or the portal. Prod must not receive the Mini local / Pixel Dev token (that bot lives in `pixel-dev-kv`). +- **Secrets:** Key Vault. Discord token, Home Assistant token, Sentry DSN, Tailscale auth key, `admins.yaml`, and optional `ghcr-pull-token` (private GHCR only) reach the container as secrets, never as image layers or Terraform state. The app ones are written only by `just secrets-push dev|prod` ([ADR 0016](adr/0016-secrets-sync.md), [ADR 0017](adr/0017-one-writer-for-app-secrets.md)); Terraform writes only the Tailscale key and `ghcr-pull-token`, from ephemeral write-only inputs (`write_secrets`). Prod must not receive the Mini local / Pixel Dev token (that bot lives in `pixel-dev-kv`). - **Access files:** `admins.yaml` is a **read-only** Key Vault secret mount. `members.yaml` is rewritten by admin commands, so it **cannot** be a read-only secret mount. It lives on a **writable Azure Files volume with snapshots**. `config/home-assistant/` is on the same volume: `devices.yaml` is a human allow-list (changed with `just ha-devices`). There is no block-list (#6). - **Runtime files:** the Azure Files share is mounted at `/app/persist` as uid 1000 (`node`), covering `members.yaml`, `home-assistant/`, and `PIXEL_DATA_DIR` (`schedules.yaml` must persist; `home-switches.state` lost → doors start **off**; `space.state` and `announcement-posts/` only cost details; `announcement-routes.yaml` is where `/admin announcements` choices live, so keep it). A container's own filesystem is thrown away on every deploy. - **Environments:** `dev` (Pixel Dev bot, test guild) is `infra/envs/dev`. `prod` (production Pixel bot, production guild) is `infra/envs/prod`. Separate bots, tokens and vaults. **`just deploy-dev`** (a local build) rolls Azure `dev` (one replica, then `just register dev`, then Sentry release). **Merges to `main`** publish GHCR only. **Release** (`workflow_dispatch` on `main`) cuts the version, publishes that image, GitHub Environment `prod` approval rolls `pixel-prod` to that digest (no Terraform apply) and then registers prod slash commands, and publishes the docs site. CI does not deploy pull requests. Fail closed: never copy the Mini / Pixel Dev Discord token into prod. See [ADR 0012](adr/0012-prod-environment.md), [ADR 0013](adr/0013-release-workflow.md) and [ADR 0014](adr/0014-release-registers-commands.md). @@ -582,3 +582,4 @@ Record significant decisions as short ADRs in `docs/adr/NNNN-title.md`. Numbers | 0014 | Release registers prod slash commands; `pixel-github-prod` may read only `discord-token` | accepted | | 0015 | Secrets manifest (`infra/secrets.manifest.yaml`) is the one list of Key Vault secrets; Terraform reads it and a test fails on drift | accepted | | 0016 | `just secrets-pull` / `just secrets-push` sync the manifest's app secrets between Key Vault and `.env.`, diffing against the live vault before a push; `just pull` / `just push` add the shared `terraform-tfvars` and read-only share mirrors | accepted | +| 0017 | `just secrets-push` / `just push` are the only writers of the app secrets; Terraform's `write_secrets` writes only the infra group, and `removed` blocks drop the old app secrets from state without deleting them | accepted | diff --git a/docs/infra.md b/docs/infra.md index 88b6421..05b0dc3 100644 --- a/docs/infra.md +++ b/docs/infra.md @@ -129,7 +129,7 @@ flowchart TB 1. **Bootstrap** (hand-applied) created the three resource groups, state storage, and per-env GitHub identities. Env stacks deploy *into* those groups; they do not recreate them. 2. **`infra/modules/pixel`** + **`infra/envs/dev`** created the vault, files share, CAE, app, backup, logs, and `pixel-dev-app`. Remote state is container `dev`. **`infra/envs/prod`** is the same module in `pixel-prod` (state container `prod`). -3. **Secrets** are Key Vault names (`discord-token`, `admins-yaml`, `home-assistant-token`, `mastodon-access-token`, `spacestate-user`, `spacestate-token`, `sentry-dsn`). The Container App references **URIs + UAMI**. Terraform may write values as ephemeral `write_secrets` inputs, or they are set with `az` / the portal. Discord / HA / Sentry values do **not** belong in state. The Azure Files **account key** does (Container Apps can only mount SMB with the key). +3. **Secrets** are Key Vault names (`discord-token`, `admins-yaml`, `home-assistant-token`, `mastodon-access-token`, `spacestate-user`, `spacestate-token`, `sentry-dsn`). The Container App references **URIs + UAMI**. Only `just secrets-push` writes these app values ([ADR 0017](adr/0017-one-writer-for-app-secrets.md)); Terraform's ephemeral `write_secrets` inputs cover just the infra secrets `tailscale-auth-key` and `ghcr-pull-token`. Discord / HA / Sentry values do **not** belong in state. The Azure Files **account key** does (Container Apps can only mount SMB with the key). 4. **Image:** GHCR is public, so the app pulls with no registry password. `just deploy-dev` updates the running **dev** image with `az containerapp update` and does **not** apply Terraform. After first create, Terraform **ignores** that **dev** image so apply cannot roll it back (#73). Prod apply sets `container_image`; a Release then rolls `pixel-prod` to the released SHA (same digest, no Terraform apply). Pin an explicit SHA (or `dev-dirty-*` on **dev** only), never `latest` alone. 5. **Discord:** outbound gateway + REST. No FQDN. `/healthz` is process up; `/readyz` is Discord connected. Home Assistant is **not** on the probes (HA down must not restart the replica). 6. **Home Assistant:** both URL and token, or neither. Live `dev` uses a Nabu Casa URL. The allow-list is `devices.yaml` on the share. A Tailscale sidecar is the preferred path ([#43](https://github.com/pixelbar/pixel/issues/43), [#74](https://github.com/pixelbar/pixel/issues/74)); the Terraform stub is **off** and is not enough to flip on as-is. Dev must not control the real doors. @@ -194,7 +194,7 @@ just tf-plan dev # needs az login; does not apply just tf-apply dev # maintainer; first create / vault / volume (not the running dev image) # prod: PIXEL_APPLY_PROD=1 just tf-apply prod (coding agents: do not) # seed Files: members.yaml + home-assistant/*.yaml (not the example IDs) -# seed KV if write_secrets was false +# seed KV app secrets: just secrets-push dev just deploy-dev # this tree → GHCR SHA → pixel-dev, then just register dev ``` diff --git a/docs/maintainers/index.md b/docs/maintainers/index.md index 81d0a3e..30c55b1 100644 --- a/docs/maintainers/index.md +++ b/docs/maintainers/index.md @@ -38,6 +38,7 @@ Architecture decision records, newest last. | [0014](./adr/0014-release-registers-commands) | Releases register prod slash commands. | | [0015](./adr/0015-secrets-manifest) | One manifest lists every Key Vault secret. | | [0016](./adr/0016-secrets-sync) | `just pull` / `just push` (and `secrets-pull` / `secrets-push`) sync tfvars and app secrets between Key Vault and the laptop. | +| [0017](./adr/0017-one-writer-for-app-secrets) | Only the sync tool writes app secrets; Terraform writes just the infra group. | ## Working on the site diff --git a/docs/runbook.md b/docs/runbook.md index 8f1ce06..d11bd12 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -227,7 +227,7 @@ just tf-apply dev # only when told; does not roll the running image The first apply after #73 **moves** the Container App address (`azurerm_container_app.this` → `this[0]`). That is not a replace and does not change the running image. Plan should say "has moved", not destroy/create. -After apply, Discord still will not answer until Key Vault secrets exist (`write_secrets` or `az`), `members.yaml` is on the share, an image exists, commands are registered, and nothing else is using that bot token. +After apply, Discord still will not answer until Key Vault secrets exist (app secrets with `just secrets-push dev`; Terraform never writes them, [ADR 0017](adr/0017-one-writer-for-app-secrets.md)), `members.yaml` is on the share, an image exists, commands are registered, and nothing else is using that bot token. **On `dev`, the image is owned by `just deploy-dev`, not by apply** ([#73](https://github.com/pixelbar/pixel/issues/73), [ADR 0010](adr/0010-ghcr-cd.md)). First create uses `container_image` (default `:main`). After that, Terraform **ignores** the Pixel container image on the **dev** app only. A later `just tf-apply dev` still updates vault, volume, env and the replica pin; it does **not** roll the bot back to a stale SHA in gitignored `terraform.tfvars`. Do not apply Terraform from CD to "fix" that drift. Do not bump `container_image` expecting a **dev** apply to change what is running. @@ -308,8 +308,8 @@ Members with the `spacestate` capability can set the space open or closed with ` **Turn it on** (local `.env`, or the Container App env plus Key Vault for the user and token): 1. Set `SPACESTATE_API_URL` to the API address, and `SPACESTATE_USER` and `SPACESTATE_TOKEN` to the credentials it expects. Set all three or none: setting only some stops startup. -2. On Azure, set `spacestate_api_url` in the environment's `terraform.tfvars`, and create the Key Vault secrets `spacestate-user` and `spacestate-token` (with `write_secrets = true` and the `spacestate_user` / `spacestate_token` inputs, or with `az keyvault secret set`). The app reads them as `SPACESTATE_USER` and `SPACESTATE_TOKEN`. -3. Optional: if the API needs extra request headers, set `SPACESTATE_ADDITIONAL_HEADERS` to a JSON object such as `{"X-Example": "value"}`. It is a secret so the values stay out of the public repository. On Azure, set `spacestate_additional_headers_enabled = true` and create the Key Vault secret `spacestate-additional-headers` (`just secrets-push`, or the `spacestate_additional_headers` input). +2. On Azure, first put `SPACESTATE_USER` and `SPACESTATE_TOKEN` in `.env.` and `just secrets-push `, which creates the Key Vault secrets `spacestate-user` and `spacestate-token`. Then set `spacestate_api_url` in the environment's `terraform.tfvars` and apply. Terraform only references the two secrets; it never writes them. +3. Optional: if the API needs extra request headers, set `SPACESTATE_ADDITIONAL_HEADERS` to a JSON object such as `{"X-Example": "value"}`. It is a secret so the values stay out of the public repository. On Azure, push it with `just secrets-push ` (Key Vault secret `spacestate-additional-headers`) first, then set `spacestate_additional_headers_enabled = true` and apply. 4. Restart Pixel, then run `just register dev` (or `prod`) so Discord shows `/spacestate`. 5. An admin grants the capability: `/admin capabilities grant user: capability:spacestate`. Being an admin is not enough. 6. Try `/spacestate set state:open`, then check `/status`. Pixel follows the state it reads back from SpaceAPI, so the change is announced the usual way. @@ -367,11 +367,11 @@ Every secret lives in the secret store for its environment (a local `.env` for d **If a secret leaked:** rotate it first, then work out where it leaked from. Check the logs and Sentry for the time window. A leaked Discord token lets someone act as the bot in your server, so treat it as urgent. -**Azure:** "put it in the secret store" means `just secrets-push dev` (or `prod`), below, which also offers the restart. Terraform can still write the values from gitignored `terraform.tfvars` with `write_secrets = true` (not stored in state). Never commit secret values. +**Azure:** "put it in the secret store" means `just secrets-push dev` (or `prod`), below, which also offers the restart. It is the only thing that writes app secrets; Terraform's `write_secrets` writes only the Tailscale key and the registry token ([ADR 0017](adr/0017-one-writer-for-app-secrets.md)). Never commit secret values. ### Sync secrets with Key Vault -`just secrets-pull` and `just secrets-push` keep an environment's **app** secrets (the `app` group in the manifest: Discord token, `admins.yaml`, Home Assistant, Mastodon, space state, Sentry) in step between its Key Vault and your laptop ([ADR 0016](adr/0016-secrets-sync.md)). The Tailscale key and the registry token are `infra` secrets: Terraform owns them and the tool never touches them. +`just secrets-pull` and `just secrets-push` keep an environment's **app** secrets (the `app` group in the manifest: Discord token, `admins.yaml`, Home Assistant, Mastodon, space state, Sentry) in step between its Key Vault and your laptop ([ADR 0016](adr/0016-secrets-sync.md)). It is their **only** writer: Terraform references them from the Container App but never writes them ([ADR 0017](adr/0017-one-writer-for-app-secrets.md)). The Tailscale key and the registry token are `infra` secrets: Terraform owns them (`write_secrets`) and the tool never touches them. | Env | Key Vault | Local env secrets | Local `admins.yaml` | Container App | | --- | --- | --- | --- | --- | @@ -391,9 +391,11 @@ These local files are gitignored and written readable only by you. **You need:** **Guards:** it refuses an env file whose `PIXEL_ENV` is for the other bot, and a Discord token that's also in another bot's file (`.env.dev` or `.env` for prod; `.env.prod` for dev). Never copy the Pixel Dev or Mini token into `.env.prod`. -**If the stack has `write_secrets = true`** with app values in `secrets.auto.tfvars`, the next apply that bumps `secrets_version` writes those values back over the vault. Keep that file up to date, or set `write_secrets = false` once the vault is seeded and use the sync from then on. +**Old `secrets.auto.tfvars`:** Terraform no longer has app-secret inputs (`discord_token`, `admins_yaml_file`, `sentry_dsn`, `spacestate_user`, …). If your file still sets them, Terraform warns about undeclared variables; delete those lines. Only `tailscale_auth_key` and `container_registry_password` belong there now. -**A new app secret:** add it to the manifest first (above). Once the stack has it, put the value in `.env.` under its environment variable and `just secrets-push `; it shows as `+` new in Key Vault. +**The first plan after ADR 0017** shows each app secret Terraform had written as "will no longer be managed by Terraform, but will not be destroyed", and any Tailscale or registry secret as "has moved to" `azurerm_key_vault_secret.infra`. That is expected: the secrets stay in Key Vault. If the plan **destroys** an `azurerm_key_vault_secret`, stop and ask (with `write_secrets = false`, Terraform removes the infra secrets it wrote, as it always did). + +**A new app secret, or a feature that needs one:** add it to the manifest first (above). Put the value in `.env.` under its environment variable and `just secrets-push ` (it shows as `+` new in Key Vault), **then** turn the feature on in the tfvars and apply. The apply only references the secret, so it doesn't need the value. ### Pull and push an environment @@ -409,7 +411,7 @@ These local files are gitignored and written readable only by you. **You need:** All of them are gitignored. The share files are **read-only mirrors**: edit the real thing through Pixel or `just ha-devices`, not the mirror. If a mirror differs from the share, `just pull` shows the diff and asks before replacing it. -**`terraform.tfvars` holds settings, never secrets.** Discord IDs, URLs, feature switches and the like. Secret Terraform inputs (`discord_token`, `tailscale_auth_key`, `container_registry_password`, … anything the stack's `variables.tf` marks `sensitive`) go in `infra/envs//secrets.auto.tfvars` instead: Terraform loads it automatically, it's gitignored, and it never leaves your laptop. `just push` refuses a tfvars that sets a secret, sets a variable the stack doesn't declare, or names the other environment's Key Vault or storage account. It checks the file before showing anything, and the diff hides the value of every sensitive variable, so even a tfvars that wrongly holds a secret never prints it. +**`terraform.tfvars` holds settings, never secrets.** Discord IDs, URLs, feature switches and the like. Secret Terraform inputs (`tailscale_auth_key`, `container_registry_password`: anything the stack's `variables.tf` marks `sensitive`; app secrets are not Terraform inputs at all) go in `infra/envs//secrets.auto.tfvars` instead: Terraform loads it automatically, it's gitignored, and it never leaves your laptop. `just push` refuses a tfvars that sets a secret, sets a variable the stack doesn't declare, or names the other environment's Key Vault or storage account. It checks the file before showing anything, and the diff hides the value of every sensitive variable, so even a tfvars that wrongly holds a secret never prints it. **Change Terraform settings:** @@ -662,7 +664,7 @@ If `members.yaml` is lost and there's no backup, Pixel won't start (it fails clo **Bootstrap state:** the storage account `pixelbartfstate` has versioning and 14-day soft delete. Restore a previous `terraform.tfstate` blob if remote state is damaged. If bootstrap was never migrated off the laptop, that local file is the backup — migrate it. -**Azure app (`dev`):** recreate with `just tf-apply dev` after bootstrap exists (the resource group is already there). Restore Key Vault secret values from wherever you keep them (they are not in Terraform state; re-apply with `write_secrets` or `az keyvault secret set`). Restore `members.yaml` / HA files / `schedules.yaml` from Azure Backup of the file share (14 days) or a copy you kept. Practise this in `dev`. Prod is the same layout in `pixel-prod` ([`infra/envs/prod/README.md`](../infra/envs/prod/README.md)). +**Azure app (`dev`):** recreate with `just tf-apply dev` after bootstrap exists (the resource group is already there). Restore Key Vault secret values from wherever you keep them (they are not in Terraform state; app secrets with `just secrets-push dev` from your `.env.dev`, the Tailscale key and registry token with `write_secrets` or `az keyvault secret set`). Restore `members.yaml` / HA files / `schedules.yaml` from Azure Backup of the file share (14 days) or a copy you kept. Practise this in `dev`. Prod is the same layout in `pixel-prod` ([`infra/envs/prod/README.md`](../infra/envs/prod/README.md)). ## Keeping this up to date diff --git a/infra/README.md b/infra/README.md index 679e996..635a754 100644 --- a/infra/README.md +++ b/infra/README.md @@ -2,7 +2,7 @@ How the hosted pieces fit together (diagrams): [`docs/infra.md`](../docs/infra.md). -Terraform for Pixel on Azure. There is **no database**. Secret *values* never go in state. They may be passed as ephemeral write-only apply inputs (`write_secrets`) or set with `az` / the portal. +Terraform for Pixel on Azure. There is **no database**. Secret *values* never go in state. App secrets are written only by `just secrets-push` ([ADR 0017](../docs/adr/0017-one-writer-for-app-secrets.md)); the infra ones (`tailscale-auth-key`, `ghcr-pull-token`) may be passed as ephemeral write-only apply inputs (`write_secrets`) or set with `az`. | Path | What | Issue | | --- | --- | --- | diff --git a/infra/envs/dev/README.md b/infra/envs/dev/README.md index e75af9a..e5b66bf 100644 --- a/infra/envs/dev/README.md +++ b/infra/envs/dev/README.md @@ -49,7 +49,7 @@ just tf-plan dev just tf-apply dev ``` -The Container App references Key Vault secrets by URI. Values are either **ephemeral apply inputs** (`write_secrets = true`, not stored in state) or set later with `az` / the portal. `terraform.example.tfvars` leaves `write_secrets` off so CI plan cannot clobber a real vault. +The Container App references Key Vault secrets by URI. App secret values come from `just secrets-push dev` only ([ADR 0017](../../../docs/adr/0017-one-writer-for-app-secrets.md)); the Tailscale key and registry token are **ephemeral apply inputs** (`write_secrets = true`, not stored in state) or set with `az`. `terraform.example.tfvars` leaves `write_secrets` off so CI plan cannot clobber a real vault. `container_image` is the **first-create** pin for this **dev** stack only. After that, Terraform ignores the Pixel container image so a later apply cannot roll `just deploy-dev` backward (#73). Change the running image with `just deploy-dev` or `az containerapp update`, not a tfvars bump. Do not apply Terraform from CD to "fix" that drift. Prod (#12) apply will set `container_image` (no ignore on that resource). @@ -57,31 +57,28 @@ Do not run Pixel locally against the same Discord token while this app is up. ## Secrets (Key Vault) -**App secrets:** `just secrets-push dev` sets them in Key Vault directly. **For a write-only apply:** gitignored `secrets.auto.tfvars` (or `TF_VAR_discord_token`), never the shared `terraform.tfvars`. Values are write-only and are not stored in Terraform state. +**App secrets** (`discord-token`, `admins-yaml`, `home-assistant-token`, `mastodon-access-token`, `spacestate-user`, `spacestate-token`, `sentry-dsn`): `just secrets-push dev` is their only writer ([ADR 0017](../../../docs/adr/0017-one-writer-for-app-secrets.md)). Terraform has no inputs for them. Push them **before** the apply that turns their feature on; the apply only references them. + +**Infra secrets** (`tailscale-auth-key`, `ghcr-pull-token`): Terraform writes them when `write_secrets = true`, from gitignored `secrets.auto.tfvars` (or `TF_VAR_*`), never the shared `terraform.tfvars`. Values are write-only and are not stored in Terraform state. ```hcl -write_secrets = true -secrets_version = 1 -discord_token = "..." -admins_yaml_file = "../../../config/admins.yaml" +write_secrets = true +secrets_version = 1 +tailscale_auth_key = "..." # when tailscale_enabled +container_registry_password = "..." # when container_registry_server is set ``` -Bump `secrets_version` to rotate. HA / Sentry / Tailscale use `home_assistant_token`, `sentry_dsn`, and `tailscale_auth_key` the same way when those features are on. - -**Without `write_secrets`:** create the names yourself, then apply again: +Bump `secrets_version` to rotate them. **Without `write_secrets`:** set them yourself: ```sh vault=$(terraform -chdir=infra/envs/dev output -raw key_vault_name) - -az keyvault secret set --vault-name "$vault" --name discord-token --file - # paste token, Ctrl-D -az keyvault secret set --vault-name "$vault" --name admins-yaml --file config/admins.yaml # If GHCR is still private: # az keyvault secret set --vault-name "$vault" --name ghcr-pull-token --file - ``` Never commit secret values. Never put them in `terraform.example.tfvars`. -`admins.yaml` is mounted read-only at `/app/secrets/admins-yaml`. After changing it, bump `secrets_version` and re-apply, or set the secret with `az` and restart the Container App. Key Vault versions keep history. Azure Container Apps secret volumes also expose the other secrets as files in `/app/secrets/`; treat that directory as private. +`admins.yaml` is mounted read-only at `/app/secrets/admins-yaml`. After changing it, `just secrets-push dev` and restart (the tool offers `just restart dev`). Key Vault versions keep history. Azure Container Apps secret volumes also expose the other secrets as files in `/app/secrets/`; treat that directory as private. ## Volume files (Azure Files) @@ -126,7 +123,7 @@ Dev must not control the real doors. Use a test HA, or leave HA off. ## After apply, Discord still will not answer until -1. Key Vault secrets exist (at least `discord-token` and `admins-yaml`), via `write_secrets` or `az`. +1. Key Vault secrets exist (at least `discord-token` and `admins-yaml`), via `just secrets-push dev`. 2. `members.yaml` (and HA files if used) are on the share. 3. The image exists (`ghcr.io/pixelbar/pixel:`). If that GHCR package is still **private**, set `container_registry_server` / `container_registry_username` and create Key Vault `ghcr-pull-token` (a `read:packages` token; Azure otherwise gets 401). 4. `just register dev` has been run against the Dev guild (or `just deploy-dev` does it). diff --git a/infra/envs/dev/main.tf b/infra/envs/dev/main.tf index 560ca58..d9fa5c5 100644 --- a/infra/envs/dev/main.tf +++ b/infra/envs/dev/main.tf @@ -19,21 +19,12 @@ module "pixel" { home_assistant_url = var.home_assistant_url mastodon_instance = var.mastodon_instance spacestate_api_url = var.spacestate_api_url + spacestate_additional_headers_enabled = var.spacestate_additional_headers_enabled sentry_enabled = var.sentry_enabled tailscale_enabled = var.tailscale_enabled tailscale_image = var.tailscale_image purge_protection_enabled = false write_secrets = var.write_secrets secrets_version = var.secrets_version - discord_token = var.discord_token - admins_yaml = var.admins_yaml - admins_yaml_file = var.admins_yaml_file - home_assistant_token = var.home_assistant_token - mastodon_access_token = var.mastodon_access_token - spacestate_user = var.spacestate_user - spacestate_token = var.spacestate_token - spacestate_additional_headers = var.spacestate_additional_headers - spacestate_additional_headers_enabled = var.spacestate_additional_headers_enabled - sentry_dsn = var.sentry_dsn tailscale_auth_key = var.tailscale_auth_key } diff --git a/infra/envs/dev/outputs.tf b/infra/envs/dev/outputs.tf index c36affb..909d072 100644 --- a/infra/envs/dev/outputs.tf +++ b/infra/envs/dev/outputs.tf @@ -12,7 +12,7 @@ output "key_vault_name" { output "key_vault_secret_names" { value = module.pixel.key_vault_secret_names - description = "Secrets the Container App reads. write_secrets writes them from ephemeral inputs; otherwise create each name before the app can start." + description = "Secrets the Container App reads. Push the app ones with just secrets-push; write_secrets writes only tailscale-auth-key and ghcr-pull-token. Each must exist before the app can start." } output "data_storage_account_name" { diff --git a/infra/envs/dev/terraform.example.tfvars b/infra/envs/dev/terraform.example.tfvars index ed7f78a..78a206a 100644 --- a/infra/envs/dev/terraform.example.tfvars +++ b/infra/envs/dev/terraform.example.tfvars @@ -1,9 +1,8 @@ # Copy to terraform.tfvars and replace the fake Discord IDs with Pixel Dev. # terraform.tfvars is gitignored. # -# To write Key Vault secrets at apply (values are ephemeral / write-only and -# are not stored in state), set write_secrets = true and the inputs below in -# terraform.tfvars — never in this example file. CI plans with this file. +# App secrets (discord-token, admins-yaml, ...) go in Key Vault with +# just secrets-push dev, never through Terraform. CI plans with this file. subscription_id = "d150e252-e2f0-47fb-8a4a-c3f29e9aebd4" prefix = "pixel" @@ -43,17 +42,10 @@ discord_guild_id = "100000000000000002" sentry_enabled = false tailscale_enabled = false -# Leave false here. Real apply: write_secrets = true plus the values below in -# gitignored secrets.auto.tfvars, never in terraform.tfvars (just push refuses it). -# write_secrets = true -# secrets_version = 1 -# discord_token = "..." -# admins_yaml_file = "../../../config/admins.yaml" -# home_assistant_token = "..." -# mastodon_access_token = "..." -# spacestate_user = "..." -# spacestate_token = "..." -# spacestate_additional_headers = "{\"X-Example\": \"...\"}" -# sentry_dsn = "..." -# tailscale_auth_key = "..." +# write_secrets writes only the infra group (tailscale-auth-key, ghcr-pull-token) +# from gitignored secrets.auto.tfvars, never terraform.tfvars (just push refuses +# it). Leave it false here. +# write_secrets = true +# secrets_version = 1 +# tailscale_auth_key = "..." # container_registry_password = "..." diff --git a/infra/envs/dev/variables.tf b/infra/envs/dev/variables.tf index ce15810..711b803 100644 --- a/infra/envs/dev/variables.tf +++ b/infra/envs/dev/variables.tf @@ -119,7 +119,7 @@ variable "spacestate_api_url" { variable "write_secrets" { type = bool default = false - description = "If true, write ephemeral secret inputs into Key Vault (not stored in state). Leave false in terraform.example.tfvars so CI plan cannot clobber a real vault." + description = "If true, write the infra-group secrets (tailscale-auth-key, ghcr-pull-token) into Key Vault from ephemeral inputs (not stored in state). Never the app group: just secrets-push owns those (ADR 0017). Leave false in the example tfvars so a CI plan cannot clobber a real vault." } variable "secrets_version" { @@ -128,82 +128,12 @@ variable "secrets_version" { description = "Bump to rotate write-only Key Vault secret values. Not a secret." } -variable "discord_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "DISCORD_TOKEN. Used when write_secrets is true. Gitignored tfvars or TF_VAR_discord_token. Never stored in state." -} - -variable "admins_yaml" { - type = string - ephemeral = true - sensitive = true - default = null - description = "Contents of admins.yaml. Used when write_secrets is true and admins_yaml_file is unset." -} - -variable "admins_yaml_file" { - type = string - default = null - description = "Path to admins.yaml (gitignored). Contents are passed write-only into Key Vault; the path is not a secret." -} - -variable "home_assistant_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "HOME_ASSISTANT_TOKEN. Required when write_secrets is true and home_assistant_url is set." -} - -variable "mastodon_access_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "MASTODON_ACCESS_TOKEN. Required when write_secrets is true and mastodon_instance is set." -} - -variable "spacestate_user" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SPACESTATE_USER. Required when write_secrets is true and spacestate_api_url is set." -} - -variable "spacestate_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SPACESTATE_TOKEN. Required when write_secrets is true and spacestate_api_url is set." -} - -variable "spacestate_additional_headers" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SPACESTATE_ADDITIONAL_HEADERS: a JSON object of extra headers for the space state API. Required when write_secrets is true and spacestate_additional_headers_enabled is true." -} - variable "spacestate_additional_headers_enabled" { type = bool default = false description = "If true (and spacestate_api_url is set), the app reads the extra headers from Key Vault secret spacestate-additional-headers." } -variable "sentry_dsn" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SENTRY_DSN. Required when write_secrets is true and sentry_enabled is true." -} - variable "tailscale_auth_key" { type = string ephemeral = true diff --git a/infra/envs/prod/README.md b/infra/envs/prod/README.md index 5186ac1..e0ae4c8 100644 --- a/infra/envs/prod/README.md +++ b/infra/envs/prod/README.md @@ -31,7 +31,7 @@ The app sets `PIXEL_RUNTIME=cloud` and `PIXEL_ENV=prod`. There is **no database* 3. A **new** Discord application (not Pixel Dev). Invite it with scopes `bot` and `applications.commands`. Put that application ID and guild ID in gitignored `terraform.tfvars`. See [Adding Pixel to a Discord server](../../../docs/runbook.md#adding-pixel-to-a-discord-server). 4. Azure CLI, signed in as someone who can write role assignments on `pixel-prod`. 5. Terraform 1.11+ (`just tf-validate` uses 1.16.5, same as CI). -6. `just pull prod` to get the shared `terraform.tfvars` (Key Vault `terraform-tfvars`). The very first time, copy `terraform.example.tfvars` to `terraform.tfvars` and `just push prod`. Secret values go in gitignored `secrets.auto.tfvars` with `write_secrets = true`, or in `TF_VAR_*` — never in `terraform.tfvars`, never from `pixel-dev-kv`. +6. `just pull prod` to get the shared `terraform.tfvars` (Key Vault `terraform-tfvars`). The very first time, copy `terraform.example.tfvars` to `terraform.tfvars` and `just push prod`. App secret values go in `.env.prod` / `config/admins.prod.yaml` and reach Key Vault with `just secrets-push prod` (maintainers only). Infra secret inputs go in gitignored `secrets.auto.tfvars` with `write_secrets = true`, or in `TF_VAR_*`. Never in `terraform.tfvars`, never from `pixel-dev-kv`. ## Apply (maintainer only) @@ -45,16 +45,20 @@ PIXEL_APPLY_PROD=1 just tf-apply prod ## Secrets (Key Vault) -Same names as `dev`. Preferred local apply: +Same names as `dev`. **App secrets** (`discord-token` for the production bot only, `admins-yaml` with the production list, `sentry-dsn`, `spacestate-user`, `spacestate-token`, …): `just secrets-push prod` is their only writer ([ADR 0017](../../../docs/adr/0017-one-writer-for-app-secrets.md)). Push them **before** the apply that turns their feature on. Coding agents never run it. + +**Infra secrets** (`tailscale-auth-key`, `ghcr-pull-token`), only when those features are on: ```hcl -write_secrets = true -secrets_version = 1 -discord_token = "..." # production bot only -admins_yaml_file = "../../../config/admins.yaml" # production list, not Dev +write_secrets = true +secrets_version = 1 +tailscale_auth_key = "..." +container_registry_password = "..." ``` -Or `az keyvault secret set --vault-name pixel-prod-kv --name discord-token --file -`. Never commit values. +Never commit values. + +**The first plan after ADR 0017** lists the app secrets Terraform had written as "will no longer be managed by Terraform, but will not be destroyed". Nothing leaves Key Vault. If it plans to **destroy** an `azurerm_key_vault_secret`, stop. ## Volume files @@ -67,7 +71,7 @@ az storage file upload --auth-mode login --account-name "$account" --share-name ## After apply, Discord still will not answer until -1. Key Vault secrets exist (`discord-token`, `admins-yaml`) for the **prod** bot. +1. Key Vault secrets exist (`discord-token`, `admins-yaml`) for the **prod** bot (`just secrets-push prod`). 2. `members.yaml` is on the prod share. 3. The image exists (`ghcr.io/pixelbar/pixel:` from a Release, or the `:main` tag from CD). 4. Commands are registered against the **prod** guild. A Release does this in its `register-prod` job once `github_register_principal_id` is set to `pixel-github-prod`'s principal ID (`az identity show -g pixel-bootstrap -n pixel-github-prod --query principalId -o tsv`) and applied; it can read `discord-token` and nothing else ([ADR 0014](../../../docs/adr/0014-release-registers-commands.md)). By hand: `just register prod` with `.env.prod`, never `.env` or `.env.dev`. diff --git a/infra/envs/prod/main.tf b/infra/envs/prod/main.tf index 8d6677e..185f008 100644 --- a/infra/envs/prod/main.tf +++ b/infra/envs/prod/main.tf @@ -19,22 +19,13 @@ module "pixel" { home_assistant_url = var.home_assistant_url mastodon_instance = var.mastodon_instance spacestate_api_url = var.spacestate_api_url + spacestate_additional_headers_enabled = var.spacestate_additional_headers_enabled sentry_enabled = var.sentry_enabled tailscale_enabled = var.tailscale_enabled tailscale_image = var.tailscale_image purge_protection_enabled = true write_secrets = var.write_secrets secrets_version = var.secrets_version - discord_token = var.discord_token - admins_yaml = var.admins_yaml - admins_yaml_file = var.admins_yaml_file - home_assistant_token = var.home_assistant_token - mastodon_access_token = var.mastodon_access_token - spacestate_user = var.spacestate_user - spacestate_token = var.spacestate_token - spacestate_additional_headers = var.spacestate_additional_headers - spacestate_additional_headers_enabled = var.spacestate_additional_headers_enabled - sentry_dsn = var.sentry_dsn tailscale_auth_key = var.tailscale_auth_key github_register_principal_id = var.github_register_principal_id } diff --git a/infra/envs/prod/outputs.tf b/infra/envs/prod/outputs.tf index c36affb..909d072 100644 --- a/infra/envs/prod/outputs.tf +++ b/infra/envs/prod/outputs.tf @@ -12,7 +12,7 @@ output "key_vault_name" { output "key_vault_secret_names" { value = module.pixel.key_vault_secret_names - description = "Secrets the Container App reads. write_secrets writes them from ephemeral inputs; otherwise create each name before the app can start." + description = "Secrets the Container App reads. Push the app ones with just secrets-push; write_secrets writes only tailscale-auth-key and ghcr-pull-token. Each must exist before the app can start." } output "data_storage_account_name" { diff --git a/infra/envs/prod/terraform.example.tfvars b/infra/envs/prod/terraform.example.tfvars index f66e859..038ef0d 100644 --- a/infra/envs/prod/terraform.example.tfvars +++ b/infra/envs/prod/terraform.example.tfvars @@ -36,11 +36,11 @@ discord_guild_id = "100000000000000002" sentry_enabled = false tailscale_enabled = false -# Leave false here. Real apply: write_secrets = true plus the values below in -# gitignored secrets.auto.tfvars, never in terraform.tfvars (just push refuses it). -# Never paste the Pixel Dev token. -# write_secrets = true -# secrets_version = 1 -# discord_token = "..." -# admins_yaml_file = "../../../config/admins.yaml" -# sentry_dsn = "..." +# App secrets (discord-token, admins-yaml, sentry-dsn, ...) are never Terraform +# inputs: set them with just secrets-push prod (a maintainer, never an agent). +# write_secrets writes only the infra group, from gitignored secrets.auto.tfvars, +# never terraform.tfvars (just push refuses it). Leave it false here. +# write_secrets = true +# secrets_version = 1 +# tailscale_auth_key = "..." +# container_registry_password = "..." diff --git a/infra/envs/prod/variables.tf b/infra/envs/prod/variables.tf index a30cab2..ef3d056 100644 --- a/infra/envs/prod/variables.tf +++ b/infra/envs/prod/variables.tf @@ -119,7 +119,7 @@ variable "spacestate_api_url" { variable "write_secrets" { type = bool default = false - description = "If true, write ephemeral secret inputs into Key Vault (not stored in state). Leave false in terraform.example.tfvars so CI plan cannot clobber a real vault." + description = "If true, write the infra-group secrets (tailscale-auth-key, ghcr-pull-token) into Key Vault from ephemeral inputs (not stored in state). Never the app group: just secrets-push owns those (ADR 0017). Leave false in the example tfvars so a CI plan cannot clobber a real vault." } variable "secrets_version" { @@ -128,82 +128,12 @@ variable "secrets_version" { description = "Bump to rotate write-only Key Vault secret values. Not a secret." } -variable "discord_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "DISCORD_TOKEN. Used when write_secrets is true. Gitignored tfvars or TF_VAR_discord_token. Never stored in state." -} - -variable "admins_yaml" { - type = string - ephemeral = true - sensitive = true - default = null - description = "Contents of admins.yaml. Used when write_secrets is true and admins_yaml_file is unset." -} - -variable "admins_yaml_file" { - type = string - default = null - description = "Path to admins.yaml (gitignored). Contents are passed write-only into Key Vault; the path is not a secret." -} - -variable "home_assistant_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "HOME_ASSISTANT_TOKEN. Required when write_secrets is true and home_assistant_url is set." -} - -variable "mastodon_access_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "MASTODON_ACCESS_TOKEN. Required when write_secrets is true and mastodon_instance is set." -} - -variable "spacestate_user" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SPACESTATE_USER. Required when write_secrets is true and spacestate_api_url is set." -} - -variable "spacestate_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SPACESTATE_TOKEN. Required when write_secrets is true and spacestate_api_url is set." -} - -variable "spacestate_additional_headers" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SPACESTATE_ADDITIONAL_HEADERS: a JSON object of extra headers for the space state API. Required when write_secrets is true and spacestate_additional_headers_enabled is true." -} - variable "spacestate_additional_headers_enabled" { type = bool default = false description = "If true (and spacestate_api_url is set), the app reads the extra headers from Key Vault secret spacestate-additional-headers." } -variable "sentry_dsn" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SENTRY_DSN. Required when write_secrets is true and sentry_enabled is true." -} - variable "tailscale_auth_key" { type = string ephemeral = true diff --git a/infra/modules/pixel/README.md b/infra/modules/pixel/README.md index 5f91a54..a8299a4 100644 --- a/infra/modules/pixel/README.md +++ b/infra/modules/pixel/README.md @@ -2,6 +2,6 @@ Reusable Terraform for one Pixel environment (Container App, Azure Files volume, Key Vault, optional Tailscale sidecar). Called from `infra/envs/dev` and `infra/envs/prod`. -Secret values may be passed as ephemeral write-only inputs (`write_secrets`); they are never stored in state. Max replicas is 1. No database. On **dev**, after first create, apply ignores the Pixel container image (`just deploy-dev` / `az` owns the pin). On **prod** (#12), apply sets `container_image`. +Infra secret values (`tailscale-auth-key`, `ghcr-pull-token`) may be passed as ephemeral write-only inputs (`write_secrets`); app secrets are referenced only and written by `just secrets-push` (ADR 0017). Values are never stored in state. Max replicas is 1. No database. On **dev**, after first create, apply ignores the Pixel container image (`just deploy-dev` / `az` owns the pin). On **prod** (#12), apply sets `container_image`. See [`infra/envs/dev/README.md`](../../envs/dev/README.md) and [ADR 0009](../../../docs/adr/0009-container-apps-dev.md). diff --git a/infra/modules/pixel/container_app.tf b/infra/modules/pixel/container_app.tf index c8d82ed..2ada129 100644 --- a/infra/modules/pixel/container_app.tf +++ b/infra/modules/pixel/container_app.tf @@ -224,7 +224,7 @@ resource "azurerm_container_app" "this" { depends_on = [ azurerm_role_assignment.kv_secrets_app, azurerm_container_app_environment_storage.persist, - azurerm_key_vault_secret.managed, + azurerm_key_vault_secret.infra, ] } @@ -407,6 +407,6 @@ resource "azurerm_container_app" "managed" { depends_on = [ azurerm_role_assignment.kv_secrets_app, azurerm_container_app_environment_storage.persist, - azurerm_key_vault_secret.managed, + azurerm_key_vault_secret.infra, ] } diff --git a/infra/modules/pixel/identity.tf b/infra/modules/pixel/identity.tf index 453868a..3fdb3ca 100644 --- a/infra/modules/pixel/identity.tf +++ b/infra/modules/pixel/identity.tf @@ -27,7 +27,8 @@ resource "azurerm_role_assignment" "files_applyer" { } # The Release workflow's GitHub identity may read discord-token, and only that -# secret, so it can register slash commands after a roll (ADR 0014). +# secret, so it can register slash commands after a roll (ADR 0014). just secrets-push +# creates discord-token (ADR 0017); push it before the first apply that sets this. resource "azurerm_role_assignment" "kv_discord_token_register" { count = var.github_register_principal_id != null ? 1 : 0 @@ -35,6 +36,4 @@ resource "azurerm_role_assignment" "kv_discord_token_register" { role_definition_name = "Key Vault Secrets User" principal_id = var.github_register_principal_id skip_service_principal_aad_check = true - - depends_on = [azurerm_key_vault_secret.managed] } diff --git a/infra/modules/pixel/locals.tf b/infra/modules/pixel/locals.tf index 8087b69..9c15d88 100644 --- a/infra/modules/pixel/locals.tf +++ b/infra/modules/pixel/locals.tf @@ -41,6 +41,10 @@ locals { kv_secret_names = toset([for s in local.enabled_secrets : s.vault]) + # The only secrets Terraform writes (write_secrets). The app group is written by + # just secrets-push / just push alone (ADR 0017); Terraform only references it. + kv_infra_secret_names = toset([for s in local.enabled_secrets : s.vault if s.group == "infra"]) + # URIs only. Terraform never reads the secret values into state. key_vault_secret_ids = { for name in local.kv_secret_names : diff --git a/infra/modules/pixel/outputs.tf b/infra/modules/pixel/outputs.tf index 2e4fcec..eb21b1c 100644 --- a/infra/modules/pixel/outputs.tf +++ b/infra/modules/pixel/outputs.tf @@ -13,7 +13,7 @@ output "container_app_environment_name" { output "key_vault_name" { value = azurerm_key_vault.this.name - description = "Secret values are write-only Terraform inputs (write_secrets) or az / the portal. Never stored in state." + description = "App secret values come from just secrets-push; infra ones from write-only Terraform inputs (write_secrets) or az. Never stored in state." } output "key_vault_uri" { @@ -22,7 +22,7 @@ output "key_vault_uri" { output "key_vault_secret_names" { value = sort(tolist(local.kv_secret_names)) - description = "Secrets the Container App reads. write_secrets writes them from ephemeral inputs; otherwise create each name before the app can start." + description = "Secrets the Container App reads. Push the app ones with just secrets-push; write_secrets writes only tailscale-auth-key and ghcr-pull-token. Each must exist before the app can start." } output "data_storage_account_name" { diff --git a/infra/modules/pixel/secrets.tf b/infra/modules/pixel/secrets.tf index e78ca17..cfe0ee3 100644 --- a/infra/modules/pixel/secrets.tf +++ b/infra/modules/pixel/secrets.tf @@ -1,45 +1,58 @@ -# Write-only secret values. Terraform never stores them in state. -# Set write_secrets = true and pass the ephemeral inputs, or leave it false -# and create the same names with az / the portal. +# Write-only secret values for the manifest's infra group (tailscale-auth-key, +# ghcr-pull-token). Terraform never stores them in state. Set write_secrets = true +# and pass the ephemeral inputs, or leave it false and create the same names with +# az / the portal. +# +# The app group (discord-token, admins-yaml, ...) is written only by +# just secrets-push / just push (ADR 0016, ADR 0017). Terraform references those +# secrets from the Container App but never writes them. locals { managed_secret_values = { - "discord-token" = var.discord_token - "admins-yaml" = var.admins_yaml - "home-assistant-token" = var.home_assistant_token - "mastodon-access-token" = var.mastodon_access_token - "spacestate-user" = var.spacestate_user - "spacestate-token" = var.spacestate_token - "spacestate-additional-headers" = var.spacestate_additional_headers - "sentry-dsn" = var.sentry_dsn - "tailscale-auth-key" = var.tailscale_auth_key - "ghcr-pull-token" = var.container_registry_password + "tailscale-auth-key" = var.tailscale_auth_key + "ghcr-pull-token" = var.container_registry_password } } -resource "azurerm_key_vault_secret" "managed" { - for_each = var.write_secrets ? local.kv_secret_names : toset([]) - - name = each.key - key_vault_id = azurerm_key_vault.this.id - content_type = each.key == "admins-yaml" ? "text/yaml" : "text/plain" - value_wo = ( - each.key == "admins-yaml" && var.admins_yaml_file != null ? - file(var.admins_yaml_file) : - local.managed_secret_values[each.key] - ) +resource "azurerm_key_vault_secret" "infra" { + for_each = var.write_secrets ? local.kv_infra_secret_names : toset([]) + + name = each.key + key_vault_id = azurerm_key_vault.this.id + content_type = "text/plain" + value_wo = local.managed_secret_values[each.key] value_wo_version = var.secrets_version depends_on = [azurerm_role_assignment.kv_admin_applyer] lifecycle { precondition { - condition = ( - each.key == "admins-yaml" ? - (var.admins_yaml_file != null || (var.admins_yaml != null && var.admins_yaml != "")) : - (local.managed_secret_values[each.key] != null && local.managed_secret_values[each.key] != "") - ) - error_message = "write_secrets = true requires an input for Key Vault secret ${each.key} (write-only, not stored in state). Set it in terraform.tfvars (gitignored) or TF_VAR_*." + condition = local.managed_secret_values[each.key] != null && local.managed_secret_values[each.key] != "" + error_message = "write_secrets = true requires an input for Key Vault secret ${each.key} (write-only, not stored in state). Set it in gitignored secrets.auto.tfvars or TF_VAR_*." } } } + +# Until ADR 0017, one resource wrote every secret. The infra ones keep their state +# under the new name, so nothing is rewritten. +moved { + from = azurerm_key_vault_secret.managed["tailscale-auth-key"] + to = azurerm_key_vault_secret.infra["tailscale-auth-key"] +} + +moved { + from = azurerm_key_vault_secret.managed["ghcr-pull-token"] + to = azurerm_key_vault_secret.infra["ghcr-pull-token"] +} + +# Whatever is left under the old name is an app secret. Terraform forgets it and +# leaves it in Key Vault, where just secrets-push owns it. Plan shows "will no +# longer be managed by Terraform, but will not be destroyed". Keep this block +# until every stack has applied it. +removed { + from = azurerm_key_vault_secret.managed + + lifecycle { + destroy = false + } +} diff --git a/infra/modules/pixel/variables.tf b/infra/modules/pixel/variables.tf index 2a9934a..1134387 100644 --- a/infra/modules/pixel/variables.tf +++ b/infra/modules/pixel/variables.tf @@ -172,7 +172,7 @@ variable "github_register_principal_id" { variable "write_secrets" { type = bool default = false - description = "If true, write ephemeral secret inputs into Key Vault (write-only; values are not stored in state). If false, set secrets with az or the portal before the Container App will start. Leave false in CI example tfvars so a plan cannot clobber a real vault." + description = "If true, write the infra-group secrets (tailscale-auth-key, ghcr-pull-token) into Key Vault from ephemeral inputs (not stored in state). Never the app group: just secrets-push owns those (ADR 0017). Leave false in the example tfvars so a CI plan cannot clobber a real vault." } variable "secrets_version" { @@ -186,82 +186,12 @@ variable "secrets_version" { } } -variable "discord_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "DISCORD_TOKEN. Written to Key Vault when write_secrets is true. Never stored in state." -} - -variable "admins_yaml" { - type = string - ephemeral = true - sensitive = true - default = null - description = "Contents of admins.yaml. Written to Key Vault when write_secrets is true and admins_yaml_file is unset. Never stored in state." -} - -variable "admins_yaml_file" { - type = string - default = null - description = "Path to admins.yaml. Contents are written write-only into Key Vault; the path is not a secret." -} - -variable "home_assistant_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "HOME_ASSISTANT_TOKEN. Required when write_secrets is true and home_assistant_url is set." -} - -variable "mastodon_access_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "MASTODON_ACCESS_TOKEN. Required when write_secrets is true and mastodon_instance is set." -} - -variable "spacestate_user" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SPACESTATE_USER. Required when write_secrets is true and spacestate_api_url is set." -} - -variable "spacestate_token" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SPACESTATE_TOKEN. Required when write_secrets is true and spacestate_api_url is set." -} - -variable "spacestate_additional_headers" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SPACESTATE_ADDITIONAL_HEADERS: a JSON object of extra headers for the space state API. Required when write_secrets is true and spacestate_additional_headers_enabled is true." -} - variable "spacestate_additional_headers_enabled" { type = bool default = false description = "If true (and spacestate_api_url is set), the app reads SPACESTATE_ADDITIONAL_HEADERS from Key Vault secret spacestate-additional-headers." } -variable "sentry_dsn" { - type = string - ephemeral = true - sensitive = true - default = null - description = "SENTRY_DSN. Required when write_secrets is true and sentry_enabled is true." -} - variable "tailscale_auth_key" { type = string ephemeral = true diff --git a/infra/secrets.manifest.yaml b/infra/secrets.manifest.yaml index fb55268..94e2451 100644 --- a/infra/secrets.manifest.yaml +++ b/infra/secrets.manifest.yaml @@ -7,9 +7,10 @@ # - scripts/secrets-manifest.test.ts, which fails when this file, config.ts, .env.example, # the Terraform variables or the runbook disagree. See ADR 0015. # -# To add a secret: add an entry here, add its Terraform variable and write-only entry -# (infra/modules/pixel/secrets.tf, both envs), add the setting to src/config.ts and -# .env.example, and describe it in docs/runbook.md. The test names whatever you missed. +# To add a secret: add an entry here, add the setting to src/config.ts and .env.example, +# and describe it in docs/runbook.md. An infra-group secret also needs its Terraform +# variable and write-only entry (infra/modules/pixel/secrets.tf, both envs); an app one +# must not have them (ADR 0017). The test names whatever you missed. version: 1 @@ -26,8 +27,9 @@ features: container-registry: Private image registry (container_registry_server is set) # group -# app belongs to the Pixel app. These are the ones just secrets-pull / secrets-push sync. -# infra belongs to the platform around the app. Terraform keeps writing these. +# app belongs to the Pixel app. Only just secrets-push / just push write these (ADR 0017); +# Terraform references them from the Container App but never writes them. +# infra belongs to the platform around the app. Terraform writes these (write_secrets). # tooling read only by the laptop tools (just pull / just push). Terraform and the Container # App never see these: Terraform skips the group, so the app can't reference them. # delivery diff --git a/scripts/secrets-manifest.test.ts b/scripts/secrets-manifest.test.ts index 9ad3ece..ba2759d 100644 --- a/scripts/secrets-manifest.test.ts +++ b/scripts/secrets-manifest.test.ts @@ -8,6 +8,7 @@ import { parseSecretsManifest, SecretsManifestError, terraformSecrets, + terraformWrittenSecrets, } from "./secrets-manifest.ts"; const read = (file: string) => readFileSync(file, "utf8"); @@ -216,12 +217,31 @@ describe("the real manifest", () => { it("lists every secret Terraform can write, and nothing else", () => { expect(writeMap.map((w) => w.vault).sort()).toEqual( - terraformSecrets(manifest) + terraformWrittenSecrets(manifest) .map((s) => s.vault) .sort(), ); }); + it("never lets Terraform write the app group, which just secrets-push owns", () => { + expect(localsTf).toMatch( + /kv_infra_secret_names = toset\(\[for s in local\.enabled_secrets : s\.vault if s\.group == "infra"\]\)/, + ); + expect(secretsTf).toMatch(/for_each = var\.write_secrets \? local\.kv_infra_secret_names :/); + for (const s of manifest.secrets.filter((s) => s.group === "app")) { + expect(writeMap.map((w) => w.vault)).not.toContain(s.vault); + } + // The old all-secrets resource leaves state without deleting anything from Key Vault. + expect(secretsTf).toMatch( + /removed \{\s*from = azurerm_key_vault_secret\.managed\s*lifecycle \{\s*destroy = false\s*\}/, + ); + }); + + it("still references every enabled secret from the Container App, app group included", () => { + expect(terraformSecrets(manifest).some((s) => s.group === "app")).toBe(true); + expect(localsTf).toMatch(/for name in local\.kv_secret_names :/); + }); + it("keeps the tooling group out of Terraform", () => { expect(localsTf).toMatch(/if s\.group != "tooling" && local\.secret_features\[s\.feature\]/); for (const s of manifest.secrets.filter((s) => s.group === "tooling")) { diff --git a/scripts/secrets-manifest.ts b/scripts/secrets-manifest.ts index 07403c8..5473820 100644 --- a/scripts/secrets-manifest.ts +++ b/scripts/secrets-manifest.ts @@ -131,6 +131,14 @@ export function terraformSecrets(manifest: SecretsManifest): SecretEntry[] { return manifest.secrets.filter((s) => s.group !== "tooling"); } +/** + * The secrets Terraform may write (write_secrets): the infra group only. The app group is + * written by just secrets-push / just push alone (ADR 0017). + */ +export function terraformWrittenSecrets(manifest: SecretsManifest): SecretEntry[] { + return manifest.secrets.filter((s) => s.group === "infra"); +} + /** The secrets a stack needs, given which features it turns on. `always` is implied. */ export function enabledSecrets( manifest: SecretsManifest, diff --git a/scripts/secrets-sync-lib.test.ts b/scripts/secrets-sync-lib.test.ts index 7ee28d5..768625a 100644 --- a/scripts/secrets-sync-lib.test.ts +++ b/scripts/secrets-sync-lib.test.ts @@ -332,10 +332,12 @@ variable "discord_token" { expect([...variables.sensitive]).toEqual(["discord_token"]); }); - it("reads the real stacks' variables, with the Discord token as a secret", () => { + it("reads the real stacks' variables, with the Tailscale key as a secret", () => { for (const env of ["dev", "prod"] as const) { const real = stackVariables(readFileSync(`infra/envs/${env}/variables.tf`, "utf8")); - expect(real.sensitive.has("discord_token")).toBe(true); + expect(real.sensitive.has("tailscale_auth_key")).toBe(true); + // App secrets are no longer Terraform inputs at all (ADR 0017). + expect(real.all.has("discord_token")).toBe(false); expect(real.sensitive.has("discord_app_id")).toBe(false); expect(real.all.has("discord_app_id")).toBe(true); } diff --git a/scripts/secrets-sync.ts b/scripts/secrets-sync.ts index 5241044..952dd72 100644 --- a/scripts/secrets-sync.ts +++ b/scripts/secrets-sync.ts @@ -424,10 +424,6 @@ async function push( ); } if (!changing.some((e) => e.group === "app")) return; - say( - "If this env's tfvars set write_secrets = true, update secrets.auto.tfvars too, or the next apply that bumps secrets_version puts the old values back.", - ); - say(); say(`${vault.app} reads secrets when it starts, so it keeps the old values until it restarts.`); const restart = await confirm({ message: `Restart ${vault.app} now (just restart ${target})?`,