diff --git a/docs/adopting.md b/docs/adopting.md index 99bec61..fe1b4de 100644 --- a/docs/adopting.md +++ b/docs/adopting.md @@ -214,6 +214,153 @@ marks alphas, betas and release candidates as pre-releases. `publish-release` pu the draft. All three default to the tag that triggered the run (`tag` input) and to `github.token` (`token` input), and need `pipx`, which GitHub-hosted runners have. +### Publishing to AWS CodeArtifact + +A package for the organisation's private index on AWS CodeArtifact keeps the same +structure. Its upload job assumes the upload role with GitHub's OIDC token and runs +`uv publish`; the `CODEARTIFACT_*` variables come from the organisation +([One-time organisation setup](organisation-setup.md), step 5), so no repository's +`pyproject.toml` changes. A build that needs private dependencies assumes +`vars.CODEARTIFACT_READ_ROLE` the same way before building. + +- **A failed upload can be re-run.** The job hands uv the index as a configuration file + (`--index`), and uv reads it with the same credentials to skip the files already + there. `UV_PUBLISH_URL` with `--check-url` alone doesn't do this for a private index: + uv reads the check URL without credentials. +- **No PEP 740 attestations** (`--no-attestations`): they are verified and served by + indexes with Trusted Publishing, such as PyPI, and CodeArtifact has none. The job + attests the files with GitHub's build provenance instead, which works with any index: + `gh attestation verify --owner Algorithmiq`. In private and internal + repositories this needs GitHub Enterprise Cloud. + +```yaml +# Publishing to AWS CodeArtifact: see https://github.com/Algorithmiq/release-tools#readme +# +# Runs on the v* tags that the release workflow pushes: build, draft the GitHub release +# with every file attached, upload to the organisation's private package index on AWS +# CodeArtifact, then publish the GitHub release, which makes it immutable. Never on +# `release: published`. Only the build job is repository-specific; the CODEARTIFACT_* +# variables are set once for the organisation. +name: Publish + +on: + push: + tags: ["v*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # A publish run must never be cut short half-way through. + cancel-in-progress: false + +jobs: + build: + name: Build sdist and wheel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@v10.3.2 + - name: Build distributions + run: uv build --out-dir dist + - name: Check the version matches the release tag + run: | + expected="${GITHUB_REF_NAME#v}" + wheels=(dist/*.whl) + actual="$(basename "${wheels[0]}" | cut -d- -f2)" + if [[ "$actual" != "$expected" ]]; then + echo "::error::built version is '$actual', but the release tag is '$expected'" + exit 1 + fi + - uses: actions/upload-artifact@v7.0.2 + with: + name: dist + path: dist/* + + draft_release: + # Immutable releases: files can only be attached while the release is a draft. + name: Create the draft GitHub release + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 # git-cliff needs the full history and the tags + - uses: actions/download-artifact@v8.0.2 + with: + name: dist + path: dist + - id: notes + uses: Algorithmiq/release-tools/actions/release-notes@v1.2.0 + - uses: Algorithmiq/release-tools/actions/draft-release@v1.2.0 + with: + notes-file: ${{ steps.notes.outputs.path }} + files: | + dist/* + + upload_codeartifact: + name: Upload to AWS CodeArtifact + needs: draft_release + runs-on: ubuntu-latest + permissions: + id-token: write # the upload role and the attestations use GitHub's OIDC token + attestations: write + contents: read + steps: + - uses: actions/download-artifact@v8.0.2 + with: + name: dist + path: dist + # Build provenance, stored by GitHub: `gh attestation verify --owner `. + - uses: actions/attest-build-provenance@v4.2.2 + with: + subject-path: "dist/*" + - uses: aws-actions/configure-aws-credentials@v6.3.0 + with: + role-to-assume: ${{ vars.CODEARTIFACT_WRITE_ROLE }} + aws-region: ${{ vars.CODEARTIFACT_REGION }} + - uses: astral-sh/setup-uv@v10.3.2 + with: + enable-cache: false + - name: Upload with uv publish + env: + DOMAIN: ${{ vars.CODEARTIFACT_DOMAIN }} + DOMAIN_OWNER: ${{ vars.CODEARTIFACT_DOMAIN_OWNER }} + REPOSITORY: ${{ vars.CODEARTIFACT_REPOSITORY }} + run: | + token="$(aws codeartifact get-authorization-token \ + --domain "$DOMAIN" --domain-owner "$DOMAIN_OWNER" \ + --query authorizationToken --output text)" + echo "::add-mask::$token" + endpoint="$(aws codeartifact get-repository-endpoint \ + --domain "$DOMAIN" --domain-owner "$DOMAIN_OWNER" --repository "$REPOSITORY" \ + --format pypi --query repositoryEndpoint --output text)" + endpoint="${endpoint%/}/" + # With --index, uv reads the index, with its credentials, and skips the files + # already there, so a failed run can be re-run. + config="$RUNNER_TEMP/codeartifact.toml" + printf '[[index]]\nname = "codeartifact"\nurl = "%s"\npublish-url = "%s"\nexplicit = true\n' \ + "${endpoint}simple/" "$endpoint" > "$config" + export UV_INDEX_CODEARTIFACT_USERNAME=aws UV_INDEX_CODEARTIFACT_PASSWORD="$token" + uv publish --config-file "$config" --index codeartifact \ + --trusted-publishing never --no-attestations dist/* + + publish_release: + # Publishing locks the tag and the files and creates GitHub's release attestation. + name: Publish the GitHub release + needs: upload_codeartifact + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: Algorithmiq/release-tools/actions/publish-release@v1.2.0 +``` + ## `justfile`: preview the changelog `just changelog` shows what the next release's entry will look like, with the central diff --git a/docs/organisation-setup.md b/docs/organisation-setup.md index 3a66925..5d1c2e7 100644 --- a/docs/organisation-setup.md +++ b/docs/organisation-setup.md @@ -50,3 +50,19 @@ An organisation owner does this once. The commands need `gh` with the `admin:org Nothing in the workflows changes: they read release-tools with a read-only token of the release bot either way (see [Design](design.md)). +5. **CodeArtifact variables**, for repositories that publish to the organisation's + private package index on AWS CodeArtifact. Their publish workflow reads them as + `vars.CODEARTIFACT_*`; visibility `private` makes them available to every private and + internal repository, with no list to maintain: + + ```bash + gh variable set CODEARTIFACT_DOMAIN --org Algorithmiq --visibility private --body "" + gh variable set CODEARTIFACT_DOMAIN_OWNER --org Algorithmiq --visibility private --body "" + gh variable set CODEARTIFACT_REGION --org Algorithmiq --visibility private --body "" + gh variable set CODEARTIFACT_REPOSITORY --org Algorithmiq --visibility private --body "" + gh variable set CODEARTIFACT_READ_ROLE --org Algorithmiq --visibility private --body "" + gh variable set CODEARTIFACT_WRITE_ROLE --org Algorithmiq --visibility private --body "" + ``` + + The workflows assume the roles with GitHub's OIDC token, so the roles' trust policies + must accept the adopting repositories (for example `repo:Algorithmiq/*`). diff --git a/docs/repository-setup.md b/docs/repository-setup.md index 4c8c4fe..605be89 100644 --- a/docs/repository-setup.md +++ b/docs/repository-setup.md @@ -64,9 +64,11 @@ merging the PR that adds the stubs, 8 after it. EOF ``` -7. **PyPI Trusted Publishing** for the publish workflow: on PyPI, add a (pending) - publisher with the repository, the workflow file name and the environment (`pypi` in - the example). The first pre-release is published to PyPI straight away. +7. **The package index** of the publish workflow. PyPI: add a (pending) Trusted + Publisher with the repository, the workflow file name and the environment (`pypi` in + the example). AWS CodeArtifact: nothing per repository; the organisation variables + ([One-time organisation setup](organisation-setup.md), step 5) and the roles' trust + policies cover every repository. The first pre-release is published straight away. 8. **After the stubs are on `main`: require the PR checks on `main` and let the bot bypass the `main` ruleset.** `pull_request_target` runs the workflow file of the base branch, so the checks never report until the stub is on `main`; requiring them earlier diff --git a/examples/stubs/deploy-codeartifact.yml b/examples/stubs/deploy-codeartifact.yml new file mode 100644 index 0000000..973b02b --- /dev/null +++ b/examples/stubs/deploy-codeartifact.yml @@ -0,0 +1,125 @@ +# Publishing to AWS CodeArtifact: see https://github.com/Algorithmiq/release-tools#readme +# +# Runs on the v* tags that the release workflow pushes: build, draft the GitHub release +# with every file attached, upload to the organisation's private package index on AWS +# CodeArtifact, then publish the GitHub release, which makes it immutable. Never on +# `release: published`. Only the build job is repository-specific; the CODEARTIFACT_* +# variables are set once for the organisation. +name: Publish + +on: + push: + tags: ["v*"] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + # A publish run must never be cut short half-way through. + cancel-in-progress: false + +jobs: + build: + name: Build sdist and wheel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@v10.3.2 + - name: Build distributions + run: uv build --out-dir dist + - name: Check the version matches the release tag + run: | + expected="${GITHUB_REF_NAME#v}" + wheels=(dist/*.whl) + actual="$(basename "${wheels[0]}" | cut -d- -f2)" + if [[ "$actual" != "$expected" ]]; then + echo "::error::built version is '$actual', but the release tag is '$expected'" + exit 1 + fi + - uses: actions/upload-artifact@v7.0.2 + with: + name: dist + path: dist/* + + draft_release: + # Immutable releases: files can only be attached while the release is a draft. + name: Create the draft GitHub release + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 # git-cliff needs the full history and the tags + - uses: actions/download-artifact@v8.0.2 + with: + name: dist + path: dist + - id: notes + uses: Algorithmiq/release-tools/actions/release-notes@v1.2.0 + - uses: Algorithmiq/release-tools/actions/draft-release@v1.2.0 + with: + notes-file: ${{ steps.notes.outputs.path }} + files: | + dist/* + + upload_codeartifact: + name: Upload to AWS CodeArtifact + needs: draft_release + runs-on: ubuntu-latest + permissions: + id-token: write # the upload role and the attestations use GitHub's OIDC token + attestations: write + contents: read + steps: + - uses: actions/download-artifact@v8.0.2 + with: + name: dist + path: dist + # Build provenance, stored by GitHub: `gh attestation verify --owner `. + - uses: actions/attest-build-provenance@v4.2.2 + with: + subject-path: "dist/*" + - uses: aws-actions/configure-aws-credentials@v6.3.0 + with: + role-to-assume: ${{ vars.CODEARTIFACT_WRITE_ROLE }} + aws-region: ${{ vars.CODEARTIFACT_REGION }} + - uses: astral-sh/setup-uv@v10.3.2 + with: + enable-cache: false + - name: Upload with uv publish + env: + DOMAIN: ${{ vars.CODEARTIFACT_DOMAIN }} + DOMAIN_OWNER: ${{ vars.CODEARTIFACT_DOMAIN_OWNER }} + REPOSITORY: ${{ vars.CODEARTIFACT_REPOSITORY }} + run: | + token="$(aws codeartifact get-authorization-token \ + --domain "$DOMAIN" --domain-owner "$DOMAIN_OWNER" \ + --query authorizationToken --output text)" + echo "::add-mask::$token" + endpoint="$(aws codeartifact get-repository-endpoint \ + --domain "$DOMAIN" --domain-owner "$DOMAIN_OWNER" --repository "$REPOSITORY" \ + --format pypi --query repositoryEndpoint --output text)" + endpoint="${endpoint%/}/" + # With --index, uv reads the index, with its credentials, and skips the files + # already there, so a failed run can be re-run. + config="$RUNNER_TEMP/codeartifact.toml" + printf '[[index]]\nname = "codeartifact"\nurl = "%s"\npublish-url = "%s"\nexplicit = true\n' \ + "${endpoint}simple/" "$endpoint" > "$config" + export UV_INDEX_CODEARTIFACT_USERNAME=aws UV_INDEX_CODEARTIFACT_PASSWORD="$token" + uv publish --config-file "$config" --index codeartifact \ + --trusted-publishing never --no-attestations dist/* + + publish_release: + # Publishing locks the tag and the files and creates GitHub's release attestation. + name: Publish the GitHub release + needs: upload_codeartifact + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: Algorithmiq/release-tools/actions/publish-release@v1.2.0 diff --git a/tests/test_codeartifact_stub.py b/tests/test_codeartifact_stub.py new file mode 100644 index 0000000..7860308 --- /dev/null +++ b/tests/test_codeartifact_stub.py @@ -0,0 +1,174 @@ +# Copyright 2026 Algorithmiq +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests of the upload step of examples/stubs/deploy-codeartifact.yml. + +The step runs in bash with stand-ins for `aws` and `uv` that record what they get. +""" + +from __future__ import annotations + +import json +import os +import stat +import subprocess +import tomllib +from typing import TYPE_CHECKING, Any + +import pytest +import yaml +from conftest import ROOT + +if TYPE_CHECKING: + from pathlib import Path + +STUB = ROOT / "examples" / "stubs" / "deploy-codeartifact.yml" +TOKEN = "token-from-codeartifact" # noqa: S105 (a stand-in's output) +ENDPOINT = "https://example.d.codeartifact.test/pypi/packages/" + + +def upload_job() -> dict[str, Any]: + """Return the upload job of the stub.""" + return yaml.safe_load(STUB.read_text(encoding="utf-8"))["jobs"][ + "upload_codeartifact" + ] + + +def upload_step() -> dict[str, Any]: + """Return the step of the upload job that runs uv publish.""" + (step,) = [s for s in upload_job()["steps"] if "uv publish" in s.get("run", "")] + return step + + +@pytest.fixture +def uploaded(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + """Run the upload step with stand-ins; return what uv got and the step's output.""" + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + record = tmp_path / "uv.json" + stand_ins = { + # aws codeartifact get-authorization-token | get-repository-endpoint ... + "aws": ( + f'echo "$@" >> "{tmp_path}/aws-args"\n' + f'case "$2" in get-authorization-token) echo "{TOKEN}" ;;' + f' get-repository-endpoint) echo "{ENDPOINT}" ;; *) exit 1 ;; esac' + ), + # Record the arguments, the credentials and the config file uv was given. + "uv": ( + 'config=""; prev=""\n' + 'for a in "$@"; do\n' + ' [ "$prev" = --config-file ] && config="$a"; prev="$a"\n' + "done\n" + f'python3 -c \'import json, os, sys; json.dump({{"args": sys.argv[2:], ' + '"username": os.environ.get("UV_INDEX_CODEARTIFACT_USERNAME"), ' + '"password": os.environ.get("UV_INDEX_CODEARTIFACT_PASSWORD"), ' + '"config": open(sys.argv[1]).read()}, ' + f'open("{record}", "w"))\' "$config" "$@"' + ), + } + for name, body in stand_ins.items(): + path = bin_dir / name + path.write_text(f"#!/usr/bin/env bash\n{body}\n", encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + work = tmp_path / "work" + (work / "dist").mkdir(parents=True) + for name in ("demo-1.0.0-py3-none-any.whl", "demo-1.0.0.tar.gz"): + (work / "dist" / name).write_text("", encoding="utf-8") + runner_temp = tmp_path / "runner-temp" + runner_temp.mkdir() + monkeypatch.chdir(work) + env = { + **os.environ, + "PATH": f"{bin_dir}{os.pathsep}{os.environ['PATH']}", + "RUNNER_TEMP": str(runner_temp), + "DOMAIN": "example", + "DOMAIN_OWNER": "owner", + "REPOSITORY": "packages", + } + result = subprocess.run( + [ + "bash", + "--noprofile", + "--norc", + "-eo", + "pipefail", + "-c", + upload_step()["run"], + ], + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + return {**json.loads(record.read_text(encoding="utf-8")), "stdout": result.stdout} + + +def test_uploads_every_distribution_to_the_index_with_uv( + uploaded: dict[str, Any], +) -> None: + args = uploaded["args"] + assert args[0] == "publish" + assert args[args.index("--index") + 1] == "codeartifact" + assert args[args.index("--trusted-publishing") + 1] == "never" + assert "--no-attestations" in args + assert sorted(a for a in args if a.startswith("dist/")) == [ + "dist/demo-1.0.0-py3-none-any.whl", + "dist/demo-1.0.0.tar.gz", + ] + + +def test_the_index_lets_uv_skip_files_already_uploaded( + uploaded: dict[str, Any], +) -> None: + # With --index, uv reads the index's url, with the index's credentials, and skips + # files that are already there: a failed run can be re-run. + (index,) = tomllib.loads(uploaded["config"])["index"] + assert index["name"] == "codeartifact" + assert index["publish-url"] == ENDPOINT + assert index["url"] == f"{ENDPOINT}simple/" + assert uploaded["username"] == "aws" + assert uploaded["password"] == TOKEN + + +def test_the_token_is_masked_and_never_an_argument(uploaded: dict[str, Any]) -> None: + assert f"::add-mask::{TOKEN}" in uploaded["stdout"] + assert not any(TOKEN in arg for arg in uploaded["args"]) + assert TOKEN not in uploaded["config"] + + +def test_the_upload_job_attests_the_distributions() -> None: + job = upload_job() + assert job["permissions"]["attestations"] == "write" + assert job["permissions"]["id-token"] == "write" + (attest,) = [ + s + for s in job["steps"] + if s.get("uses", "").startswith("actions/attest-build-provenance@") + ] + assert attest["with"]["subject-path"] == "dist/*" + + +def test_the_upload_step_passes_shellcheck() -> None: + run = upload_step()["run"] + # Values reach the script through env:, never through ${{ }} in the script itself. + assert "${{" not in run + result = subprocess.run( + ["shellcheck", "--shell=bash", "-"], + input=run, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stdout diff --git a/tests/test_examples.py b/tests/test_examples.py index fb485b9..d75d5f1 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -79,6 +79,7 @@ def test_dogfooding_stub_is_the_consumer_stub(name: str) -> None: "backport.yml", "pr-checks.yml", "deploy.yml", + "deploy-codeartifact.yml", "justfile", "justfile-internal", ],