diff --git a/.github/workflows/chart-release.yml b/.github/workflows/chart-release.yml index 8c70197..1810a64 100644 --- a/.github/workflows/chart-release.yml +++ b/.github/workflows/chart-release.yml @@ -16,6 +16,16 @@ on: required: false default: false type: boolean + operator_image: + description: Optional immutable operator image used by lifecycle tests + required: false + default: "" + type: string + operator_index_digest: + description: Expected operator OCI index digest for the lifecycle pod + required: false + default: "" + type: string workflow_dispatch: inputs: operator_version: @@ -62,13 +72,14 @@ jobs: - name: Upload candidate artifact uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 with: - name: helm-chart-${{ inputs.chart_version }} + name: helm-chart-${{ inputs.chart_version }}-${{ github.run_id }} path: | dist/chart/webhookrelay-operator-${{ inputs.chart_version }}.tgz dist/chart/webhookrelay-operator-${{ inputs.chart_version }}.tgz.sha256 dist/chart/release-metadata.json if-no-files-found: error retention-days: 14 + overwrite: true lifecycle: name: Test exact chart artifact in k3s @@ -87,12 +98,14 @@ jobs: - name: Download validated artifact uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 with: - name: helm-chart-${{ inputs.chart_version }} + name: helm-chart-${{ inputs.chart_version }}-${{ github.run_id }} path: validated - name: Run packaged chart lifecycle env: WHR_E2E_CANDIDATE_CHART: ${{ github.workspace }}/validated/webhookrelay-operator-${{ inputs.chart_version }}.tgz + WHR_E2E_OPERATOR_IMAGE: ${{ inputs.operator_image }} + WHR_E2E_OPERATOR_INDEX_DIGEST: ${{ inputs.operator_index_digest }} run: make e2e - name: Upload failure diagnostics @@ -121,7 +134,7 @@ jobs: - name: Download validated artifact uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6 with: - name: helm-chart-${{ inputs.chart_version }} + name: helm-chart-${{ inputs.chart_version }}-${{ github.run_id }} path: validated - name: Set up Go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47f6f7a..212f7f7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,9 @@ jobs: make chart-package HELM_BIN=.test/helm make chart-release-check HELM_BIN=.test/helm make chart-release-test HELM_BIN=.test/helm + make release-workflow-test + .test/helm template webhookrelay-operator charts/webhookrelay-operator \ + --set-string image.digest=sha256:test | grep -Fq 'webhookrelay/webhookrelay-operator@sha256:test' e2e: name: End-to-End and Helm Lifecycle (k3s) diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml deleted file mode 100644 index 1c412f5..0000000 --- a/.github/workflows/image.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: image - -on: - release: - types: [published] - workflow_dispatch: - inputs: - version: - description: Image version to publish - required: true - type: string - -permissions: - contents: read - packages: write - id-token: write - -jobs: - image: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - with: - ref: ${{ github.event.release.tag_name || inputs.version }} - - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 - with: - go-version: "1.27.1" - - uses: imjasonh/setup-ko@61b4d1d396f5b2e7d6bb6fefdce3dc38d1a13445 # v0.10 - env: - KO_DOCKER_REPO: webhookrelay/webhookrelay-operator - - name: Build - env: - VERSION: ${{ github.event.release.tag_name || inputs.version }} - run: | - echo "${{ secrets.DOCKER_PASSWORD }}" | ko login docker.io --username "${{ secrets.DOCKER_USERNAME }}" --password-stdin - make ko-build VERSION="${VERSION}" diff --git a/.github/workflows/production-e2e.yml b/.github/workflows/production-e2e.yml index 7df11da..59fc401 100644 --- a/.github/workflows/production-e2e.yml +++ b/.github/workflows/production-e2e.yml @@ -1,6 +1,25 @@ name: Production End-to-End Validation on: + workflow_call: + inputs: + operator_image: + description: Immutable operator image reference to validate + required: true + type: string + agent_image: + description: Relay-agent image to validate + required: true + type: string + operator_index_digest: + description: Expected operator OCI index digest + required: true + type: string + secrets: + WHR_E2E_RELAY_KEY: + required: true + WHR_E2E_RELAY_SECRET: + required: true workflow_dispatch: inputs: confirmation: @@ -12,6 +31,16 @@ on: required: false default: webhookrelay/webhookrelayd-ubi8:1.37.0 type: string + operator_image: + description: Published operator image to validate; blank builds the checkout + required: false + default: "" + type: string + operator_index_digest: + description: Optional expected operator OCI index digest + required: false + default: "" + type: string permissions: contents: read @@ -19,9 +48,9 @@ permissions: jobs: production-e2e: name: Production Live Delivery (k3s) - if: inputs.confirmation == 'production' + if: github.event_name == 'workflow_call' || inputs.confirmation == 'production' runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 22 environment: production env: WHR_OPERATOR_E2E_PRODUCTION: "true" @@ -30,13 +59,20 @@ jobs: WHR_E2E_RELAY_SECRET: ${{ secrets.WHR_E2E_RELAY_SECRET }} WHR_E2E_FUNCTION_ID: ${{ vars.WHR_E2E_FUNCTION_ID }} WHR_E2E_AGENT_IMAGE: ${{ inputs.agent_image }} + WHR_E2E_OPERATOR_IMAGE: ${{ inputs.operator_image }} + WHR_E2E_OPERATOR_INDEX_DIGEST: ${{ inputs.operator_index_digest }} steps: - name: Checkout code uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 - name: Validate production reconciliation and live delivery + timeout-minutes: 17 run: make e2e + - name: Verify exact cleanup after failure or cancellation + if: always() + run: ./.test/cleanup-production.sh + - name: Upload failure diagnostics if: failure() || cancelled() uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..6710452 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,270 @@ +name: Operator Release + +on: + push: + tags: + - "0.*" + +permissions: + contents: read + +concurrency: + group: operator-release + cancel-in-progress: false + +env: + OPERATOR_VERSION: 0.8.0 + CHART_VERSION: 0.7.0 + AGENT_VERSION: 1.37.0 + OPERATOR_IMAGE: webhookrelay/webhookrelay-operator:build-${{ github.sha }} + +jobs: + metadata: + name: Validate release mapping + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + fetch-depth: 0 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: "1.27.1" + - name: Validate tag and version mapping + run: ./.scripts/image-release.sh validate-source + - name: Re-run source and generated-asset gates + run: | + make test + make golangci-lint + make lint + make go-gen + git diff --exit-code + ./.test/install-helm.sh .test/helm + make chart-release-check HELM_BIN=.test/helm + + image: + name: Publish immutable operator image + needs: metadata + runs-on: ubuntu-latest + environment: release + outputs: + digest: ${{ steps.build.outputs.digest }} + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + fetch-depth: 0 + - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6 + with: + go-version: "1.27.1" + - uses: imjasonh/setup-ko@61b4d1d396f5b2e7d6bb6fefdce3dc38d1a13445 # v0.10 + with: + version: v0.18.1 + env: + KO_DOCKER_REPO: webhookrelay/webhookrelay-operator + - name: Authenticate to Docker Hub + env: + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + run: printf '%s' "${DOCKER_PASSWORD}" | ko login docker.io --username "${DOCKER_USERNAME}" --password-stdin + - id: build + name: Build or resume immutable image + run: | + ./.scripts/image-release.sh build > operator-image.json + echo "digest=$(jq -r .digest operator-image.json)" >> "${GITHUB_OUTPUT}" + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 + with: + name: operator-image-${{ github.ref_name }}-${{ github.run_id }} + path: operator-image.json + if-no-files-found: error + retention-days: 30 + overwrite: true + + image-validation: + name: Verify and smoke multi-architecture images + needs: image + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + fetch-depth: 0 + - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - id: verify + name: Verify manifests, provenance labels, and executability + run: | + ./.scripts/image-release.sh verify-smoke '${{ needs.image.outputs.digest }}' > release-images.json + - uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5 + with: + name: release-images-${{ github.ref_name }}-${{ github.run_id }} + path: release-images.json + if-no-files-found: error + retention-days: 30 + overwrite: true + + production: + name: Production live delivery + needs: [image, image-validation] + uses: ./.github/workflows/production-e2e.yml + with: + operator_image: webhookrelay/webhookrelay-operator@${{ needs.image.outputs.digest }} + operator_index_digest: ${{ needs.image.outputs.digest }} + agent_image: webhookrelay/webhookrelayd-ubi8@sha256:79d86f8dbc71831a16a2e1cd826b5a90a4c5a8a165fc53e62f43243b8274fb22 + secrets: inherit + + version-promotion: + name: Promote immutable operator version + needs: [image, production] + runs-on: ubuntu-latest + environment: release + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + fetch-depth: 0 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - run: ./.scripts/image-release.sh promote-version '${{ needs.image.outputs.digest }}' + + chart-publication: + name: Validate and publish immutable chart + needs: [image, image-validation, version-promotion] + permissions: + contents: read + id-token: write + uses: ./.github/workflows/chart-release.yml + with: + operator_version: 0.8.0 + chart_version: 0.7.0 + publish: true + operator_image: webhookrelay/webhookrelay-operator@${{ needs.image.outputs.digest }} + operator_index_digest: ${{ needs.image.outputs.digest }} + secrets: inherit + + promote: + name: Promote operator latest + needs: [image, chart-publication] + runs-on: ubuntu-latest + environment: release + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + fetch-depth: 0 + - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 + - uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - name: Promote without rebuilding + run: ./.scripts/image-release.sh promote-latest '${{ needs.image.outputs.digest }}' + + release: + name: Create GitHub release last + needs: [image, promote] + runs-on: ubuntu-latest + environment: release + permissions: + actions: read + contents: write + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5 + with: + fetch-depth: 0 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: helm-chart-0.7.0-${{ github.run_id }} + path: release-assets + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: operator-image-${{ github.ref_name }}-${{ github.run_id }} + path: release-assets + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: release-images-${{ github.ref_name }}-${{ github.run_id }} + path: release-assets + - name: Cross-check release metadata + env: + OPERATOR_DIGEST: ${{ needs.image.outputs.digest }} + run: | + (cd release-assets && sha256sum --check --strict webhookrelay-operator-0.7.0.tgz.sha256) + chart_sha=$(awk '{print $1}' release-assets/webhookrelay-operator-0.7.0.tgz.sha256) + jq -e --arg revision "${GITHUB_SHA}" --arg digest "${OPERATOR_DIGEST}" ' + .revision == $revision and .digest == $digest and + ([.platforms[] | .os + "/" + .architecture] | sort) == ["linux/amd64","linux/arm64"] + ' release-assets/operator-image.json >/dev/null + jq -e --arg revision "${GITHUB_SHA}" --arg digest "${OPERATOR_DIGEST}" ' + .revision == $revision and .operator.digest == $digest and + .agent.digest == "sha256:4d73b9e6096d4d653c3b46733aa4dbf45dfde1ed1e5f142da8da5a396eb9ffab" and + .agentUbi.digest == "sha256:79d86f8dbc71831a16a2e1cd826b5a90a4c5a8a165fc53e62f43243b8274fb22" and + all(.operator,.agent,.agentUbi; + ([.platforms[] | .os + "/" + .architecture] | sort) == ["linux/amd64","linux/arm64"]) + ' release-assets/release-images.json >/dev/null + jq -e --arg sha "${chart_sha}" ' + .operatorVersion == "0.8.0" and .chartVersion == "0.7.0" and + .agentImage == "webhookrelay/webhookrelayd-ubi8:1.37.0" and .sha256 == $sha + ' release-assets/release-metadata.json >/dev/null + - name: Write release notes + env: + OPERATOR_DIGEST: ${{ needs.image.outputs.digest }} + run: | + cat > release-notes.md < release-state.json 2>/dev/null; then + ./.scripts/verify-release-state.sh release-state.json release-notes.md "${GITHUB_REF_NAME}" + if [ "$(jq '.assets | length' release-state.json)" != 0 ]; then + gh release download "${GITHUB_REF_NAME}" --dir existing-assets + for asset in existing-assets/*; do + cmp "${asset}" "release-assets/$(basename "${asset}")" + done + fi + else + gh release create "${GITHUB_REF_NAME}" --verify-tag --draft \ + --title "${GITHUB_REF_NAME}" --notes-file release-notes.md + fi + for asset in release-assets/*; do + if [ ! -f "existing-assets/$(basename "${asset}")" ]; then + gh release upload "${GITHUB_REF_NAME}" "${asset}" + fi + done + - name: Verify assets and publish as final mutation + env: + GH_TOKEN: ${{ github.token }} + run: | + remote_sha=$(git ls-remote origin "refs/tags/${GITHUB_REF_NAME}^{}" | awk '{print $1}') + if [ -z "${remote_sha}" ]; then + remote_sha=$(git ls-remote origin "refs/tags/${GITHUB_REF_NAME}" | awk '{print $1}') + fi + test "${remote_sha}" = "${GITHUB_SHA}" + gh release view "${GITHUB_REF_NAME}" --json name,body,isDraft,assets > final-release-state.json + ./.scripts/verify-release-state.sh final-release-state.json release-notes.md "${GITHUB_REF_NAME}" + test "$(jq '.assets | length' final-release-state.json)" = 5 + mkdir downloaded-assets + gh release download "${GITHUB_REF_NAME}" --dir downloaded-assets + for asset in release-assets/*; do + cmp "${asset}" "downloaded-assets/$(basename "${asset}")" + done + if jq -e .isDraft final-release-state.json >/dev/null; then + gh release edit "${GITHUB_REF_NAME}" --draft=false + fi diff --git a/.ko.yaml b/.ko.yaml new file mode 100644 index 0000000..3b0c72b --- /dev/null +++ b/.ko.yaml @@ -0,0 +1,4 @@ +defaultBaseImage: cgr.dev/chainguard/static@sha256:f51c2493951313c3ad4069080b2814ffb6ed6fe3909dabeb84a9482f42d5600b +defaultPlatforms: + - linux/amd64 + - linux/arm64 diff --git a/.scripts/image-release.sh b/.scripts/image-release.sh new file mode 100755 index 0000000..e5f3574 --- /dev/null +++ b/.scripts/image-release.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly OPERATOR_VERSION="${OPERATOR_VERSION:-0.8.0}" +readonly CHART_VERSION="${CHART_VERSION:-0.7.0}" +readonly AGENT_VERSION="${AGENT_VERSION:-1.37.0}" +readonly OPERATOR_IMAGE="${OPERATOR_IMAGE:-webhookrelay/webhookrelay-operator:${OPERATOR_VERSION}}" +readonly AGENT_IMAGE="webhookrelay/webhookrelayd:${AGENT_VERSION}" +readonly AGENT_UBI_IMAGE="webhookrelay/webhookrelayd-ubi8:${AGENT_VERSION}" +readonly AGENT_DIGEST="sha256:4d73b9e6096d4d653c3b46733aa4dbf45dfde1ed1e5f142da8da5a396eb9ffab" +readonly AGENT_UBI_DIGEST="sha256:79d86f8dbc71831a16a2e1cd826b5a90a4c5a8a165fc53e62f43243b8274fb22" +readonly PREVIOUS_OPERATOR_DIGEST="sha256:63e40035e33c9485c4909868cc5f2372cf06c97eee26b342f2fa8f7562f5e520" +readonly REVISION="${GITHUB_SHA:-$(git rev-parse HEAD)}" +readonly SOURCE="https://github.com/webhookrelay/webhookrelay-operator" + +fail() { + printf 'image-release: %s\n' "$*" >&2 + exit 1 +} + +semver_greater() { + awk -v left="$1" -v right="$2" 'BEGIN { + split(left, l, "."); split(right, r, ".") + for (i = 1; i <= 3; i++) { + if ((l[i] + 0) > (r[i] + 0)) exit 0 + if ((l[i] + 0) < (r[i] + 0)) exit 1 + } + exit 1 + }' +} + +manifest_json() { + docker buildx imagetools inspect "$1" --format '{{json .Manifest}}' +} + +verify_image() { + local image="$1" expected_digest="${2:-}" expected_revision="${3:-}" + local digest label manifest platform + manifest="$(manifest_json "${image}")" || fail "cannot inspect ${image}" + jq -e ' + (.manifests | length) == 2 and + ([.manifests[] | select(.platform.os == "linux") | + (.platform.os + "/" + .platform.architecture)] | sort) == + ["linux/amd64", "linux/arm64"] + ' <<<"${manifest}" >/dev/null || fail "${image} does not have exactly linux/amd64 and linux/arm64" + digest="$(jq -r '.digest' <<<"${manifest}")" + [[ -z "${expected_digest}" || "${digest}" == "${expected_digest}" ]] || \ + fail "${image} digest ${digest} does not equal ${expected_digest}" + if [[ -n "${expected_revision}" ]]; then + for platform in linux/amd64 linux/arm64; do + docker pull --platform "${platform}" "${image}" >/dev/null + label="$(docker image inspect --format \ + '{{ index .Config.Labels "org.opencontainers.image.revision" }}' "${image}")" + [[ "${label}" == "${expected_revision}" ]] || \ + fail "${image} ${platform} revision ${label:-unset} does not equal ${expected_revision}" + label="$(docker image inspect --format \ + '{{ index .Config.Labels "org.opencontainers.image.version" }}' "${image}")" + [[ "${label}" == "${OPERATOR_VERSION}" ]] || \ + fail "${image} ${platform} version label ${label:-unset} does not equal ${OPERATOR_VERSION}" + label="$(docker image inspect --format \ + '{{ index .Config.Labels "org.opencontainers.image.source" }}' "${image}")" + [[ "${label}" == "${SOURCE}" ]] || fail "${image} ${platform} source label is invalid" + done + fi + printf '%s\n' "${digest}" +} + +validate_source() { + local app_version chart_version default_agent default_tag tag + tag="${GITHUB_REF_NAME:-${OPERATOR_VERSION}}" + [[ "${GITHUB_REF_TYPE:-tag}" == tag ]] || fail "release ref is not a tag" + [[ "${GITHUB_REF:-refs/tags/${tag}}" == "refs/tags/${OPERATOR_VERSION}" ]] || \ + fail "release ref does not name ${OPERATOR_VERSION}" + [[ "${tag}" == "${OPERATOR_VERSION}" ]] || fail "tag ${tag} does not equal ${OPERATOR_VERSION}" + git merge-base --is-ancestor "${REVISION}" origin/master || \ + fail "release revision ${REVISION} is not reachable from origin/master" + chart_version="$(awk -F ': *' '$1 == "version" {print $2; exit}' charts/webhookrelay-operator/Chart.yaml)" + app_version="$(awk -F ': *' '$1 == "appVersion" {gsub(/"/, "", $2); print $2; exit}' charts/webhookrelay-operator/Chart.yaml)" + default_tag="$(awk '/^image:/ {found=1; next} found && /^ tag:/ {gsub(/[" ]/, "", $2); print $2; exit}' charts/webhookrelay-operator/values.yaml)" + default_agent="$(awk -F '"' '/Image string `default:/ {print $2; exit}' pkg/config/config.go)" + [[ "${chart_version}" == "${CHART_VERSION}" ]] || fail "chart version mismatch" + [[ "${app_version}" == "${OPERATOR_VERSION}" ]] || fail "chart appVersion mismatch" + [[ "${default_tag}" == "${OPERATOR_VERSION}" ]] || fail "operator image tag mismatch" + [[ "${default_agent}" == "${AGENT_UBI_IMAGE}" ]] || fail "relay-agent default mismatch" +} + +build_operator() { + local digest manifest + if manifest_json "${OPERATOR_IMAGE}" >/dev/null 2>&1; then + digest="$(verify_image "${OPERATOR_IMAGE}" "" "${REVISION}")" + printf 'image-release: reusing verified %s@%s\n' "${OPERATOR_IMAGE}" "${digest}" >&2 + else + KO_DOCKER_REPO="${OPERATOR_IMAGE%:*}" ko build --bare \ + --platform=linux/amd64,linux/arm64 --tags="${OPERATOR_IMAGE##*:}" --sbom=none \ + --image-label="org.opencontainers.image.revision=${REVISION}" \ + --image-label="org.opencontainers.image.version=${OPERATOR_VERSION}" \ + --image-label="org.opencontainers.image.source=${SOURCE}" ./cmd/manager >/dev/null + digest="$(verify_image "${OPERATOR_IMAGE}" "" "${REVISION}")" + fi + manifest="$(manifest_json "${OPERATOR_IMAGE}")" + jq -n --arg image "${OPERATOR_IMAGE}" --arg digest "${digest}" \ + --arg revision "${REVISION}" --argjson manifest "${manifest}" \ + '{image:$image,digest:$digest,revision:$revision, + platforms:[$manifest.manifests[] | {os:.platform.os,architecture:.platform.architecture,digest:.digest}]}' +} + +verify_and_smoke() { + local operator_digest="$1" image platform operator_manifest agent_manifest agent_ubi_manifest + verify_image "${OPERATOR_IMAGE}" "${operator_digest}" "${REVISION}" >/dev/null + verify_image "${AGENT_IMAGE}" "${AGENT_DIGEST}" >/dev/null + verify_image "${AGENT_UBI_IMAGE}" "${AGENT_UBI_DIGEST}" >/dev/null + for platform in linux/amd64 linux/arm64; do + docker pull --platform "${platform}" "${OPERATOR_IMAGE}@${operator_digest}" >/dev/null + docker run --rm --platform "${platform}" "${OPERATOR_IMAGE}@${operator_digest}" --help >/dev/null + for image in "${AGENT_IMAGE}@${AGENT_DIGEST}" "${AGENT_UBI_IMAGE}@${AGENT_UBI_DIGEST}"; do + docker pull --platform "${platform}" "${image}" >/dev/null + docker run --rm --platform "${platform}" "${image}" --version >/dev/null + done + done + operator_manifest="$(manifest_json "${OPERATOR_IMAGE}@${operator_digest}")" + agent_manifest="$(manifest_json "${AGENT_IMAGE}@${AGENT_DIGEST}")" + agent_ubi_manifest="$(manifest_json "${AGENT_UBI_IMAGE}@${AGENT_UBI_DIGEST}")" + jq -n --arg revision "${REVISION}" \ + --arg operator_image "${OPERATOR_IMAGE}" --argjson operator "${operator_manifest}" \ + --arg agent_image "${AGENT_IMAGE}" --argjson agent "${agent_manifest}" \ + --arg agent_ubi_image "${AGENT_UBI_IMAGE}" --argjson agent_ubi "${agent_ubi_manifest}" ' + def record($name; $manifest): { + image:$name, digest:$manifest.digest, + platforms:[$manifest.manifests[] | {os:.platform.os,architecture:.platform.architecture,digest:.digest}] + }; + {revision:$revision, operator:record($operator_image;$operator), + agent:record($agent_image;$agent), agentUbi:record($agent_ubi_image;$agent_ubi)}' +} + +promote_tag() { + local expected_digest="$1" target_tag="$2" target_image target_digest="" + verify_image "${OPERATOR_IMAGE}" "${expected_digest}" "${REVISION}" >/dev/null + target_image="${OPERATOR_IMAGE%:*}:${target_tag}" + if target_digest="$(manifest_json "${target_image}" 2>/dev/null | jq -r '.digest')"; then + [[ "${target_digest}" == "${expected_digest}" ]] || \ + fail "refusing to replace ${target_image} digest ${target_digest}" + printf 'image-release: %s already points to %s\n' "${target_tag}" "${expected_digest}" >&2 + return + fi + docker buildx imagetools create --tag "${target_image}" \ + "${OPERATOR_IMAGE}@${expected_digest}" >/dev/null + target_digest="$(manifest_json "${target_image}" | jq -r '.digest')" + [[ "${target_digest}" == "${expected_digest}" ]] || fail "${target_tag} promotion digest mismatch" +} + +promote_latest() { + local expected_digest="$1" latest_digest="" latest_version="" + verify_image "${OPERATOR_IMAGE}" "${expected_digest}" "${REVISION}" >/dev/null + if latest_digest="$(manifest_json "${OPERATOR_IMAGE%:*}:latest" 2>/dev/null | jq -r '.digest')"; then + [[ "${latest_digest}" != "${expected_digest}" ]] || { + printf 'image-release: latest already points to %s\n' "${expected_digest}" >&2 + return + } + if [[ "${latest_digest}" != "${PREVIOUS_OPERATOR_DIGEST}" ]]; then + docker pull --platform linux/amd64 "${OPERATOR_IMAGE%:*}:latest" >/dev/null + latest_version="$(docker image inspect --format \ + '{{ index .Config.Labels "org.opencontainers.image.version" }}' \ + "${OPERATOR_IMAGE%:*}:latest")" + if [[ "${latest_version}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] && \ + semver_greater "${latest_version}" "${OPERATOR_VERSION}"; then + printf 'image-release: leaving newer latest version %s unchanged\n' "${latest_version}" >&2 + return + fi + fail "refusing to replace unexpected latest digest ${latest_digest}" + fi + fi + docker buildx imagetools create --tag "${OPERATOR_IMAGE%:*}:latest" \ + "${OPERATOR_IMAGE}@${expected_digest}" >/dev/null + latest_digest="$(manifest_json "${OPERATOR_IMAGE%:*}:latest" | jq -r '.digest')" + [[ "${latest_digest}" == "${expected_digest}" ]] || fail "latest promotion digest mismatch" +} + +case "${1:-}" in + validate-source) validate_source ;; + build) build_operator ;; + verify-smoke) verify_and_smoke "${2:?operator digest required}" ;; + promote-version) promote_tag "${2:?operator digest required}" "${OPERATOR_VERSION}" ;; + promote-latest) promote_latest "${2:?operator digest required}" ;; + *) fail "usage: $0 validate-source|build|verify-smoke DIGEST|promote-version DIGEST|promote-latest DIGEST" ;; +esac diff --git a/.scripts/verify-release-state.sh b/.scripts/verify-release-state.sh new file mode 100755 index 0000000..f799555 --- /dev/null +++ b/.scripts/verify-release-state.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly STATE_FILE="${1:?release state JSON required}" +readonly NOTES_FILE="${2:?release notes required}" +readonly TAG="${3:?release tag required}" +readonly EXPECTED='["operator-image.json","release-images.json","release-metadata.json","webhookrelay-operator-0.7.0.tgz","webhookrelay-operator-0.7.0.tgz.sha256"]' + +jq -e --arg title "${TAG}" --rawfile body "${NOTES_FILE}" --argjson expected "${EXPECTED}" ' + .name == $title and .body == $body and + ([.assets[].name] - $expected | length) == 0 and + (if .isDraft then true else ([.assets[].name] | sort) == ($expected | sort) end) +' "${STATE_FILE}" >/dev/null diff --git a/.test/cleanup-production.sh b/.test/cleanup-production.sh new file mode 100755 index 0000000..9f3fb71 --- /dev/null +++ b/.test/cleanup-production.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly RUN_ID="${WHR_OPERATOR_E2E_RUN_ID:?run ID is required}" +readonly BUCKET_NAME="operator-e2e-${RUN_ID}" +readonly BUCKET_DESCRIPTION="Webhook Relay operator production e2e run ${RUN_ID}" +readonly API_URL="https://my.webhookrelay.com/v1/buckets" +readonly REPO_ROOT="$(git rev-parse --show-toplevel)" +readonly RUN_DIR="${REPO_ROOT}/.test/.runs/${RUN_ID}" +readonly K3S_DATA_DIR="/var/lib/webhookrelay-operator-e2e-${RUN_ID}" +readonly AUTH_FILE="$(mktemp)" +readonly RESPONSE_FILE="$(mktemp)" +trap 'rm -f -- "${AUTH_FILE}" "${RESPONSE_FILE}"' EXIT + +umask 077 + +# If the timed main step left its task-owned cluster alive, make a bounded best +# effort to stop reconciliation cleanly, then stop the task-owned K3s process +# before deleting remote state so the bucket cannot be recreated afterward. +if [[ -s "${RUN_DIR}/k3s.pid" ]]; then + pid="$(<"${RUN_DIR}/k3s.pid")" + [[ "${pid}" =~ ^[0-9]+$ ]] || { printf 'invalid task-owned k3s PID\n' >&2; exit 1; } + if sudo kill -0 "${pid}" 2>/dev/null; then + if [[ -x "${RUN_DIR}/bin/k3s" && -s "${RUN_DIR}/kubeconfig" ]]; then + kube=(sudo env K3S_DATA_DIR="${K3S_DATA_DIR}" "${RUN_DIR}/bin/k3s" kubectl \ + --kubeconfig "${RUN_DIR}/kubeconfig" --namespace webhookrelay-operator-e2e) + timeout 20s "${kube[@]}" scale deployment/webhookrelay-operator --replicas=0 \ + --request-timeout=10s >/dev/null 2>&1 || true + timeout 20s "${kube[@]}" delete webhookrelayforward/e2e-forward \ + --ignore-not-found --wait=true --timeout=10s --request-timeout=10s >/dev/null 2>&1 || true + fi + sudo kill -TERM -- "-${pid}" 2>/dev/null || sudo kill -TERM "${pid}" 2>/dev/null || true + for _ in $(seq 1 30); do + sudo kill -0 "${pid}" 2>/dev/null || break + sleep 1 + done + if sudo kill -0 "${pid}" 2>/dev/null; then + sudo kill -KILL -- "-${pid}" 2>/dev/null || sudo kill -KILL "${pid}" 2>/dev/null || true + sleep 1 + fi + sudo kill -0 "${pid}" 2>/dev/null && { printf 'could not stop task-owned k3s\n' >&2; exit 1; } + fi + [[ -x "${RUN_DIR}/bin/k3s" ]] || { printf 'task-owned k3s binary is missing\n' >&2; exit 1; } + sudo env K3S_DATA_DIR="${K3S_DATA_DIR}" "${RUN_DIR}/bin/k3s" killall >/dev/null +fi + +printf 'user = "%s:%s"\n' "${WHR_E2E_RELAY_KEY:?relay key is required}" \ + "${WHR_E2E_RELAY_SECRET:?relay secret is required}" >"${AUTH_FILE}" +curl --fail --show-error --silent --config "${AUTH_FILE}" "${API_URL}" >"${RESPONSE_FILE}" +count="$(jq --arg name "${BUCKET_NAME}" --arg description "${BUCKET_DESCRIPTION}" \ + '[.[] | select(.name == $name and .description == $description)] | length' "${RESPONSE_FILE}")" +[[ "${count}" != "0" ]] || exit 0 +[[ "${count}" == "1" ]] || { printf 'refusing ambiguous production cleanup\n' >&2; exit 1; } +bucket_id="$(jq -r --arg name "${BUCKET_NAME}" --arg description "${BUCKET_DESCRIPTION}" \ + '.[] | select(.name == $name and .description == $description) | .id' "${RESPONSE_FILE}")" +curl --fail --show-error --silent --config "${AUTH_FILE}" --request DELETE \ + "${API_URL}/${bucket_id}?force=true" >/dev/null +curl --fail --show-error --silent --config "${AUTH_FILE}" "${API_URL}" >"${RESPONSE_FILE}" +jq -e --arg name "${BUCKET_NAME}" --arg description "${BUCKET_DESCRIPTION}" \ + 'all(.[]; .name != $name or .description != $description)' "${RESPONSE_FILE}" >/dev/null diff --git a/.test/e2e-k3s.sh b/.test/e2e-k3s.sh index de8203e..93c192f 100755 --- a/.test/e2e-k3s.sh +++ b/.test/e2e-k3s.sh @@ -22,7 +22,18 @@ K3S_LOG="${ARTIFACT_DIR}/k3s.log" K3S_DATA_DIR="/var/lib/webhookrelay-operator-e2e-${RUN_ID}" K3S_CLIENT_DATA_DIR="${RUN_DIR}/client-data" NAMESPACE="webhookrelay-operator-e2e" -IMAGE="webhookrelay-operator-e2e:${RUN_ID}" +IMAGE="${WHR_E2E_OPERATOR_IMAGE:-webhookrelay-operator-e2e:${RUN_ID}}" +OPERATOR_IMAGE_REPOSITORY="${IMAGE%@*}" +OPERATOR_IMAGE_DIGEST="" +OPERATOR_IMAGE_TAG="${IMAGE##*:}" +if [[ "${IMAGE}" == *@* ]]; then + OPERATOR_IMAGE_DIGEST="${IMAGE#*@}" + OPERATOR_IMAGE_TAG="" +else + OPERATOR_IMAGE_REPOSITORY="${IMAGE%:*}" +fi +OPERATOR_PULL_POLICY="IfNotPresent" +[[ -z "${WHR_E2E_OPERATOR_IMAGE:-}" ]] || OPERATOR_PULL_POLICY="Always" RECEIVER_IMAGE="webhookrelay-operator-e2e-receiver:${RUN_ID}" FAKE_API_IMAGE="webhookrelay-operator-e2e-fake-api:${RUN_ID}" PRODUCTION_MODE="${WHR_OPERATOR_E2E_PRODUCTION:-false}" @@ -149,11 +160,17 @@ helm() { build_and_import_images() { local archive="${RUN_DIR}/operator-image.tar" - log "building operator image ${IMAGE}" - docker build --tag "${IMAGE}" --file "${REPO_ROOT}/build/Dockerfile" "${REPO_ROOT}" - docker save --output "${archive}" "${IMAGE}" - sudo env K3S_DATA_DIR="${K3S_DATA_DIR}" "${K3S_BIN}" ctr \ - --address /run/k3s/containerd/containerd.sock --namespace k8s.io images import "${archive}" + if [[ -n "${WHR_E2E_OPERATOR_IMAGE:-}" ]]; then + [[ "${IMAGE}" == *@sha256:* || "${OPERATOR_IMAGE_REPOSITORY}" != "${IMAGE}" ]] || \ + fail "WHR_E2E_OPERATOR_IMAGE must use a version tag or sha256 digest" + log "using published operator image ${IMAGE}" + else + log "building operator image ${IMAGE}" + docker build --tag "${IMAGE}" --file "${REPO_ROOT}/build/Dockerfile" "${REPO_ROOT}" + docker save --output "${archive}" "${IMAGE}" + sudo env K3S_DATA_DIR="${K3S_DATA_DIR}" "${K3S_BIN}" ctr \ + --address /run/k3s/containerd/containerd.sock --namespace k8s.io images import "${archive}" + fi if [[ "${PRODUCTION_MODE}" == "false" ]]; then archive="${RUN_DIR}/fake-api-image.tar" @@ -179,9 +196,10 @@ install_operator() { log "installing the chart" helm_args=(upgrade --install webhookrelay-operator "${CANDIDATE_CHART}" \ --namespace "${NAMESPACE}" --create-namespace --wait --timeout 180s \ - --set-string image.repository=webhookrelay-operator-e2e \ - --set-string "image.tag=${RUN_ID}" \ - --set image.pullPolicy=IfNotPresent) + --set-string "image.repository=${OPERATOR_IMAGE_REPOSITORY}" \ + --set-string "image.tag=${OPERATOR_IMAGE_TAG}" \ + --set-string "image.digest=${OPERATOR_IMAGE_DIGEST}" \ + --set "image.pullPolicy=${OPERATOR_PULL_POLICY}") if [[ "${PRODUCTION_MODE}" == "false" ]]; then helm_args+=(--set-string apiEndpointURL=http://relay-api:8080/v1) fi @@ -191,6 +209,14 @@ install_operator() { .spec.template.spec.serviceAccountName == "webhookrelay-operator" and .spec.template.spec.containers[0].image == $image ' >/dev/null || fail "operator Deployment wiring is invalid" + if [[ -n "${WHR_E2E_OPERATOR_INDEX_DIGEST:-}" ]]; then + kubectl -n "${NAMESPACE}" get pods -l app.kubernetes.io/name=webhookrelay-operator -o json | \ + jq -e --arg digest "${WHR_E2E_OPERATOR_INDEX_DIGEST}" ' + (.items | length) == 1 and + all(.items[]; any(.status.containerStatuses[]?; + .name == "webhookrelay-operator" and (.imageID | endswith($digest)))) + ' >/dev/null || fail "operator pod did not run the expected image index digest" + fi } install_fake_api() { @@ -554,6 +580,7 @@ exercise_reconcile() { wait_for_isolated_condition Ready False AgentNotReady log "recovering isolated agent Deployment from an invalid image" apply_isolated_forward operator-e2e-v1 true "${expected_agent_image}" + wait_for_object_creation deployment/e2e-forward-whr-deployment kubectl -n "${NAMESPACE}" rollout status deployment/e2e-forward-whr-deployment --timeout=180s wait_for_isolated_condition AgentReady True DeploymentAvailable wait_for_isolated_condition Ready True Ready @@ -625,6 +652,7 @@ exercise_production_reconcile() { '.[] | select(.name == $name) | .outputs[] | select(.name == "e2e-replay-output") | .id' "${RUN_DIR}/production-buckets.json")" [[ -n "${bucket_id}" && -n "${input_id}" && -n "${output_id}" && -n "${replay_output_id}" ]] || fail "production resources did not converge" + wait_for_object_creation deployment/e2e-forward-whr-deployment kubectl -n "${NAMESPACE}" rollout status deployment/e2e-forward-whr-deployment --timeout=180s for _ in $(seq 1 60); do kubectl -n "${NAMESPACE}" get webhookrelayforward/e2e-forward -o json \ @@ -745,6 +773,17 @@ wait_for_object_deletion() { fail "${resource} was not deleted" } +wait_for_object_creation() { + local resource="$1" + for _ in $(seq 1 60); do + if kubectl -n "${NAMESPACE}" get "${resource}" >/dev/null 2>&1; then + return + fi + sleep 1 + done + fail "${resource} was not created" +} + scale_operator_to_zero() { kubectl -n "${NAMESPACE}" scale deployment/webhookrelay-operator --replicas=0 >/dev/null for _ in $(seq 1 60); do @@ -862,9 +901,10 @@ exercise_helm_lifecycle() { log "upgrading the published chart to the packaged candidate" helm upgrade webhookrelay-operator "${CANDIDATE_CHART}" --namespace "${NAMESPACE}" \ --wait --timeout 180s \ - --set-string image.repository=webhookrelay-operator-e2e \ - --set-string "image.tag=${RUN_ID}" \ - --set image.pullPolicy=IfNotPresent \ + --set-string "image.repository=${OPERATOR_IMAGE_REPOSITORY}" \ + --set-string "image.tag=${OPERATOR_IMAGE_TAG}" \ + --set-string "image.digest=${OPERATOR_IMAGE_DIGEST}" \ + --set "image.pullPolicy=${OPERATOR_PULL_POLICY}" \ --set-string apiEndpointURL=http://relay-api:8080/v1 assert_helm_revision 2 kubectl -n "${NAMESPACE}" rollout status deployment/webhookrelay-operator --timeout=120s @@ -874,6 +914,7 @@ exercise_helm_lifecycle() { -o jsonpath='{.spec.buckets[0].outputs[0].function_id}')" == \ "00000000-0000-0000-0000-000000000000" ]] || fail "legacy function_id was not retained" wait_for_isolated_routing_status Configured + wait_for_object_creation deployment/e2e-forward-whr-deployment kubectl -n "${NAMESPACE}" rollout status deployment/e2e-forward-whr-deployment --timeout=120s fake_api_state | jq -e --arg name "${BUCKET_NAME}-lifecycle" ' any(.buckets[]; .name == $name and diff --git a/.test/fake-release-docker.sh b/.test/fake-release-docker.sh new file mode 100755 index 0000000..9d8fe3b --- /dev/null +++ b/.test/fake-release-docker.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +operator_manifest() { + local digest="${1:-sha256:operator-index}" + printf '{"digest":"%s","manifests":[{"digest":"sha256:operator-amd64","platform":{"os":"linux","architecture":"amd64"}},{"digest":"sha256:operator-arm64","platform":{"os":"linux","architecture":"arm64"}}]}\n' "${digest}" +} + +case "$*" in + "buildx imagetools inspect webhookrelay/webhookrelay-operator:build-test"*) + [[ -f "${FAKE_REGISTRY}/staging" ]] || exit 1 + operator_manifest + ;; + "buildx imagetools inspect webhookrelay/webhookrelay-operator:0.8.0"*) + [[ -f "${FAKE_REGISTRY}/version" ]] || exit 1 + operator_manifest "$(<"${FAKE_REGISTRY}/version")" + ;; + "buildx imagetools inspect webhookrelay/webhookrelay-operator:latest"*) + [[ -f "${FAKE_REGISTRY}/latest" ]] || exit 1 + operator_manifest "$(<"${FAKE_REGISTRY}/latest")" + ;; + *"webhookrelay/webhookrelayd-ubi8:1.37.0"*"imagetools inspect"*|*"imagetools inspect webhookrelay/webhookrelayd-ubi8"*) + sed 's/sha256:operator-index/sha256:79d86f8dbc71831a16a2e1cd826b5a90a4c5a8a165fc53e62f43243b8274fb22/' "${FAKE_REGISTRY}/manifest" + ;; + *"webhookrelay/webhookrelayd:1.37.0"*"imagetools inspect"*|*"imagetools inspect webhookrelay/webhookrelayd:"*) + sed 's/sha256:operator-index/sha256:4d73b9e6096d4d653c3b46733aa4dbf45dfde1ed1e5f142da8da5a396eb9ffab/' "${FAKE_REGISTRY}/manifest" + ;; + "buildx imagetools inspect webhookrelay/webhookrelay-operator:build-test@sha256:operator-index"*) + operator_manifest + ;; + "buildx imagetools create --tag webhookrelay/webhookrelay-operator:0.8.0"*) + printf 'sha256:operator-index\n' >"${FAKE_REGISTRY}/version" + printf 'create-version\n' >>"${FAKE_REGISTRY}/writes" + ;; + "buildx imagetools create --tag webhookrelay/webhookrelay-operator:latest"*) + printf 'sha256:operator-index\n' >"${FAKE_REGISTRY}/latest" + printf 'create-latest\n' >>"${FAKE_REGISTRY}/writes" + ;; + image\ inspect*) + case "$*" in + *org.opencontainers.image.revision*) printf '%s\n' "${GITHUB_SHA}" ;; + *org.opencontainers.image.version*) + if [[ "$*" == *":latest"* && -f "${FAKE_REGISTRY}/latest-version" ]]; then + cat "${FAKE_REGISTRY}/latest-version" + else + printf '0.8.0\n' + fi + ;; + *org.opencontainers.image.source*) printf 'https://github.com/webhookrelay/webhookrelay-operator\n' ;; + *) exit 2 ;; + esac + ;; + pull*|run*) ;; + *) printf 'unsupported fake docker: %s\n' "$*" >&2; exit 2 ;; +esac diff --git a/.test/fake-release-ko.sh b/.test/fake-release-ko.sh new file mode 100755 index 0000000..635906f --- /dev/null +++ b/.test/fake-release-ko.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +[[ "${1:-}" == build ]] || exit 2 +[[ "$*" == *"--tags=build-test"* ]] || { printf 'ko used the wrong staging tag\n' >&2; exit 1; } +touch "${FAKE_REGISTRY}/staging" diff --git a/.test/image-release.sh b/.test/image-release.sh new file mode 100755 index 0000000..d05b131 --- /dev/null +++ b/.test/image-release.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly TEST_ROOT="$(mktemp -d)" +trap 'rm -rf -- "${TEST_ROOT}"' EXIT +mkdir -p "${TEST_ROOT}/bin" "${TEST_ROOT}/registry" +cp .test/fake-release-docker.sh "${TEST_ROOT}/bin/docker" +cp .test/fake-release-ko.sh "${TEST_ROOT}/bin/ko" +chmod 0755 "${TEST_ROOT}/bin/docker" "${TEST_ROOT}/bin/ko" +cat >"${TEST_ROOT}/registry/manifest" <<'EOF' +{"digest":"sha256:operator-index","manifests":[{"digest":"sha256:amd64","platform":{"os":"linux","architecture":"amd64"}},{"digest":"sha256:arm64","platform":{"os":"linux","architecture":"arm64"}}]} +EOF + +run_release() { + PATH="${TEST_ROOT}/bin:${PATH}" FAKE_REGISTRY="${TEST_ROOT}/registry" \ + GITHUB_SHA=test OPERATOR_IMAGE=webhookrelay/webhookrelay-operator:build-test \ + ./.scripts/image-release.sh "$@" +} + +run_release build >"${TEST_ROOT}/operator.json" +jq -e '.digest == "sha256:operator-index" and (.platforms | length) == 2' \ + "${TEST_ROOT}/operator.json" >/dev/null +run_release verify-smoke sha256:operator-index >"${TEST_ROOT}/release.json" +jq -e '.operator.digest == "sha256:operator-index" and + .agent.digest == "sha256:4d73b9e6096d4d653c3b46733aa4dbf45dfde1ed1e5f142da8da5a396eb9ffab" and + .agentUbi.digest == "sha256:79d86f8dbc71831a16a2e1cd826b5a90a4c5a8a165fc53e62f43243b8274fb22"' \ + "${TEST_ROOT}/release.json" >/dev/null +run_release promote-version sha256:operator-index +run_release promote-version sha256:operator-index +printf 'sha256:63e40035e33c9485c4909868cc5f2372cf06c97eee26b342f2fa8f7562f5e520\n' \ + >"${TEST_ROOT}/registry/latest" +run_release promote-latest sha256:operator-index +run_release promote-latest sha256:operator-index +[[ "$(grep -c create-version "${TEST_ROOT}/registry/writes")" == "1" ]] +[[ "$(grep -c create-latest "${TEST_ROOT}/registry/writes")" == "1" ]] +printf 'sha256:newer\n' >"${TEST_ROOT}/registry/latest" +printf '0.9.0\n' >"${TEST_ROOT}/registry/latest-version" +run_release promote-latest sha256:operator-index +[[ "$(<"${TEST_ROOT}/registry/latest")" == "sha256:newer" ]] +printf 'sha256:different\n' >"${TEST_ROOT}/registry/version" +if run_release promote-version sha256:operator-index >/dev/null 2>&1; then + printf 'image release replaced a colliding immutable version\n' >&2 + exit 1 +fi + +printf 'image release collision, resume, platform, and promotion checks passed\n' diff --git a/.test/release-workflow.sh b/.test/release-workflow.sh new file mode 100755 index 0000000..cd2a161 --- /dev/null +++ b/.test/release-workflow.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +readonly WORKFLOW=".github/workflows/release.yml" + +job_block() { + local job="$1" + awk -v job=" ${job}:" ' + $0 == job {inside=1; next} + inside && /^ [a-zA-Z0-9-]+:$/ {exit} + inside {print} + ' "${WORKFLOW}" +} + +assert_needs() { + local job="$1" expected="$2" + job_block "${job}" | grep -Fqx " needs: ${expected}" || { + printf '%s must need %s\n' "${job}" "${expected}" >&2 + exit 1 + } +} + +assert_needs image metadata +assert_needs image-validation image +assert_needs production '[image, image-validation]' +assert_needs version-promotion '[image, production]' +assert_needs chart-publication '[image, image-validation, version-promotion]' +assert_needs promote '[image, chart-publication]' +assert_needs release '[image, promote]' + +[[ "$(grep -c '^ release:$' "${WORKFLOW}")" == "1" ]] +job_block release | grep -Fq 'gh release create' +job_block promote | grep -Fq 'image-release.sh promote-latest' +job_block chart-publication | grep -Fq 'id-token: write' +[[ "$(grep -c -- '--rawfile body' .scripts/verify-release-state.sh)" == "1" ]] +[[ "$(job_block release | grep -c 'verify-release-state.sh')" == "2" ]] +notes_fixture="$(mktemp)" +trap 'rm -f -- "${notes_fixture}" "${notes_fixture}.state" "${notes_fixture}.subset" "${notes_fixture}.draft"' EXIT +printf 'release notes with trailing newline\n' >"${notes_fixture}" +jq -n --rawfile body "${notes_fixture}" '{body:$body}' | \ + jq -e --rawfile body "${notes_fixture}" '.body == $body' >/dev/null +full_assets='[{"name":"operator-image.json"},{"name":"release-images.json"},{"name":"release-metadata.json"},{"name":"webhookrelay-operator-0.7.0.tgz"},{"name":"webhookrelay-operator-0.7.0.tgz.sha256"}]' +jq -n --arg body "$(<"${notes_fixture}")" --argjson assets "${full_assets}" \ + '{name:"0.8.0",body:($body+"\n"),isDraft:false,assets:$assets}' >"${notes_fixture}.state" +./.scripts/verify-release-state.sh "${notes_fixture}.state" "${notes_fixture}" 0.8.0 +jq '.assets |= .[0:1]' "${notes_fixture}.state" >"${notes_fixture}.subset" +if ./.scripts/verify-release-state.sh "${notes_fixture}.subset" "${notes_fixture}" 0.8.0; then + printf 'public release subset was accepted as complete\n' >&2 + exit 1 +fi +jq '.isDraft = true' "${notes_fixture}.subset" >"${notes_fixture}.draft" +./.scripts/verify-release-state.sh "${notes_fixture}.draft" "${notes_fixture}" 0.8.0 +[[ "$(grep -c 'helm-chart-${{ inputs.chart_version }}-${{ github.run_id }}' \ + .github/workflows/chart-release.yml)" == "3" ]] +! grep -E 'name: (helm-chart|operator-image|release-images).*run_attempt' \ + .github/workflows/chart-release.yml .github/workflows/release.yml +! grep -Fq 'release:' .github/workflows/image.yaml 2>/dev/null + +printf 'release workflow ordering checks passed\n' diff --git a/Makefile b/Makefile index c167871..152b865 100644 --- a/Makefile +++ b/Makefile @@ -49,7 +49,7 @@ test: e2e: ./.test/e2e-k3s.sh -.PHONY: chart-package chart-release-check chart-release-test +.PHONY: chart-package chart-release-check chart-release-test release-workflow-test chart-package: OPERATOR_VERSION=$(OPERATOR_VERSION) CHART_VERSION=$(CHART_VERSION) \ HELM_BIN=$${HELM_BIN:-helm} ./.scripts/chart-release.sh package @@ -61,6 +61,10 @@ chart-release-check: chart-release-test: HELM_BIN=$${HELM_BIN:-helm} ./.test/chart-release.sh +release-workflow-test: + ./.test/release-workflow.sh + ./.test/image-release.sh + ## Start local Webhook Relay operator local-run: WATCH_NAMESPACE=$${WATCH_NAMESPACE:-default} go run ./cmd/manager --zap-devel diff --git a/charts/webhookrelay-operator/README.md b/charts/webhookrelay-operator/README.md index d1bd4be..4b403fc 100644 --- a/charts/webhookrelay-operator/README.md +++ b/charts/webhookrelay-operator/README.md @@ -26,6 +26,15 @@ helm upgrade webhookrelay-operator webhookrelay/webhookrelay-operator \ --namespace webhookrelay --version "${CHART_VERSION}" --wait ``` +For an immutable deployment, set `image.digest` to the published multi-platform +operator index digest. It takes precedence over `image.tag`: + +```sh +helm upgrade --install webhookrelay-operator webhookrelay/webhookrelay-operator \ + --namespace webhookrelay --create-namespace --version "${CHART_VERSION}" \ + --set-string image.digest=sha256:REPLACE_WITH_RELEASE_DIGEST +``` + Uninstalling the chart leaves the CRD and all `WebhookRelayForward` objects in place. Delete those objects before uninstalling if their owned relay-agent Deployments should be garbage-collected. Delete the CRD separately only when diff --git a/charts/webhookrelay-operator/templates/deployment.yaml b/charts/webhookrelay-operator/templates/deployment.yaml index 9c5cdf1..b0dc647 100644 --- a/charts/webhookrelay-operator/templates/deployment.yaml +++ b/charts/webhookrelay-operator/templates/deployment.yaml @@ -33,7 +33,7 @@ spec: - name: {{ .Chart.Name }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + image: "{{ .Values.image.repository }}{{ if .Values.image.digest }}@{{ .Values.image.digest }}{{ else }}:{{ .Values.image.tag | default .Chart.AppVersion }}{{ end }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - name: metrics diff --git a/charts/webhookrelay-operator/values.yaml b/charts/webhookrelay-operator/values.yaml index 0672d25..5861e5c 100644 --- a/charts/webhookrelay-operator/values.yaml +++ b/charts/webhookrelay-operator/values.yaml @@ -7,6 +7,8 @@ replicaCount: 1 image: repository: webhookrelay/webhookrelay-operator pullPolicy: Always + # Optional immutable OCI digest. When set, this takes precedence over tag. + digest: "" # Overrides the image tag whose default is the chart appVersion. tag: "0.8.0" diff --git a/docs/modernization-plan.md b/docs/modernization-plan.md index 8c4fbd6..a3fa02a 100644 --- a/docs/modernization-plan.md +++ b/docs/modernization-plan.md @@ -48,8 +48,8 @@ shipped operator default and validated rather than assumed to match either. | 12 | Bump the candidate chart to `0.6.0`. Add an operator-admin-only Relay API endpoint override (environment/Helm configuration, never a CR field) and a stateful fake Relay API used by isolated K3s. Replace the deliberately failing isolated reconcile with deterministic create, update, idempotency, error, and cleanup assertions. | Unit tests validate the production default, absolute HTTP(S) URL and no-userinfo rules, proxy composition, and absence of credentials from logs/status. Isolated K3s proves remote-style convergence without production credentials; production delivery remains separate. | Merged ([#43](https://github.com/webhookrelay/webhookrelay-operator/pull/43)); [CI](https://github.com/webhookrelay/webhookrelay-operator/actions/runs/34352337807) passed unit/lint, Helm, and isolated K3s create/update/output-prune/input-retention/idempotency/failure/recovery coverage | | 13 | Add Kubernetes-standard `RoutingReady`, `AgentReady`, and aggregate `Ready` conditions, top-level and per-condition `observedGeneration`, and actual Deployment rollout readiness while retaining existing status fields as compatibility mirrors. Watch owned Deployment changes, handle missing/unobserved/partial/available/`ProgressDeadlineExceeded` states, retry status conflicts, and return Deployment reconcile failures. | Focused tests cover condition reasons, stable transition times/no write loop, generation changes, status conflicts, deletion, routing failures, and legacy mirrors. Generated CRDs agree; isolated K3s proves false-to-true readiness and update transitions; protected production delivery passes. | Merged ([#44](https://github.com/webhookrelay/webhookrelay-operator/pull/44)); [CI](https://github.com/webhookrelay/webhookrelay-operator/actions/runs/34355001359) and [protected production delivery](https://github.com/webhookrelay/webhookrelay-operator/actions/runs/34355358893) passed | | 14 | Remove/deprecate the duplicate Helm-managed CRD template and define `crds/` as the supported retained CRD path. Add a packaged-chart lifecycle suite: clean install, CR deletion and owned Deployment garbage collection, Helm uninstall with CRD retention, and separately guarded CRD removal only in disposable K3s. Upgrade from the checksum-pinned published chart `0.4.1`, explicitly apply the candidate CRD, and test application-only safe rollback after scaling the operator to zero. | CI verifies checksum `7de5d0afa11405d603c41b81bab1b33f93e0cf6f9ddb3fc7ae5e9a1fd85a4907`, Helm revision transitions, CR UID and legacy `function_id`, Lease/probes/RBAC/schema, zero old/new pod overlap, old-controller-compatible rollback fixtures, no remote duplication, namespaced cleanup, CRD retention, and guarded test teardown. Documentation states CRD upgrade, rollback limitations, and uninstall behavior. | Merged ([#45](https://github.com/webhookrelay/webhookrelay-operator/pull/45)); [CI](https://github.com/webhookrelay/webhookrelay-operator/actions/runs/34358692419) passed the packaged install, real legacy-CRD upgrade, alias mapping, isolated rollback, uninstall, retention, and guarded cleanup lifecycle | -| 15 | Replace legacy chart scripts with deterministic GitHub Actions packaging/publication. Pin tooling, validate the explicit operator `0.8.0` / chart `0.7.0` mapping, refuse overwriting an existing version, produce chart SHA256/checksum metadata, and expose reusable publish steps without creating a GitHub release. | The exact packaged chart passes lint and lifecycle tests; authenticated retries accept only an already-published artifact with identical bytes; generation-bound index updates retain every old entry; the public index URL/digest and artifact converge. Manual dispatch is validation-only. Configure a protected `release` environment with required reviewers and a scoped charts storage identity before enabling writes; image credentials remain part of phase 16. | In progress; legacy Cloud Build trigger disabled; protected `release` environment and bucket-scoped keyless identity configured; Artifact Hub recovered chart `0.6.0` after the legacy stale-index alert | -| 16 | Orchestrate publication from an operator tag reachable from `master`: validate explicit operator/chart/agent mapping; build immutable image and chart artifacts; assert exact amd64/arm64 platform sets; smoke-run both platforms for the operator and both pinned agent variants; run packaged-chart K3s and protected production live delivery against immutable tags; publish the chart; promote `latest`; and create the GitHub release last with chart/checksum assets and image digests. | A dry-run fixture proves job ordering. Build, validation, or chart-publication failures cannot create a public release or move `latest`. If the final release API call fails after promotion, recovery must resume from the digest-verified immutable artifacts without rebuilding or changing `latest`. The production gate includes function behavior and exact cleanup. Release notes record operator, chart, and agent versions plus artifact digests. | Planned; depends on phase 15 | +| 15 | Replace legacy chart scripts with deterministic GitHub Actions packaging/publication. Pin tooling, validate the explicit operator `0.8.0` / chart `0.7.0` mapping, refuse overwriting an existing version, produce chart SHA256/checksum metadata, and expose reusable publish steps without creating a GitHub release. | The exact packaged chart passes lint and lifecycle tests; authenticated retries accept only an already-published artifact with identical bytes; generation-bound index updates retain every old entry; the public index URL/digest and artifact converge. Manual dispatch is validation-only. Configure a protected `release` environment with required reviewers and a scoped charts storage identity before enabling writes; image credentials remain part of phase 16. | Merged ([#46](https://github.com/webhookrelay/webhookrelay-operator/pull/46)); [CI](https://github.com/webhookrelay/webhookrelay-operator/actions/runs/34365477398) passed deterministic packaging, authenticated retry fixtures, and exact packaged-chart lifecycle. Legacy Cloud Build is disabled; the protected environment, bucket-scoped keyless identity, mutable-index revalidation policy, and one-time cache migration are active. | +| 16 | Orchestrate publication from an operator tag reachable from `master`: validate explicit operator/chart/agent mapping; build immutable image and chart artifacts; assert exact amd64/arm64 platform sets; smoke-run both platforms for the operator and both pinned agent variants; run packaged-chart K3s and protected production live delivery against immutable tags; publish the chart; promote `latest`; and create the GitHub release last with chart/checksum assets and image digests. | A dry-run fixture proves job ordering. Build, validation, or chart-publication failures cannot create a public release or move `latest`. If the final release API call fails after promotion, recovery must resume from the digest-verified immutable artifacts without rebuilding or changing `latest`. The production gate includes function behavior and exact cleanup. Release notes record operator, chart, and agent versions plus artifact digests. | In progress ([#47](https://github.com/webhookrelay/webhookrelay-operator/pull/47)); commit-scoped staging, digest deployments, multi-platform manifests/smoke tests, protected live delivery, collision-safe promotion, draft-to-public finalization, and administrator-only `0.*` tag rules are implemented for review | Known follow-ups outside this sequence are per-object API clients for safe concurrent reconciliation and Secret watches/cache invalidation for credential diff --git a/docs/releasing.md b/docs/releasing.md index b440576..72703bd 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -57,3 +57,37 @@ the same bytes, digest, and URL. The existing index received the same metadata policy as a one-time migration before this pipeline was enabled. This prevents the stale-index mismatch previously reported by Artifact Hub. The workflow never creates a GitHub release. + +## Coordinated operator release + +Operator releases run only from a `0.*` tag whose commit is already reachable +from `master`. The tag, chart `appVersion`, default operator image tag, chart +version, and pinned relay-agent version are validated before any credentials are +available. The release workflow then: + +1. builds amd64/arm64 images to a commit-scoped staging tag, or resumes only if + that tag has the expected revision labels; +2. records the OCI index and child-manifest digests, validates the pinned + standard and UBI relay-agent indexes, and executes all three images on both + architectures; +3. deploys the operator by digest into disposable K3s and runs production live + delivery with the relay agent pinned by digest; +4. promotes the verified operator index to the immutable version tag, publishes + the already lifecycle-tested chart, and only then moves `latest`; +5. creates or resumes a matching draft GitHub release, byte-verifies all remote + assets, rechecks the tag commit, and publishes the draft as the final public + mutation. + +Release jobs use one non-cancelling concurrency group. A retry never rebuilds a +published version: staging, version, chart, and draft assets are accepted only +when their revisions, digests, or bytes match. A different existing version tag +or chart fails closed. The legacy release-triggered image workflow is removed, +so publishing the GitHub release cannot start a second build or move `latest`. + +The protected `release` environment gates Docker Hub writes, chart publication, +and the final release. Relay credentials remain confined to the `production` +environment. An active repository ruleset restricts creation, update, and +deletion of `0.*` tags to repository administrators. The release assets include +the deterministic chart, its checksum, +chart metadata, operator metadata, and the complete operator/agent platform +digest manifest.