From 2808e2bb7ba1f44fc363e45e21fd95f18243c921 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 3 Sep 2026 11:57:15 +0200 Subject: [PATCH] fix: build releases on a BuildKit that honours rewrite-timestamp The docker driver bundled with dockerd accepts the exporter's rewrite-timestamp option and then ignores it: layer mtimes keep the build clock, so two builds of one revision produce different images and different measurements. It still normalises the image config's created field, which is the only thing the old guard compared, so the check passed and unreproducible releases shipped anyway. Three builds of 96e1c23b demonstrate it -- CI attempt 1 measured PCR0 41459ff7, attempt 2 measured 98c3915c, and a local rebuild measured f8abaf0f. Only a docker-container BuildKit honours the option, so the release now runs on one and refuses to build otherwise. Replace the created-only guard with a real one: build the enclave image twice and require both to be identical. A normalised created field is not evidence that the layers were normalised, and the manifest's reproducibility claim should be tested by the release that makes it rather than merely asserted. verify-build.sh gets the same driver check. Without it a verifier on the docker driver sees a mismatch against an honest release, which reads as an accusation that the published measurements are dishonest. Both scripts now build the nitro-cli image with buildx --load: a docker-container builder leaves results in the build cache, so build-enclave would either fail on a clean machine or silently reuse a stale image of the same tag. Verified locally: verify-build.sh now reports PASS against a manifest produced by the fixed release path, and both scripts refuse to run on the docker driver. Co-Authored-By: Claude Opus 5 --- .github/workflows/release-and-push.yml | 7 +++ deploy/nitro/build-release-assets.sh | 61 ++++++++++++++++++++------ deploy/nitro/verify-build.sh | 17 ++++++- 3 files changed, 70 insertions(+), 15 deletions(-) diff --git a/.github/workflows/release-and-push.yml b/.github/workflows/release-and-push.yml index 1600373..2c21e0c 100644 --- a/.github/workflows/release-and-push.yml +++ b/.github/workflows/release-and-push.yml @@ -102,6 +102,13 @@ jobs: echo "digest=$image_digest" >> "$GITHUB_OUTPUT" docker buildx imagetools inspect "$ECR_REGISTRY/$IMAGE_NAME@$image_digest" + # The docker driver silently ignores rewrite-timestamp, so every release + # measures differently. Only docker-container honours it. + - name: Set up Buildx (docker-container driver) + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 + with: + driver: docker-container + - name: Build EIF and release metadata id: enclave env: diff --git a/deploy/nitro/build-release-assets.sh b/deploy/nitro/build-release-assets.sh index fb8c02d..3f58741 100755 --- a/deploy/nitro/build-release-assets.sh +++ b/deploy/nitro/build-release-assets.sh @@ -88,20 +88,35 @@ artifact_dir_absolute="$(realpath "$ARTIFACT_DIR")" # --provenance=false keeps a build-time attestation (which embeds the clock and # the runner identity) out of the image. Without both, two builds of identical # source produce different layers and therefore different measurements. -docker buildx build --pull --no-cache --provenance=false \ - --label "org.opencontainers.image.source=https://github.com/${SOURCE_REPOSITORY}" \ - --label "org.opencontainers.image.revision=${SOURCE_REVISION}" \ - --label "org.opencontainers.image.version=${VERSION}" \ - --build-arg "SOURCE_REVISION=${SOURCE_REVISION}" \ - --build-arg "AWS_REGION=${AWS_REGION}" \ - --build-arg "KMS_KEY_ARN=${KMS_KEY_ARN}" \ - -f apps/gateway/Dockerfile.enclave \ - --output "type=docker,name=${enclave_image},rewrite-timestamp=true" \ - . +# +# The docker driver accepts rewrite-timestamp and ignores it, normalising only +# the config's created field, so it must not be used to cut a release. +# awk must not exit early here: SIGPIPE on docker would abort the script. +builder_driver="$(docker buildx inspect --bootstrap 2>/dev/null | awk -F': *' '/^Driver:/{if (driver == "") driver = $2} END {print driver}')" || builder_driver="" +if [[ "$builder_driver" != "docker-container" ]]; then + printf 'reproducible build check failed: buildx driver is %s, need docker-container\n' \ + "${builder_driver:-unknown}" >&2 + printf 'The docker driver silently ignores rewrite-timestamp, so this build would\n' >&2 + printf 'produce measurements nobody else can reproduce. Create a builder with:\n' >&2 + printf ' docker buildx create --driver docker-container --use\n' >&2 + exit 1 +fi + +build_enclave_image() { + docker buildx build --pull --no-cache --provenance=false \ + --label "org.opencontainers.image.source=https://github.com/${SOURCE_REPOSITORY}" \ + --label "org.opencontainers.image.revision=${SOURCE_REVISION}" \ + --label "org.opencontainers.image.version=${VERSION}" \ + --build-arg "SOURCE_REVISION=${SOURCE_REVISION}" \ + --build-arg "AWS_REGION=${AWS_REGION}" \ + --build-arg "KMS_KEY_ARN=${KMS_KEY_ARN}" \ + -f apps/gateway/Dockerfile.enclave \ + --output "type=docker,name=$1,rewrite-timestamp=true" \ + . +} + +build_enclave_image "$enclave_image" -# A buildx too old to honour rewrite-timestamp would still succeed here and -# publish a release that nobody can reproduce, so prove the normalisation -# actually applied rather than assuming it did. image_created_epoch="$(date -u -d "$(docker image inspect --format '{{.Created}}' "$enclave_image")" +%s)" if [[ "$image_created_epoch" != "$SOURCE_DATE_EPOCH" ]]; then printf 'reproducible build check failed: image timestamp is %s, expected %s\n' \ @@ -111,7 +126,25 @@ if [[ "$image_created_epoch" != "$SOURCE_DATE_EPOCH" ]]; then exit 1 fi -docker build --pull \ +# A normalised created field is not proof the layers were normalised, so test +# determinism by building twice instead of asserting it in the manifest. +determinism_image="${enclave_image}-determinism-check" +build_enclave_image "$determinism_image" +first_image_id="$(docker image inspect --format '{{.Id}}' "$enclave_image")" +second_image_id="$(docker image inspect --format '{{.Id}}' "$determinism_image")" +docker image rm -f "$determinism_image" >/dev/null 2>&1 || true +if [[ "$first_image_id" != "$second_image_id" ]]; then + printf 'reproducible build check failed: two builds of %s produced different images\n' \ + "$SOURCE_REVISION" >&2 + printf ' first: %s\n' "$first_image_id" >&2 + printf ' second: %s\n' "$second_image_id" >&2 + printf 'Publishing these measurements would break deploy/nitro/verify-build.sh.\n' >&2 + exit 1 +fi + +# --load is required: a docker-container builder leaves results in the build +# cache, and build-enclave below needs the image in the daemon. +docker buildx build --pull --load \ -f deploy/nitro/nitro-cli-builder.Dockerfile \ -t "$nitro_cli_image" \ deploy/nitro diff --git a/deploy/nitro/verify-build.sh b/deploy/nitro/verify-build.sh index 8a724a7..c6d906d 100755 --- a/deploy/nitro/verify-build.sh +++ b/deploy/nitro/verify-build.sh @@ -38,6 +38,19 @@ docker buildx version >/dev/null 2>&1 || { printf 'docker buildx is required: the reproducible build depends on rewrite-timestamp\n' >&2 exit 1 } + +# The docker driver ignores rewrite-timestamp and would report a mismatch +# against an honest release, so refuse to run rather than accuse falsely. +# awk must not exit early here: SIGPIPE on docker would abort the script. +verify_builder_driver="$(docker buildx inspect --bootstrap 2>/dev/null | awk -F': *' '/^Driver:/{if (driver == "") driver = $2} END {print driver}')" || verify_builder_driver="" +if [[ "$verify_builder_driver" != "docker-container" ]]; then + printf 'cannot verify with the %s buildx driver: it ignores rewrite-timestamp\n' \ + "${verify_builder_driver:-unknown}" >&2 + printf 'and would report a mismatch for a correctly built release. Run:\n' >&2 + printf ' docker buildx create --driver docker-container --use\n' >&2 + printf 'and try again.\n' >&2 + exit 1 +fi [[ -f "$manifest_path" ]] || { printf 'no such manifest: %s\n' "$manifest_path" >&2; exit 1; } # Newline-separated rather than @tsv: tab is an IFS whitespace character, so a @@ -124,7 +137,9 @@ fail() { printf 'Rebuilding %s (source_date_epoch=%s)\nThis takes a few minutes.\n' \ "${revision:0:12}" "$source_date_epoch" -docker build --pull -q \ +# --load: a docker-container builder keeps results in the build cache, and +# build-enclave below needs the image in the daemon. +docker buildx build --pull -q --load \ -f "$context/deploy/nitro/nitro-cli-builder.Dockerfile" \ -t "$nitro_cli_image" \ "$context/deploy/nitro" >>"$build_log" 2>&1 || fail