From 4fe675d2ec10976fa6a197aad4c43ec5e563d26e Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Sun, 1 Nov 2026 11:13:17 -0500 Subject: [PATCH 1/3] fix: the nightly database backup only uploads into a bucket owned by the S3 keys' account For a remote S3 endpoint the backup now lists the account's buckets first and refuses to upload when the configured bucket is not among them. On AWS every call also passes --expected-bucket-owner with the account id from STS. Keys that cannot list buckets still upload, with a note. The in-cluster store is unchanged. --- .github/workflows/panel-scripts.yml | 3 + base/backups/postgres-backup-cronjob.yaml | 48 +++++- tests/scripts/postgres_backup_test.sh | 181 ++++++++++++++++++++++ 3 files changed, 229 insertions(+), 3 deletions(-) create mode 100644 tests/scripts/postgres_backup_test.sh diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml index 881ee2b..936d76f 100644 --- a/.github/workflows/panel-scripts.yml +++ b/.github/workflows/panel-scripts.yml @@ -18,6 +18,7 @@ on: - "tests/scripts/**" - base/kustomization.yaml - "base/system-upgrade/**" + - "base/backups/**" - .github/workflows/panel-scripts.yml pull_request: paths: @@ -35,6 +36,7 @@ on: - "tests/scripts/**" - base/kustomization.yaml - "base/system-upgrade/**" + - "base/backups/**" - .github/workflows/panel-scripts.yml jobs: @@ -55,3 +57,4 @@ jobs: - run: bash tests/scripts/dev_game_server_test.sh - run: bash tests/scripts/gpu_nodes_test.sh - run: bash tests/scripts/service_test.sh + - run: bash tests/scripts/postgres_backup_test.sh diff --git a/base/backups/postgres-backup-cronjob.yaml b/base/backups/postgres-backup-cronjob.yaml index a6d9b95..2f621c0 100644 --- a/base/backups/postgres-backup-cronjob.yaml +++ b/base/backups/postgres-backup-cronjob.yaml @@ -39,6 +39,7 @@ spec: zip -j "$ZIP_FILE" "$DUMP_FILE" ENDPOINT="${S3_ENDPOINT:-rustfs}" + REMOTE=false # Both names mean the in-cluster store (see the minio alias # Service in base/rustfs/service.yaml). Anything else is a @@ -47,6 +48,7 @@ spec: ENDPOINT="http://$ENDPOINT:9000" else ENDPOINT="https://$S3_ENDPOINT" + REMOTE=true fi export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" @@ -55,9 +57,48 @@ spec: S3_DB_BACKUP_BUCKET="${S3_DB_BACKUP_BUCKET:-5stack-db-backups}" + # Remote bucket names are global, so only upload into a bucket + # owned by the account these keys belong to. On AWS every call + # also pins that owner, which takes the 12-digit account id from + # STS (list-buckets' Owner.ID is a canonical id S3 rejects). + set -- + ACCOUNT_ID="" + if [ "$REMOTE" = true ]; then + case "$S3_ENDPOINT" in + *.amazonaws.com) + if ! ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text); then + echo "ERROR: could not look up the AWS account these S3 keys belong to" + exit 1 + fi + set -- --expected-bucket-owner "$ACCOUNT_ID" + ;; + esac + + if BUCKETS=$(aws s3api list-buckets --endpoint-url "$ENDPOINT" --query 'Buckets[].Name' --output text 2>/tmp/list-buckets.err); then + FOUND=false + for NAME in $BUCKETS; do + if [ "$NAME" = "$S3_DB_BACKUP_BUCKET" ]; then + FOUND=true + fi + done + if [ "$FOUND" = false ]; then + echo "ERROR: the backup bucket $S3_DB_BACKUP_BUCKET isn't in the account these S3 keys belong to, so the database dump was NOT uploaded. Create it in your account or set S3_DB_BACKUP_BUCKET" + exit 1 + fi + elif grep -Eq 'AccessDenied|\(403\)' /tmp/list-buckets.err; then + if [ -z "$ACCOUNT_ID" ]; then + echo "Note: couldn't check that $S3_DB_BACKUP_BUCKET is owned by this account because these S3 keys can't list buckets" + fi + else + cat /tmp/list-buckets.err + echo "ERROR: S3 upload failed, could not list the buckets on $ENDPOINT" + exit 1 + fi + fi + echo "Uploading to $ENDPOINT : $S3_DB_BACKUP_BUCKET/backup-$TS.zip" - if ! aws s3 cp "$ZIP_FILE" "s3://$S3_DB_BACKUP_BUCKET/backup-$TS.zip" --endpoint-url "$ENDPOINT"; then + if ! aws s3 cp "$ZIP_FILE" "s3://$S3_DB_BACKUP_BUCKET/backup-$TS.zip" --endpoint-url "$ENDPOINT" "$@"; then echo "ERROR: S3 upload failed" exit 1 fi @@ -68,7 +109,8 @@ spec: --bucket "$S3_DB_BACKUP_BUCKET" \ --query 'reverse(sort_by(Contents, &LastModified))[].Key' \ --output text \ - --endpoint-url "$ENDPOINT" || true) + --endpoint-url "$ENDPOINT" \ + "$@" || true) DELETEABLE_KEYS=$(echo "$KEYS" | tr '\t' '\n' | awk 'NR>7') @@ -76,7 +118,7 @@ spec: echo "Pruning old backups, keeping latest 7" echo "$DELETEABLE_KEYS" | while IFS= read -r KEY; do echo "Deleting old backup: $KEY" - aws s3 rm "s3://$S3_DB_BACKUP_BUCKET/$KEY" --endpoint-url "$ENDPOINT" || true + aws s3 rm "s3://$S3_DB_BACKUP_BUCKET/$KEY" --endpoint-url "$ENDPOINT" "$@" || true done echo "Pruning complete" fi diff --git a/tests/scripts/postgres_backup_test.sh b/tests/scripts/postgres_backup_test.sh new file mode 100644 index 0000000..d307947 --- /dev/null +++ b/tests/scripts/postgres_backup_test.sh @@ -0,0 +1,181 @@ +#!/bin/bash + +# shellcheck source=tests/scripts/lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +AWS_ENDPOINT="s3.us-east-1.amazonaws.com" +B2_ENDPOINT="s3.us-west-004.backblazeb2.com" +ACCOUNT_ID="111122223333" + +backup_script() { + awk ' + /^ *- \|$/ { in_block = 1; indent = -1; next } + in_block { + if ($0 ~ /^ *$/) { print ""; next } + match($0, /^ */) + if (indent < 0) { indent = RLENGTH } + if (RLENGTH < indent) { exit } + print substr($0, indent + 1) + } + ' "$TEST_ROOT/base/backups/postgres-backup-cronjob.yaml" +} + +setup_stubs() { + mkdir -p "$WORK_DIR/bin" + cat >"$WORK_DIR/bin/apk" <<'EOF' +#!/bin/bash +EOF + cat >"$WORK_DIR/bin/pg_dump" <<'EOF' +#!/bin/bash +while [ $# -gt 0 ]; do + if [ "$1" = "-f" ]; then + echo dump >"$2" + fi + shift +done +EOF + cat >"$WORK_DIR/bin/zip" <<'EOF' +#!/bin/bash +cp "$3" "$2" +EOF + cat >"$WORK_DIR/bin/aws" <<'EOF' +#!/bin/bash +echo "$*" >>"$WORK_DIR/aws-calls" +case "$1 $2" in + "s3api list-buckets") + if [ -f "$WORK_DIR/list-buckets-error" ]; then + cat "$WORK_DIR/list-buckets-error" >&2 + exit 254 + fi + cat "$WORK_DIR/buckets" + ;; + "sts get-caller-identity") echo "$FAKE_ACCOUNT_ID" ;; + "s3api list-objects-v2") if [ -f "$WORK_DIR/objects" ]; then cat "$WORK_DIR/objects"; fi ;; +esac +EOF + chmod +x "$WORK_DIR/bin/apk" "$WORK_DIR/bin/pg_dump" "$WORK_DIR/bin/zip" "$WORK_DIR/bin/aws" +} + +buckets() { + local IFS=$'\t' + printf '%s\n' "$*" >"$WORK_DIR/buckets" +} + +deny_bucket_listing() { + echo "An error occurred (AccessDenied) when calling the ListBuckets operation: Access Denied" >"$WORK_DIR/list-buckets-error" +} + +eight_backups() { + printf 'backup-8.zip\tbackup-7.zip\tbackup-6.zip\tbackup-5.zip\tbackup-4.zip\tbackup-3.zip\tbackup-2.zip\tbackup-1.zip\n' >"$WORK_DIR/objects" +} + +run_backup() { + setup_stubs + backup_script | sed "s#/tmp/#$WORK_DIR/#g" >"$WORK_DIR/backup.sh" + PATH="$WORK_DIR/bin:$PATH" \ + S3_ENDPOINT="$1" \ + S3_ACCESS_KEY=key \ + S3_SECRET=secret \ + POSTGRES_HOST=timescaledb \ + POSTGRES_PORT=5432 \ + POSTGRES_USER=hasura \ + POSTGRES_PASSWORD=password \ + POSTGRES_DB=hasura \ + FAKE_ACCOUNT_ID="$ACCOUNT_ID" \ + bash "$WORK_DIR/backup.sh" >"$WORK_DIR/output" 2>&1 + STATUS=$? +} + +aws_calls() { + cat "$WORK_DIR/aws-calls" 2>/dev/null || true +} + +calls_to() { + grep "^$1" "$WORK_DIR/aws-calls" 2>/dev/null || true +} + +assert_succeeded() { + [ "$STATUS" -eq 0 ] || fail "backup exited $STATUS: $(cat "$WORK_DIR/output")" +} + +test_a_listed_bucket_on_aws_is_uploaded_to_only_as_this_account() { + buckets other-bucket 5stack-db-backups + eight_backups + run_backup "$AWS_ENDPOINT" + assert_succeeded + assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" + assert_contains "$(calls_to "s3 cp ")" "--endpoint-url https://$AWS_ENDPOINT --expected-bucket-owner $ACCOUNT_ID" + assert_contains "$(calls_to "s3api list-objects-v2 ")" "--expected-bucket-owner $ACCOUNT_ID" + assert_eq "$(calls_to "s3 rm ")" "s3 rm s3://5stack-db-backups/backup-1.zip --endpoint-url https://$AWS_ENDPOINT --expected-bucket-owner $ACCOUNT_ID" +} + +test_a_listed_bucket_on_b2_is_uploaded_to_without_an_aws_owner() { + buckets 5stack-db-backups + eight_backups + run_backup "$B2_ENDPOINT" + assert_succeeded + assert_contains "$(calls_to "s3api list-buckets ")" "--endpoint-url https://$B2_ENDPOINT --query Buckets[].Name --output text" + assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" + assert_eq "$(calls_to "s3 rm ")" "s3 rm s3://5stack-db-backups/backup-1.zip --endpoint-url https://$B2_ENDPOINT" + assert_not_contains "$(aws_calls)" "--expected-bucket-owner" + assert_not_contains "$(aws_calls)" "sts " +} + +test_a_bucket_outside_the_account_is_never_uploaded_to() { + for endpoint in "$B2_ENDPOINT" "$AWS_ENDPOINT"; do + rm -f "$WORK_DIR/aws-calls" + buckets 5stack-db-backups-old other-bucket + run_backup "$endpoint" + [ "$STATUS" -ne 0 ] || fail "backup to $endpoint succeeded with a bucket outside the account" + assert_contains "$(cat "$WORK_DIR/output")" "the backup bucket 5stack-db-backups isn't in the account these S3 keys belong to, so the database dump was NOT uploaded" + assert_eq "$(calls_to "s3 cp ")" "" + done +} + +test_an_account_with_no_buckets_is_never_uploaded_to() { + buckets + run_backup "$B2_ENDPOINT" + [ "$STATUS" -ne 0 ] || fail "backup succeeded into an account with no buckets" + assert_eq "$(calls_to "s3 cp ")" "" +} + +test_keys_that_cannot_list_buckets_still_upload_with_a_note() { + deny_bucket_listing + run_backup "$B2_ENDPOINT" + assert_succeeded + assert_contains "$(cat "$WORK_DIR/output")" "because these S3 keys can't list buckets" + assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" +} + +test_aws_keys_that_cannot_list_buckets_still_upload_only_as_this_account() { + deny_bucket_listing + run_backup "$AWS_ENDPOINT" + assert_succeeded + assert_contains "$(calls_to "s3 cp ")" "--expected-bucket-owner $ACCOUNT_ID" + assert_not_contains "$(cat "$WORK_DIR/output")" "can't list buckets" +} + +test_a_listing_that_fails_for_another_reason_stops_before_uploading() { + echo "Could not connect to the endpoint URL: \"https://$B2_ENDPOINT/\"" >"$WORK_DIR/list-buckets-error" + run_backup "$B2_ENDPOINT" + [ "$STATUS" -ne 0 ] || fail "backup succeeded when the bucket listing failed" + assert_contains "$(cat "$WORK_DIR/output")" "Could not connect to the endpoint URL" + assert_eq "$(calls_to "s3 cp ")" "" +} + +test_the_in_cluster_store_uploads_without_listing_buckets() { + for endpoint in "" rustfs minio; do + rm -f "$WORK_DIR/aws-calls" + eight_backups + run_backup "$endpoint" + assert_succeeded + host="${endpoint:-rustfs}" + assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" + assert_contains "$(calls_to "s3 cp ")" "--endpoint-url http://$host:9000" + assert_eq "$(calls_to "s3 rm ")" "s3 rm s3://5stack-db-backups/backup-1.zip --endpoint-url http://$host:9000" + assert_not_contains "$(aws_calls)" "list-buckets" + assert_not_contains "$(aws_calls)" "--expected-bucket-owner" + done +} + +run_tests From 3fca854efb1729b3d7b3b42a8c411463849b7af0 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Sun, 1 Nov 2026 11:24:52 -0500 Subject: [PATCH 2/3] fix: the AWS backup checks its bucket owner with head-bucket, which takes --expected-bucket-owner where aws s3 cp and rm do not aws s3 cp and aws s3 rm reject --expected-bucket-owner, so every AWS backup failed. On AWS the owner is now checked with s3api head-bucket before the upload, s3 cp runs without the option and keeps multipart, and the prune lists and deletes through s3api with the owner. The AWS endpoint match ignores case, a scheme and a trailing slash, and covers amazonaws.com.cn. A 401 on the bucket listing counts as keys that cannot list buckets. --- base/backups/postgres-backup-cronjob.yaml | 25 ++++++--- tests/scripts/postgres_backup_test.sh | 65 +++++++++++++++++++---- 2 files changed, 71 insertions(+), 19 deletions(-) diff --git a/base/backups/postgres-backup-cronjob.yaml b/base/backups/postgres-backup-cronjob.yaml index 2f621c0..ad16dc3 100644 --- a/base/backups/postgres-backup-cronjob.yaml +++ b/base/backups/postgres-backup-cronjob.yaml @@ -58,14 +58,18 @@ spec: S3_DB_BACKUP_BUCKET="${S3_DB_BACKUP_BUCKET:-5stack-db-backups}" # Remote bucket names are global, so only upload into a bucket - # owned by the account these keys belong to. On AWS every call - # also pins that owner, which takes the 12-digit account id from - # STS (list-buckets' Owner.ID is a canonical id S3 rejects). + # owned by the account these keys belong to. On AWS the owner + # check takes the 12-digit account id from STS (list-buckets' + # Owner.ID is a canonical id S3 rejects), and only the s3api + # calls get it: aws s3 cp and rm reject --expected-bucket-owner. set -- ACCOUNT_ID="" if [ "$REMOTE" = true ]; then - case "$S3_ENDPOINT" in - *.amazonaws.com) + ENDPOINT_HOST=$(printf '%s' "$S3_ENDPOINT" | tr '[:upper:]' '[:lower:]') + ENDPOINT_HOST="${ENDPOINT_HOST#*://}" + ENDPOINT_HOST="${ENDPOINT_HOST%/}" + case "$ENDPOINT_HOST" in + *.amazonaws.com | *.amazonaws.com.cn) if ! ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text); then echo "ERROR: could not look up the AWS account these S3 keys belong to" exit 1 @@ -85,7 +89,7 @@ spec: echo "ERROR: the backup bucket $S3_DB_BACKUP_BUCKET isn't in the account these S3 keys belong to, so the database dump was NOT uploaded. Create it in your account or set S3_DB_BACKUP_BUCKET" exit 1 fi - elif grep -Eq 'AccessDenied|\(403\)' /tmp/list-buckets.err; then + elif grep -Eq 'AccessDenied|Unauthorized|\(40[13]\)' /tmp/list-buckets.err; then if [ -z "$ACCOUNT_ID" ]; then echo "Note: couldn't check that $S3_DB_BACKUP_BUCKET is owned by this account because these S3 keys can't list buckets" fi @@ -94,11 +98,16 @@ spec: echo "ERROR: S3 upload failed, could not list the buckets on $ENDPOINT" exit 1 fi + + if [ -n "$ACCOUNT_ID" ] && ! aws s3api head-bucket --bucket "$S3_DB_BACKUP_BUCKET" --endpoint-url "$ENDPOINT" "$@"; then + echo "ERROR: the backup bucket $S3_DB_BACKUP_BUCKET isn't owned by AWS account $ACCOUNT_ID that these S3 keys belong to, so the database dump was NOT uploaded. Create it in your account or set S3_DB_BACKUP_BUCKET" + exit 1 + fi fi echo "Uploading to $ENDPOINT : $S3_DB_BACKUP_BUCKET/backup-$TS.zip" - if ! aws s3 cp "$ZIP_FILE" "s3://$S3_DB_BACKUP_BUCKET/backup-$TS.zip" --endpoint-url "$ENDPOINT" "$@"; then + if ! aws s3 cp "$ZIP_FILE" "s3://$S3_DB_BACKUP_BUCKET/backup-$TS.zip" --endpoint-url "$ENDPOINT"; then echo "ERROR: S3 upload failed" exit 1 fi @@ -118,7 +127,7 @@ spec: echo "Pruning old backups, keeping latest 7" echo "$DELETEABLE_KEYS" | while IFS= read -r KEY; do echo "Deleting old backup: $KEY" - aws s3 rm "s3://$S3_DB_BACKUP_BUCKET/$KEY" --endpoint-url "$ENDPOINT" "$@" || true + aws s3api delete-object --bucket "$S3_DB_BACKUP_BUCKET" --key "$KEY" --endpoint-url "$ENDPOINT" "$@" || true done echo "Pruning complete" fi diff --git a/tests/scripts/postgres_backup_test.sh b/tests/scripts/postgres_backup_test.sh index d307947..27ace79 100644 --- a/tests/scripts/postgres_backup_test.sh +++ b/tests/scripts/postgres_backup_test.sh @@ -41,7 +41,24 @@ EOF cat >"$WORK_DIR/bin/aws" <<'EOF' #!/bin/bash echo "$*" >>"$WORK_DIR/aws-calls" +expected_owner="$(printf '%s' " $* " | sed -n 's/.* --expected-bucket-owner \([^ ]*\) .*/\1/p')" case "$1 $2" in + "s3 cp" | "s3 rm") + if [ -n "$expected_owner" ]; then + printf '\nUnknown options: --expected-bucket-owner, %s\n\n' "$expected_owner" >&2 + exit 252 + fi + ;; + "s3api head-bucket") + owner="$FAKE_ACCOUNT_ID" + if [ -f "$WORK_DIR/bucket-owner" ]; then + owner="$(cat "$WORK_DIR/bucket-owner")" + fi + if [ -n "$expected_owner" ] && [ "$expected_owner" != "$owner" ]; then + echo "An error occurred (403) when calling the HeadBucket operation: Forbidden" >&2 + exit 254 + fi + ;; "s3api list-buckets") if [ -f "$WORK_DIR/list-buckets-error" ]; then cat "$WORK_DIR/list-buckets-error" >&2 @@ -62,7 +79,7 @@ buckets() { } deny_bucket_listing() { - echo "An error occurred (AccessDenied) when calling the ListBuckets operation: Access Denied" >"$WORK_DIR/list-buckets-error" + echo "${1:-An error occurred (AccessDenied) when calling the ListBuckets operation: Access Denied}" >"$WORK_DIR/list-buckets-error" } eight_backups() { @@ -103,10 +120,30 @@ test_a_listed_bucket_on_aws_is_uploaded_to_only_as_this_account() { eight_backups run_backup "$AWS_ENDPOINT" assert_succeeded + assert_eq "$(calls_to "s3api head-bucket ")" "s3api head-bucket --bucket 5stack-db-backups --endpoint-url https://$AWS_ENDPOINT --expected-bucket-owner $ACCOUNT_ID" assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" - assert_contains "$(calls_to "s3 cp ")" "--endpoint-url https://$AWS_ENDPOINT --expected-bucket-owner $ACCOUNT_ID" assert_contains "$(calls_to "s3api list-objects-v2 ")" "--expected-bucket-owner $ACCOUNT_ID" - assert_eq "$(calls_to "s3 rm ")" "s3 rm s3://5stack-db-backups/backup-1.zip --endpoint-url https://$AWS_ENDPOINT --expected-bucket-owner $ACCOUNT_ID" + assert_eq "$(calls_to "s3api delete-object ")" "s3api delete-object --bucket 5stack-db-backups --key backup-1.zip --endpoint-url https://$AWS_ENDPOINT --expected-bucket-owner $ACCOUNT_ID" +} + +test_every_spelling_of_an_aws_endpoint_checks_the_owner() { + for endpoint in S3.US-EAST-1.AMAZONAWS.COM "$AWS_ENDPOINT/" "https://$AWS_ENDPOINT" s3.cn-north-1.amazonaws.com.cn; do + rm -f "$WORK_DIR/aws-calls" + buckets 5stack-db-backups + run_backup "$endpoint" + assert_succeeded + assert_contains "$(calls_to "sts ")" "get-caller-identity" + assert_contains "$(calls_to "s3api head-bucket ")" "--expected-bucket-owner $ACCOUNT_ID" + done +} + +test_an_aws_bucket_owned_by_another_account_is_never_uploaded_to() { + deny_bucket_listing + echo 444455556666 >"$WORK_DIR/bucket-owner" + run_backup "$AWS_ENDPOINT" + [ "$STATUS" -ne 0 ] || fail "backup succeeded into a bucket owned by another account" + assert_contains "$(cat "$WORK_DIR/output")" "the backup bucket 5stack-db-backups isn't owned by AWS account $ACCOUNT_ID that these S3 keys belong to, so the database dump was NOT uploaded" + assert_eq "$(calls_to "s3 cp ")" "" } test_a_listed_bucket_on_b2_is_uploaded_to_without_an_aws_owner() { @@ -116,9 +153,10 @@ test_a_listed_bucket_on_b2_is_uploaded_to_without_an_aws_owner() { assert_succeeded assert_contains "$(calls_to "s3api list-buckets ")" "--endpoint-url https://$B2_ENDPOINT --query Buckets[].Name --output text" assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" - assert_eq "$(calls_to "s3 rm ")" "s3 rm s3://5stack-db-backups/backup-1.zip --endpoint-url https://$B2_ENDPOINT" + assert_eq "$(calls_to "s3api delete-object ")" "s3api delete-object --bucket 5stack-db-backups --key backup-1.zip --endpoint-url https://$B2_ENDPOINT" assert_not_contains "$(aws_calls)" "--expected-bucket-owner" assert_not_contains "$(aws_calls)" "sts " + assert_not_contains "$(aws_calls)" "head-bucket" } test_a_bucket_outside_the_account_is_never_uploaded_to() { @@ -140,18 +178,23 @@ test_an_account_with_no_buckets_is_never_uploaded_to() { } test_keys_that_cannot_list_buckets_still_upload_with_a_note() { - deny_bucket_listing - run_backup "$B2_ENDPOINT" - assert_succeeded - assert_contains "$(cat "$WORK_DIR/output")" "because these S3 keys can't list buckets" - assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" + for denial in "An error occurred (AccessDenied) when calling the ListBuckets operation: Access Denied" \ + "An error occurred (401) when calling the ListBuckets operation: Unauthorized"; do + rm -f "$WORK_DIR/aws-calls" + deny_bucket_listing "$denial" + run_backup "$B2_ENDPOINT" + assert_succeeded + assert_contains "$(cat "$WORK_DIR/output")" "because these S3 keys can't list buckets" + assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" + done } test_aws_keys_that_cannot_list_buckets_still_upload_only_as_this_account() { deny_bucket_listing run_backup "$AWS_ENDPOINT" assert_succeeded - assert_contains "$(calls_to "s3 cp ")" "--expected-bucket-owner $ACCOUNT_ID" + assert_contains "$(calls_to "s3api head-bucket ")" "--expected-bucket-owner $ACCOUNT_ID" + assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" assert_not_contains "$(cat "$WORK_DIR/output")" "can't list buckets" } @@ -172,7 +215,7 @@ test_the_in_cluster_store_uploads_without_listing_buckets() { host="${endpoint:-rustfs}" assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-" assert_contains "$(calls_to "s3 cp ")" "--endpoint-url http://$host:9000" - assert_eq "$(calls_to "s3 rm ")" "s3 rm s3://5stack-db-backups/backup-1.zip --endpoint-url http://$host:9000" + assert_eq "$(calls_to "s3api delete-object ")" "s3api delete-object --bucket 5stack-db-backups --key backup-1.zip --endpoint-url http://$host:9000" assert_not_contains "$(aws_calls)" "list-buckets" assert_not_contains "$(aws_calls)" "--expected-bucket-owner" done From 468ececbb4484744c8f70156d63d39568f6d1cde Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Sun, 1 Nov 2026 11:27:01 -0500 Subject: [PATCH 3/3] Name missing s3:ListBucket in the AWS owner check error --- base/backups/postgres-backup-cronjob.yaml | 2 +- tests/scripts/postgres_backup_test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/base/backups/postgres-backup-cronjob.yaml b/base/backups/postgres-backup-cronjob.yaml index ad16dc3..e05a90a 100644 --- a/base/backups/postgres-backup-cronjob.yaml +++ b/base/backups/postgres-backup-cronjob.yaml @@ -100,7 +100,7 @@ spec: fi if [ -n "$ACCOUNT_ID" ] && ! aws s3api head-bucket --bucket "$S3_DB_BACKUP_BUCKET" --endpoint-url "$ENDPOINT" "$@"; then - echo "ERROR: the backup bucket $S3_DB_BACKUP_BUCKET isn't owned by AWS account $ACCOUNT_ID that these S3 keys belong to, so the database dump was NOT uploaded. Create it in your account or set S3_DB_BACKUP_BUCKET" + echo "ERROR: the backup bucket $S3_DB_BACKUP_BUCKET isn't owned by AWS account $ACCOUNT_ID that these S3 keys belong to, or the keys lack s3:ListBucket on it, so the database dump was NOT uploaded. Create it in your account, set S3_DB_BACKUP_BUCKET, or grant s3:ListBucket" exit 1 fi fi diff --git a/tests/scripts/postgres_backup_test.sh b/tests/scripts/postgres_backup_test.sh index 27ace79..50db6ba 100644 --- a/tests/scripts/postgres_backup_test.sh +++ b/tests/scripts/postgres_backup_test.sh @@ -142,7 +142,7 @@ test_an_aws_bucket_owned_by_another_account_is_never_uploaded_to() { echo 444455556666 >"$WORK_DIR/bucket-owner" run_backup "$AWS_ENDPOINT" [ "$STATUS" -ne 0 ] || fail "backup succeeded into a bucket owned by another account" - assert_contains "$(cat "$WORK_DIR/output")" "the backup bucket 5stack-db-backups isn't owned by AWS account $ACCOUNT_ID that these S3 keys belong to, so the database dump was NOT uploaded" + assert_contains "$(cat "$WORK_DIR/output")" "the backup bucket 5stack-db-backups isn't owned by AWS account $ACCOUNT_ID that these S3 keys belong to, or the keys lack s3:ListBucket on it, so the database dump was NOT uploaded" assert_eq "$(calls_to "s3 cp ")" "" }