lukepolo · Nov 1, 2026 · Nov 1, 2026 · Nov 1, 2026 · Nov 1, 2026
diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml
@@ -18,6 +18,7 @@ on:
       - "tests/scripts/**"
       - base/kustomization.yaml
       - "base/system-upgrade/**"
+      - "base/backups/**"
       - .github/workflows/panel-scripts.yml
   pull_request:
     paths:
@@ -35,6 +36,7 @@ on:
       - "tests/scripts/**"
       - base/kustomization.yaml
       - "base/system-upgrade/**"
+      - "base/backups/**"
       - .github/workflows/panel-scripts.yml
 
 jobs:
@@ -55,3 +57,4 @@ jobs:
       - run: bash tests/scripts/dev_game_server_test.sh
       - run: bash tests/scripts/gpu_nodes_test.sh
       - run: bash tests/scripts/service_test.sh
+      - run: bash tests/scripts/postgres_backup_test.sh
diff --git a/base/backups/postgres-backup-cronjob.yaml b/base/backups/postgres-backup-cronjob.yaml
@@ -39,6 +39,7 @@ spec:
                   zip -j "$ZIP_FILE" "$DUMP_FILE"
 
                   ENDPOINT="${S3_ENDPOINT:-rustfs}"
+                  REMOTE=false
 
                   # Both names mean the in-cluster store (see the minio alias
                   # Service in base/rustfs/service.yaml). Anything else is a
@@ -47,6 +48,7 @@ spec:
                     ENDPOINT="http://$ENDPOINT:9000"
                   else
                     ENDPOINT="https://$S3_ENDPOINT"
+                    REMOTE=true
                   fi
 
                   export AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY"
@@ -55,6 +57,54 @@ spec:
 
                   S3_DB_BACKUP_BUCKET="${S3_DB_BACKUP_BUCKET:-5stack-db-backups}"
 
+                  # Remote bucket names are global, so only upload into a bucket
+                  # owned by the account these keys belong to. On AWS the owner
+                  # check takes the 12-digit account id from STS (list-buckets'
+                  # Owner.ID is a canonical id S3 rejects), and only the s3api
+                  # calls get it: aws s3 cp and rm reject --expected-bucket-owner.
+                  set --
+                  ACCOUNT_ID=""
+                  if [ "$REMOTE" = true ]; then
+                    ENDPOINT_HOST=$(printf '%s' "$S3_ENDPOINT" | tr '[:upper:]' '[:lower:]')
+                    ENDPOINT_HOST="${ENDPOINT_HOST#*://}"
+                    ENDPOINT_HOST="${ENDPOINT_HOST%/}"
+                    case "$ENDPOINT_HOST" in
+                      *.amazonaws.com | *.amazonaws.com.cn)
+                        if ! ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text); then
+                          echo "ERROR: could not look up the AWS account these S3 keys belong to"
+                          exit 1
+                        fi
+                        set -- --expected-bucket-owner "$ACCOUNT_ID"
+                        ;;
+                    esac
+
+                    if BUCKETS=$(aws s3api list-buckets --endpoint-url "$ENDPOINT" --query 'Buckets[].Name' --output text 2>/tmp/list-buckets.err); then
+                      FOUND=false
+                      for NAME in $BUCKETS; do
+                        if [ "$NAME" = "$S3_DB_BACKUP_BUCKET" ]; then
+                          FOUND=true
+                        fi
+                      done
+                      if [ "$FOUND" = false ]; then
+                        echo "ERROR: the backup bucket $S3_DB_BACKUP_BUCKET isn't in the account these S3 keys belong to, so the database dump was NOT uploaded. Create it in your account or set S3_DB_BACKUP_BUCKET"
+                        exit 1
+                      fi
+                    elif grep -Eq 'AccessDenied|Unauthorized|\(40[13]\)' /tmp/list-buckets.err; then
+                      if [ -z "$ACCOUNT_ID" ]; then
+                        echo "Note: couldn't check that $S3_DB_BACKUP_BUCKET is owned by this account because these S3 keys can't list buckets"
+                      fi
+                    else
+                      cat /tmp/list-buckets.err
+                      echo "ERROR: S3 upload failed, could not list the buckets on $ENDPOINT"
+                      exit 1
+                    fi
+
+                    if [ -n "$ACCOUNT_ID" ] && ! aws s3api head-bucket --bucket "$S3_DB_BACKUP_BUCKET" --endpoint-url "$ENDPOINT" "$@"; then
+                      echo "ERROR: the backup bucket $S3_DB_BACKUP_BUCKET isn't owned by AWS account $ACCOUNT_ID that these S3 keys belong to, or the keys lack s3:ListBucket on it, so the database dump was NOT uploaded. Create it in your account, set S3_DB_BACKUP_BUCKET, or grant s3:ListBucket"
+                      exit 1
+                    fi
+                  fi
+
                   echo "Uploading to $ENDPOINT : $S3_DB_BACKUP_BUCKET/backup-$TS.zip"
 
                   if ! aws s3 cp "$ZIP_FILE" "s3://$S3_DB_BACKUP_BUCKET/backup-$TS.zip" --endpoint-url "$ENDPOINT"; then
@@ -68,15 +118,16 @@ spec:
                     --bucket "$S3_DB_BACKUP_BUCKET" \
                     --query 'reverse(sort_by(Contents, &LastModified))[].Key' \
                     --output text \
-                    --endpoint-url "$ENDPOINT" || true)
+                    --endpoint-url "$ENDPOINT" \
+                    "$@" || true)
 
                   DELETEABLE_KEYS=$(echo "$KEYS" | tr '\t' '\n' | awk 'NR>7')
 
                   if [ -n "$DELETEABLE_KEYS" ]; then
                     echo "Pruning old backups, keeping latest 7"
                     echo "$DELETEABLE_KEYS" | while IFS= read -r KEY; do
                       echo "Deleting old backup: $KEY"
-                      aws s3 rm "s3://$S3_DB_BACKUP_BUCKET/$KEY" --endpoint-url "$ENDPOINT" || true
+                      aws s3api delete-object --bucket "$S3_DB_BACKUP_BUCKET" --key "$KEY" --endpoint-url "$ENDPOINT" "$@" || true
                     done
                     echo "Pruning complete"
                   fi

diff --git a/tests/scripts/postgres_backup_test.sh b/tests/scripts/postgres_backup_test.sh
@@ -0,0 +1,224 @@
+#!/bin/bash
+
+# shellcheck source=tests/scripts/lib.sh
+source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
+
+AWS_ENDPOINT="s3.us-east-1.amazonaws.com"
+B2_ENDPOINT="s3.us-west-004.backblazeb2.com"
+ACCOUNT_ID="111122223333"
+
+backup_script() {
+    awk '
+        /^ *- \|$/ { in_block = 1; indent = -1; next }
+        in_block {
+            if ($0 ~ /^ *$/) { print ""; next }
+            match($0, /^ */)
+            if (indent < 0) { indent = RLENGTH }
+            if (RLENGTH < indent) { exit }
+            print substr($0, indent + 1)
+        }
+    ' "$TEST_ROOT/base/backups/postgres-backup-cronjob.yaml"
+}
+
+setup_stubs() {
+    mkdir -p "$WORK_DIR/bin"
+    cat >"$WORK_DIR/bin/apk" <<'EOF'
+#!/bin/bash
+EOF
+    cat >"$WORK_DIR/bin/pg_dump" <<'EOF'
+#!/bin/bash
+while [ $# -gt 0 ]; do
+    if [ "$1" = "-f" ]; then
+        echo dump >"$2"
+    fi
+    shift
+done
+EOF
+    cat >"$WORK_DIR/bin/zip" <<'EOF'
+#!/bin/bash
+cp "$3" "$2"
+EOF
+    cat >"$WORK_DIR/bin/aws" <<'EOF'
+#!/bin/bash
+echo "$*" >>"$WORK_DIR/aws-calls"
+expected_owner="$(printf '%s' " $* " | sed -n 's/.* --expected-bucket-owner \([^ ]*\) .*/\1/p')"
+case "$1 $2" in
+    "s3 cp" | "s3 rm")
+        if [ -n "$expected_owner" ]; then
+            printf '\nUnknown options: --expected-bucket-owner, %s\n\n' "$expected_owner" >&2
+            exit 252
+        fi
+        ;;
+    "s3api head-bucket")
+        owner="$FAKE_ACCOUNT_ID"
+        if [ -f "$WORK_DIR/bucket-owner" ]; then
+            owner="$(cat "$WORK_DIR/bucket-owner")"
+        fi
+        if [ -n "$expected_owner" ] && [ "$expected_owner" != "$owner" ]; then
+            echo "An error occurred (403) when calling the HeadBucket operation: Forbidden" >&2
+            exit 254
+        fi
+        ;;
+    "s3api list-buckets")
+        if [ -f "$WORK_DIR/list-buckets-error" ]; then
+            cat "$WORK_DIR/list-buckets-error" >&2
+            exit 254
+        fi
+        cat "$WORK_DIR/buckets"
+        ;;
+    "sts get-caller-identity") echo "$FAKE_ACCOUNT_ID" ;;
+    "s3api list-objects-v2") if [ -f "$WORK_DIR/objects" ]; then cat "$WORK_DIR/objects"; fi ;;
+esac
+EOF
+    chmod +x "$WORK_DIR/bin/apk" "$WORK_DIR/bin/pg_dump" "$WORK_DIR/bin/zip" "$WORK_DIR/bin/aws"
+}
+
+buckets() {
+    local IFS=$'\t'
+    printf '%s\n' "$*" >"$WORK_DIR/buckets"
+}
+
+deny_bucket_listing() {
+    echo "${1:-An error occurred (AccessDenied) when calling the ListBuckets operation: Access Denied}" >"$WORK_DIR/list-buckets-error"
+}
+
+eight_backups() {
+    printf 'backup-8.zip\tbackup-7.zip\tbackup-6.zip\tbackup-5.zip\tbackup-4.zip\tbackup-3.zip\tbackup-2.zip\tbackup-1.zip\n' >"$WORK_DIR/objects"
+}
+
+run_backup() {
+    setup_stubs
+    backup_script | sed "s#/tmp/#$WORK_DIR/#g" >"$WORK_DIR/backup.sh"
+    PATH="$WORK_DIR/bin:$PATH" \
+        S3_ENDPOINT="$1" \
+        S3_ACCESS_KEY=key \
+        S3_SECRET=secret \
+        POSTGRES_HOST=timescaledb \
+        POSTGRES_PORT=5432 \
+        POSTGRES_USER=hasura \
+        POSTGRES_PASSWORD=password \
+        POSTGRES_DB=hasura \
+        FAKE_ACCOUNT_ID="$ACCOUNT_ID" \
+        bash "$WORK_DIR/backup.sh" >"$WORK_DIR/output" 2>&1
+    STATUS=$?
+}
+
+aws_calls() {
+    cat "$WORK_DIR/aws-calls" 2>/dev/null || true
+}
+
+calls_to() {
+    grep "^$1" "$WORK_DIR/aws-calls" 2>/dev/null || true
+}
+
+assert_succeeded() {
+    [ "$STATUS" -eq 0 ] || fail "backup exited $STATUS: $(cat "$WORK_DIR/output")"
+}
+
+test_a_listed_bucket_on_aws_is_uploaded_to_only_as_this_account() {
+    buckets other-bucket 5stack-db-backups
+    eight_backups
+    run_backup "$AWS_ENDPOINT"
+    assert_succeeded
+    assert_eq "$(calls_to "s3api head-bucket ")" "s3api head-bucket --bucket 5stack-db-backups --endpoint-url https://$AWS_ENDPOINT --expected-bucket-owner $ACCOUNT_ID"
+    assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-"
+    assert_contains "$(calls_to "s3api list-objects-v2 ")" "--expected-bucket-owner $ACCOUNT_ID"
+    assert_eq "$(calls_to "s3api delete-object ")" "s3api delete-object --bucket 5stack-db-backups --key backup-1.zip --endpoint-url https://$AWS_ENDPOINT --expected-bucket-owner $ACCOUNT_ID"
+}
+
+test_every_spelling_of_an_aws_endpoint_checks_the_owner() {
+    for endpoint in S3.US-EAST-1.AMAZONAWS.COM "$AWS_ENDPOINT/" "https://$AWS_ENDPOINT" s3.cn-north-1.amazonaws.com.cn; do
+        rm -f "$WORK_DIR/aws-calls"
+        buckets 5stack-db-backups
+        run_backup "$endpoint"
+        assert_succeeded
+        assert_contains "$(calls_to "sts ")" "get-caller-identity"
+        assert_contains "$(calls_to "s3api head-bucket ")" "--expected-bucket-owner $ACCOUNT_ID"
+    done
+}
+
+test_an_aws_bucket_owned_by_another_account_is_never_uploaded_to() {
+    deny_bucket_listing
+    echo 444455556666 >"$WORK_DIR/bucket-owner"
+    run_backup "$AWS_ENDPOINT"
+    [ "$STATUS" -ne 0 ] || fail "backup succeeded into a bucket owned by another account"
+    assert_contains "$(cat "$WORK_DIR/output")" "the backup bucket 5stack-db-backups isn't owned by AWS account $ACCOUNT_ID that these S3 keys belong to, or the keys lack s3:ListBucket on it, so the database dump was NOT uploaded"
+    assert_eq "$(calls_to "s3 cp ")" ""
+}
+
+test_a_listed_bucket_on_b2_is_uploaded_to_without_an_aws_owner() {
+    buckets 5stack-db-backups
+    eight_backups
+    run_backup "$B2_ENDPOINT"
+    assert_succeeded
+    assert_contains "$(calls_to "s3api list-buckets ")" "--endpoint-url https://$B2_ENDPOINT --query Buckets[].Name --output text"
+    assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-"
+    assert_eq "$(calls_to "s3api delete-object ")" "s3api delete-object --bucket 5stack-db-backups --key backup-1.zip --endpoint-url https://$B2_ENDPOINT"
+    assert_not_contains "$(aws_calls)" "--expected-bucket-owner"
+    assert_not_contains "$(aws_calls)" "sts "
+    assert_not_contains "$(aws_calls)" "head-bucket"
+}
+
+test_a_bucket_outside_the_account_is_never_uploaded_to() {
+    for endpoint in "$B2_ENDPOINT" "$AWS_ENDPOINT"; do
+        rm -f "$WORK_DIR/aws-calls"
+        buckets 5stack-db-backups-old other-bucket
+        run_backup "$endpoint"
+        [ "$STATUS" -ne 0 ] || fail "backup to $endpoint succeeded with a bucket outside the account"
+        assert_contains "$(cat "$WORK_DIR/output")" "the backup bucket 5stack-db-backups isn't in the account these S3 keys belong to, so the database dump was NOT uploaded"
+        assert_eq "$(calls_to "s3 cp ")" ""
+    done
+}
+
+test_an_account_with_no_buckets_is_never_uploaded_to() {
+    buckets
+    run_backup "$B2_ENDPOINT"
+    [ "$STATUS" -ne 0 ] || fail "backup succeeded into an account with no buckets"
+    assert_eq "$(calls_to "s3 cp ")" ""
+}
+
+test_keys_that_cannot_list_buckets_still_upload_with_a_note() {
+    for denial in "An error occurred (AccessDenied) when calling the ListBuckets operation: Access Denied" \
+        "An error occurred (401) when calling the ListBuckets operation: Unauthorized"; do
+        rm -f "$WORK_DIR/aws-calls"
+        deny_bucket_listing "$denial"
+        run_backup "$B2_ENDPOINT"
+        assert_succeeded
+        assert_contains "$(cat "$WORK_DIR/output")" "because these S3 keys can't list buckets"
+        assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-"
+    done
+}
+
+test_aws_keys_that_cannot_list_buckets_still_upload_only_as_this_account() {
+    deny_bucket_listing
+    run_backup "$AWS_ENDPOINT"
+    assert_succeeded
+    assert_contains "$(calls_to "s3api head-bucket ")" "--expected-bucket-owner $ACCOUNT_ID"
+    assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-"
+    assert_not_contains "$(cat "$WORK_DIR/output")" "can't list buckets"
+}
+
+test_a_listing_that_fails_for_another_reason_stops_before_uploading() {
+    echo "Could not connect to the endpoint URL: \"https://$B2_ENDPOINT/\"" >"$WORK_DIR/list-buckets-error"
+    run_backup "$B2_ENDPOINT"
+    [ "$STATUS" -ne 0 ] || fail "backup succeeded when the bucket listing failed"
+    assert_contains "$(cat "$WORK_DIR/output")" "Could not connect to the endpoint URL"
+    assert_eq "$(calls_to "s3 cp ")" ""
+}
+
+test_the_in_cluster_store_uploads_without_listing_buckets() {
+    for endpoint in "" rustfs minio; do
+        rm -f "$WORK_DIR/aws-calls"
+        eight_backups
+        run_backup "$endpoint"
+        assert_succeeded
+        host="${endpoint:-rustfs}"
+        assert_contains "$(calls_to "s3 cp ")" "s3://5stack-db-backups/backup-"
+        assert_contains "$(calls_to "s3 cp ")" "--endpoint-url http://$host:9000"
+        assert_eq "$(calls_to "s3api delete-object ")" "s3api delete-object --bucket 5stack-db-backups --key backup-1.zip --endpoint-url http://$host:9000"
+        assert_not_contains "$(aws_calls)" "list-buckets"
+        assert_not_contains "$(aws_calls)" "--expected-bucket-owner"
+    done
+}
+
+run_tests