fix: a panel on external S3 leaves the built-in RustFS out of its overlays

.github/workflows/panel-scripts.yml

@@ -18,6 +18,9 @@ on:
       - base/hasura/deployment.yaml
       - "tests/scripts/**"
       - base/kustomization.yaml
+      - base/api/httproute.yaml
+      - "base/rustfs/**"
+      - "overlays/external-s3/**"
       - "base/system-upgrade/**"
       - "base/backups/**"
       - .github/workflows/panel-scripts.yml
@@ -37,6 +40,9 @@ on:
       - base/hasura/deployment.yaml
       - "tests/scripts/**"
       - base/kustomization.yaml
+      - base/api/httproute.yaml
+      - "base/rustfs/**"
+      - "overlays/external-s3/**"
       - "base/system-upgrade/**"
       - "base/backups/**"
       - .github/workflows/panel-scripts.yml

overlays/external-s3/kustomization.yaml

@@ -0,0 +1,85 @@
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+patches:
+  - target:
+      kind: StatefulSet
+      name: rustfs
+    patch: |-
+      $patch: delete
+      apiVersion: apps/v1
+      kind: StatefulSet
+      metadata:
+        name: rustfs
+  - target:
+      kind: Service
+      name: rustfs
+    patch: |-
+      $patch: delete
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: rustfs
+  - target:
+      kind: Service
+      name: minio
+    patch: |-
+      $patch: delete
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: minio
+  - target:
+      kind: HTTPRoute
+      name: s3-console
+    patch: |-
+      $patch: delete
+      apiVersion: gateway.networking.k8s.io/v1
+      kind: HTTPRoute
+      metadata:
+        name: s3-console
+  - target:
+      kind: PersistentVolume
+      name: minio-pv
+    patch: |-
+      $patch: delete
+      apiVersion: v1
+      kind: PersistentVolume
+      metadata:
+        name: minio-pv
+  - target:
+      kind: PersistentVolumeClaim
+      name: minio-pvc
+    patch: |-
+      $patch: delete
+      apiVersion: v1
+      kind: PersistentVolumeClaim
+      metadata:
+        name: minio-pvc
+  - target:
+      kind: Secret
+      name: rustfs-secrets
+    patch: |-
+      $patch: delete
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: rustfs-secrets
+  - target:
+      kind: ExternalSecret
+      name: rustfs-secrets
+    patch: |-
+      $patch: delete
+      apiVersion: external-secrets.io/v1
+      kind: ExternalSecret
+      metadata:
+        name: rustfs-secrets
+  - target:
+      kind: HTTPRoute
+      name: demos
+    patch: |-
+      - op: test
+        path: /spec/rules/1/backendRefs/0/name
+        value: rustfs
+      - op: remove
+        path: /spec/rules/1

tests/scripts/write_overlays_test.sh

@@ -3,21 +3,96 @@
 # shellcheck source=tests/scripts/lib.sh
 source "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 
-write() {
+in_panel() {
     (
         export PANEL_DIR="$WORK_DIR/panel"
         mkdir -p "$PANEL_DIR/overlays"
-        cp -R "$TEST_ROOT/overlays/http" "$PANEL_DIR/overlays/http"
+        [ -d "$PANEL_DIR/overlays/http" ] || cp -R "$TEST_ROOT/overlays/http" "$PANEL_DIR/overlays/http"
         # shellcheck source=utils/write_overlays.sh
         source "$TEST_ROOT/utils/write_overlays.sh"
-        write_overlays
+        "$@"
     ) >"$WORK_DIR/output" 2>&1
 }
 
+write() {
+    in_panel write_overlays
+}
+
 overlay() {
     printf '%s' "$WORK_DIR/panel/overlays/$1"
 }
 
+with_s3_endpoint() {
+    mkdir -p "$WORK_DIR/panel"
+    cp -R "$TEST_ROOT/base" "$TEST_ROOT/overlays" "$WORK_DIR/panel/"
+    find "$WORK_DIR/panel/overlays" -name '*.env.example' | while read -r example; do
+        cp "$example" "${example%.example}"
+    done
+    grep -v '^S3_ENDPOINT=' "$TEST_ROOT/overlays/config/s3-config.env.example" >"$(overlay config)/s3-config.env"
+    printf 'S3_ENDPOINT=%s\n' "$1" >>"$(overlay config)/s3-config.env"
+}
+
+builtin() {
+    in_panel s3_storage_is_builtin
+}
+
+kinds_and_names() {
+    awk '/^---/ { kind = ""; next } /^kind: / { kind = $2 } /^  name: / && kind != "" { print kind "/" $2; kind = "" }' <<<"$1"
+}
+
+test_rustfs_is_built_in_storage_by_any_of_its_names() {
+    for endpoint in "" rustfs minio RustFS "http://rustfs:9000" "https://minio:9000/" '"rustfs"' "rustfs:9000"; do
+        with_s3_endpoint "$endpoint"
+        builtin || fail "'$endpoint' was not taken for the built-in RustFS"
+    done
+}
+
+test_without_an_s3_config_storage_is_built_in() {
+    builtin || fail "a panel with no s3-config.env was not taken for the built-in RustFS"
+}
+
+test_any_other_s3_endpoint_is_external() {
+    for endpoint in s3.us-east-005.backblazeb2.com "https://s3.us-east-005.backblazeb2.com" minio.example.com "rustfs.example.com:9000" s3.amazonaws.com; do
+        with_s3_endpoint "$endpoint"
+        builtin && fail "'$endpoint' was taken for the built-in RustFS"
+    done
+    true
+}
+
+test_built_in_storage_keeps_rustfs_in_every_overlay() {
+    with_s3_endpoint ""
+    write || fail "write_overlays failed: $(cat "$WORK_DIR/output")"
+    for o in vault-http vault-https local-secrets-http local-secrets-https; do
+        assert_not_contains "$(cat "$(overlay "$o")/kustomization.yaml")" "external-s3"
+    done
+    manifest="$(kubectl kustomize "$(overlay local-secrets-https)")" || fail "kubectl kustomize failed"
+    rendered="$(kinds_and_names "$manifest")"
+    for object in StatefulSet/rustfs Service/rustfs Service/minio HTTPRoute/s3-console PersistentVolumeClaim/minio-pvc; do
+        assert_contains "$rendered" "$object"
+    done
+}
+
+test_an_external_s3_endpoint_leaves_rustfs_out_of_every_overlay() {
+    with_s3_endpoint s3.us-east-005.backblazeb2.com
+    write || fail "write_overlays failed: $(cat "$WORK_DIR/output")"
+    for o in vault-http vault-https local-secrets-http local-secrets-https; do
+        manifest="$(kubectl kustomize "$(overlay "$o")")" || fail "kubectl kustomize $o failed"
+        rendered="$(kinds_and_names "$manifest")"
+        for object in StatefulSet/rustfs Service/rustfs Service/minio HTTPRoute/s3-console PersistentVolume/minio-pv PersistentVolumeClaim/minio-pvc Secret/rustfs-secrets ExternalSecret/rustfs-secrets; do
+            assert_not_contains "$rendered" "$object"
+        done
+        assert_not_contains "$manifest" "name: rustfs"
+        assert_contains "$rendered" "HTTPRoute/demos"
+        assert_contains "$rendered" "Deployment/api"
+        assert_contains "$manifest" "value: /demos"
+        assert_contains "$(cat "$(overlay "$o")/kustomization.yaml")" "- ../external-s3"
+    done
+}
+
+test_update_sh_hands_a_minio_volume_to_rustfs_only_on_built_in_storage() {
+    assert_contains "$(grep 'get statefulset minio' "$TEST_ROOT/update.sh")" "if s3_storage_is_builtin && "
+}
+
 test_a_reverse_proxy_overlay_answers_on_443_with_its_own_certificate() {
     write || fail "write_overlays failed: $(cat "$WORK_DIR/output")"
     openssl x509 -noout -in "$(overlay local-secrets-http)/reverse-proxy-tls.crt" || fail "no certificate"

update.sh

@@ -89,7 +89,7 @@ fi
 # node, and Kubernetes happily lets two pods on one node mount the same RWO
 # claim. Two object stores writing one directory corrupts it, so the old
 # StatefulSet has to be gone before the apply below creates the new one.
-if kubectl --kubeconfig=$KUBECONFIG get statefulset minio -n 5stack >/dev/null 2>&1; then
+if s3_storage_is_builtin && kubectl --kubeconfig=$KUBECONFIG get statefulset minio -n 5stack >/dev/null 2>&1; then
     step "Migrating MinIO to RustFS"
     kubectl --kubeconfig=$KUBECONFIG delete statefulset minio -n 5stack >/dev/null 2>&1
     kubectl --kubeconfig=$KUBECONFIG wait --for=delete pod -l app=minio -n 5stack --timeout=120s >/dev/null 2>&1

utils/write_overlays.sh

@@ -21,11 +21,26 @@ ensure_reverse_proxy_certificate() {
     }
 }
 
+# Empty, rustfs or minio (base/rustfs/service.yaml's alias), with or without a
+# scheme or port, is the panel's own RustFS.
+s3_storage_is_builtin() {
+    local endpoint
+    endpoint="$(grep -s '^S3_ENDPOINT=' "$PANEL_DIR/overlays/config/s3-config.env" | tail -1 | cut -d= -f2- | tr -d "\"' \t\r" | tr '[:upper:]' '[:lower:]')"
+    endpoint="${endpoint#*://}"
+    endpoint="${endpoint%%/*}"
+    endpoint="${endpoint%%:*}"
+    case "$endpoint" in
+        "" | rustfs | minio) return 0 ;;
+    esac
+    return 1
+}
+
 # Writes overlays/{vault,local-secrets}-{http,https}, of which update.sh applies
 # the one REVERSE_PROXY and VAULT_MANAGER select. Reads TURN_DOMAIN,
-# GPU_VENDOR, AI_ENABLED and HASURA_REPLICAS.
+# GPU_VENDOR, AI_ENABLED and HASURA_REPLICAS, and S3_ENDPOINT from
+# overlays/config/s3-config.env.
 write_overlays() {
-    local BASE PROTOCOL OVERLAY STREAMING_RESOURCES AI_RESOURCES HASURA_REPLICAS_ENTRY
+    local BASE PROTOCOL OVERLAY STREAMING_RESOURCES AI_RESOURCES HASURA_REPLICAS_ENTRY STORAGE_COMPONENTS
     # How many Hasuras serve the api (HASURA_REPLICAS, default 1). In the api's
     # load lab one is busy from about 3,000 users, and a second halved query and
     # first-data times there. Each keeps its own Postgres pool (the api
@@ -40,6 +55,13 @@ write_overlays() {
         HASURA_REPLICAS_ENTRY="replicas:
 - name: hasura
   count: $HASURA_REPLICAS"
+    fi
+    # Left out of the render only: a RustFS or MinIO already running, and its
+    # volume, stay as they are.
+    STORAGE_COMPONENTS=""
+    if ! s3_storage_is_builtin; then
+        STORAGE_COMPONENTS="components:
+- ../external-s3"
     fi
     for BASE in vault local-secrets; do
         for PROTOCOL in "http" "https"; do
@@ -81,6 +103,7 @@ $STREAMING_RESOURCES
 $AI_RESOURCES
 $(if [[ "$PROTOCOL" == "https" ]]; then echo "- ../cert-manager"; echo "- https-redirect.yaml"; else echo "- gateway-tls.yaml"; fi)
 $HASURA_REPLICAS_ENTRY
+$STORAGE_COMPONENTS
 EOF
             if [ "$PROTOCOL" = "https" ]; then
                 cp "$PANEL_DIR/overlays/http/gateway-https-patch.yaml" "$PANEL_DIR/overlays/http/https-redirect.yaml" "$OVERLAY/"