diff --git a/.github/workflows/panel-scripts.yml b/.github/workflows/panel-scripts.yml index f6b1580..77efab0 100644 --- a/.github/workflows/panel-scripts.yml +++ b/.github/workflows/panel-scripts.yml @@ -18,6 +18,9 @@ on: - base/hasura/deployment.yaml - "tests/scripts/**" - base/kustomization.yaml + - base/api/httproute.yaml + - "base/rustfs/**" + - "overlays/external-s3/**" - "base/system-upgrade/**" - "base/backups/**" - .github/workflows/panel-scripts.yml @@ -37,6 +40,9 @@ on: - base/hasura/deployment.yaml - "tests/scripts/**" - base/kustomization.yaml + - base/api/httproute.yaml + - "base/rustfs/**" + - "overlays/external-s3/**" - "base/system-upgrade/**" - "base/backups/**" - .github/workflows/panel-scripts.yml diff --git a/overlays/external-s3/kustomization.yaml b/overlays/external-s3/kustomization.yaml new file mode 100644 index 0000000..ea4305f --- /dev/null +++ b/overlays/external-s3/kustomization.yaml @@ -0,0 +1,85 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component + +patches: + - target: + kind: StatefulSet + name: rustfs + patch: |- + $patch: delete + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: rustfs + - target: + kind: Service + name: rustfs + patch: |- + $patch: delete + apiVersion: v1 + kind: Service + metadata: + name: rustfs + - target: + kind: Service + name: minio + patch: |- + $patch: delete + apiVersion: v1 + kind: Service + metadata: + name: minio + - target: + kind: HTTPRoute + name: s3-console + patch: |- + $patch: delete + apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: s3-console + - target: + kind: PersistentVolume + name: minio-pv + patch: |- + $patch: delete + apiVersion: v1 + kind: PersistentVolume + metadata: + name: minio-pv + - target: + kind: PersistentVolumeClaim + name: minio-pvc + patch: |- + $patch: delete + apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: minio-pvc + - target: + kind: Secret + name: rustfs-secrets + patch: |- + $patch: delete + apiVersion: v1 + kind: Secret + metadata: + name: rustfs-secrets + - target: + kind: ExternalSecret + name: rustfs-secrets + patch: |- + $patch: delete + apiVersion: external-secrets.io/v1 + kind: ExternalSecret + metadata: + name: rustfs-secrets + - target: + kind: HTTPRoute + name: demos + patch: |- + - op: test + path: /spec/rules/1/backendRefs/0/name + value: rustfs + - op: remove + path: /spec/rules/1 diff --git a/tests/scripts/write_overlays_test.sh b/tests/scripts/write_overlays_test.sh index bea0037..dd3589f 100644 --- a/tests/scripts/write_overlays_test.sh +++ b/tests/scripts/write_overlays_test.sh @@ -3,21 +3,96 @@ # shellcheck source=tests/scripts/lib.sh source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -write() { +in_panel() { ( export PANEL_DIR="$WORK_DIR/panel" mkdir -p "$PANEL_DIR/overlays" - cp -R "$TEST_ROOT/overlays/http" "$PANEL_DIR/overlays/http" + [ -d "$PANEL_DIR/overlays/http" ] || cp -R "$TEST_ROOT/overlays/http" "$PANEL_DIR/overlays/http" # shellcheck source=utils/write_overlays.sh source "$TEST_ROOT/utils/write_overlays.sh" - write_overlays + "$@" ) >"$WORK_DIR/output" 2>&1 } +write() { + in_panel write_overlays +} + overlay() { printf '%s' "$WORK_DIR/panel/overlays/$1" } +with_s3_endpoint() { + mkdir -p "$WORK_DIR/panel" + cp -R "$TEST_ROOT/base" "$TEST_ROOT/overlays" "$WORK_DIR/panel/" + find "$WORK_DIR/panel/overlays" -name '*.env.example' | while read -r example; do + cp "$example" "${example%.example}" + done + grep -v '^S3_ENDPOINT=' "$TEST_ROOT/overlays/config/s3-config.env.example" >"$(overlay config)/s3-config.env" + printf 'S3_ENDPOINT=%s\n' "$1" >>"$(overlay config)/s3-config.env" +} + +builtin() { + in_panel s3_storage_is_builtin +} + +kinds_and_names() { + awk '/^---/ { kind = ""; next } /^kind: / { kind = $2 } /^ name: / && kind != "" { print kind "/" $2; kind = "" }' <<<"$1" +} + +test_rustfs_is_built_in_storage_by_any_of_its_names() { + for endpoint in "" rustfs minio RustFS "http://rustfs:9000" "https://minio:9000/" '"rustfs"' "rustfs:9000"; do + with_s3_endpoint "$endpoint" + builtin || fail "'$endpoint' was not taken for the built-in RustFS" + done +} + +test_without_an_s3_config_storage_is_built_in() { + builtin || fail "a panel with no s3-config.env was not taken for the built-in RustFS" +} + +test_any_other_s3_endpoint_is_external() { + for endpoint in s3.us-east-005.backblazeb2.com "https://s3.us-east-005.backblazeb2.com" minio.example.com "rustfs.example.com:9000" s3.amazonaws.com; do + with_s3_endpoint "$endpoint" + builtin && fail "'$endpoint' was taken for the built-in RustFS" + done + true +} + +test_built_in_storage_keeps_rustfs_in_every_overlay() { + with_s3_endpoint "" + write || fail "write_overlays failed: $(cat "$WORK_DIR/output")" + for o in vault-http vault-https local-secrets-http local-secrets-https; do + assert_not_contains "$(cat "$(overlay "$o")/kustomization.yaml")" "external-s3" + done + manifest="$(kubectl kustomize "$(overlay local-secrets-https)")" || fail "kubectl kustomize failed" + rendered="$(kinds_and_names "$manifest")" + for object in StatefulSet/rustfs Service/rustfs Service/minio HTTPRoute/s3-console PersistentVolumeClaim/minio-pvc; do + assert_contains "$rendered" "$object" + done +} + +test_an_external_s3_endpoint_leaves_rustfs_out_of_every_overlay() { + with_s3_endpoint s3.us-east-005.backblazeb2.com + write || fail "write_overlays failed: $(cat "$WORK_DIR/output")" + for o in vault-http vault-https local-secrets-http local-secrets-https; do + manifest="$(kubectl kustomize "$(overlay "$o")")" || fail "kubectl kustomize $o failed" + rendered="$(kinds_and_names "$manifest")" + for object in StatefulSet/rustfs Service/rustfs Service/minio HTTPRoute/s3-console PersistentVolume/minio-pv PersistentVolumeClaim/minio-pvc Secret/rustfs-secrets ExternalSecret/rustfs-secrets; do + assert_not_contains "$rendered" "$object" + done + assert_not_contains "$manifest" "name: rustfs" + assert_contains "$rendered" "HTTPRoute/demos" + assert_contains "$rendered" "Deployment/api" + assert_contains "$manifest" "value: /demos" + assert_contains "$(cat "$(overlay "$o")/kustomization.yaml")" "- ../external-s3" + done +} + +test_update_sh_hands_a_minio_volume_to_rustfs_only_on_built_in_storage() { + assert_contains "$(grep 'get statefulset minio' "$TEST_ROOT/update.sh")" "if s3_storage_is_builtin && " +} + test_a_reverse_proxy_overlay_answers_on_443_with_its_own_certificate() { write || fail "write_overlays failed: $(cat "$WORK_DIR/output")" openssl x509 -noout -in "$(overlay local-secrets-http)/reverse-proxy-tls.crt" || fail "no certificate" diff --git a/update.sh b/update.sh index fdea13a..9e71aad 100755 --- a/update.sh +++ b/update.sh @@ -89,7 +89,7 @@ fi # node, and Kubernetes happily lets two pods on one node mount the same RWO # claim. Two object stores writing one directory corrupts it, so the old # StatefulSet has to be gone before the apply below creates the new one. -if kubectl --kubeconfig=$KUBECONFIG get statefulset minio -n 5stack >/dev/null 2>&1; then +if s3_storage_is_builtin && kubectl --kubeconfig=$KUBECONFIG get statefulset minio -n 5stack >/dev/null 2>&1; then step "Migrating MinIO to RustFS" kubectl --kubeconfig=$KUBECONFIG delete statefulset minio -n 5stack >/dev/null 2>&1 kubectl --kubeconfig=$KUBECONFIG wait --for=delete pod -l app=minio -n 5stack --timeout=120s >/dev/null 2>&1 diff --git a/utils/write_overlays.sh b/utils/write_overlays.sh index 8632daf..d282f4e 100644 --- a/utils/write_overlays.sh +++ b/utils/write_overlays.sh @@ -21,11 +21,26 @@ ensure_reverse_proxy_certificate() { } } +# Empty, rustfs or minio (base/rustfs/service.yaml's alias), with or without a +# scheme or port, is the panel's own RustFS. +s3_storage_is_builtin() { + local endpoint + endpoint="$(grep -s '^S3_ENDPOINT=' "$PANEL_DIR/overlays/config/s3-config.env" | tail -1 | cut -d= -f2- | tr -d "\"' \t\r" | tr '[:upper:]' '[:lower:]')" + endpoint="${endpoint#*://}" + endpoint="${endpoint%%/*}" + endpoint="${endpoint%%:*}" + case "$endpoint" in + "" | rustfs | minio) return 0 ;; + esac + return 1 +} + # Writes overlays/{vault,local-secrets}-{http,https}, of which update.sh applies # the one REVERSE_PROXY and VAULT_MANAGER select. Reads TURN_DOMAIN, -# GPU_VENDOR, AI_ENABLED and HASURA_REPLICAS. +# GPU_VENDOR, AI_ENABLED and HASURA_REPLICAS, and S3_ENDPOINT from +# overlays/config/s3-config.env. write_overlays() { - local BASE PROTOCOL OVERLAY STREAMING_RESOURCES AI_RESOURCES HASURA_REPLICAS_ENTRY + local BASE PROTOCOL OVERLAY STREAMING_RESOURCES AI_RESOURCES HASURA_REPLICAS_ENTRY STORAGE_COMPONENTS # How many Hasuras serve the api (HASURA_REPLICAS, default 1). In the api's # load lab one is busy from about 3,000 users, and a second halved query and # first-data times there. Each keeps its own Postgres pool (the api @@ -40,6 +55,13 @@ write_overlays() { HASURA_REPLICAS_ENTRY="replicas: - name: hasura count: $HASURA_REPLICAS" + fi + # Left out of the render only: a RustFS or MinIO already running, and its + # volume, stay as they are. + STORAGE_COMPONENTS="" + if ! s3_storage_is_builtin; then + STORAGE_COMPONENTS="components: +- ../external-s3" fi for BASE in vault local-secrets; do for PROTOCOL in "http" "https"; do @@ -81,6 +103,7 @@ $STREAMING_RESOURCES $AI_RESOURCES $(if [[ "$PROTOCOL" == "https" ]]; then echo "- ../cert-manager"; echo "- https-redirect.yaml"; else echo "- gateway-tls.yaml"; fi) $HASURA_REPLICAS_ENTRY +$STORAGE_COMPONENTS EOF if [ "$PROTOCOL" = "https" ]; then cp "$PANEL_DIR/overlays/http/gateway-https-patch.yaml" "$PANEL_DIR/overlays/http/https-redirect.yaml" "$OVERLAY/"