feature: HASURA_REPLICAS runs more than one Hasura

base/hasura/deployment.yaml

@@ -33,6 +33,16 @@ spec:
                 operator: In
                 values:
                   - "true"
+        # With HASURA_REPLICAS above 1, on separate nodes where there are
+        # several labelled for Hasura.
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                topologyKey: kubernetes.io/hostname
+                labelSelector:
+                  matchLabels:
+                    app: hasura
       containers:
         - image: hasura/graphql-engine:v2.51.3-ce.cli-migrations-v3@sha256:fc93e226608dcd44fa57968d74f8ed1854d7075825d1f8655fa0c9f2a6a9846d
           name: hasura

tests/scripts/write_overlays_test.sh

@@ -28,6 +28,24 @@ test_a_reverse_proxy_overlay_answers_on_443_with_its_own_certificate() {
     [ -f "$(overlay local-secrets-http)/gateway-tls.yaml" ] || fail "gateway-tls.yaml was not copied"
 }
 
+test_hasura_keeps_one_replica_by_default() {
+    write || fail "write_overlays failed: $(cat "$WORK_DIR/output")"
+    assert_not_contains "$(cat "$(overlay vault-https)/kustomization.yaml")" "replicas:"
+}
+
+test_hasura_replicas_sets_how_many_hasuras_run() {
+    HASURA_REPLICAS=2 write || fail "write_overlays failed: $(cat "$WORK_DIR/output")"
+    for name in vault-http vault-https local-secrets-http local-secrets-https; do
+        assert_contains "$(cat "$(overlay "$name")/kustomization.yaml")" "- name: hasura"
+        assert_contains "$(cat "$(overlay "$name")/kustomization.yaml")" "count: 2"
+    done
+}
+
+test_hasura_replicas_must_be_a_whole_number() {
+    HASURA_REPLICAS=two write && fail "a replica count of 'two' was accepted"
+    assert_contains "$(cat "$WORK_DIR/output")" "HASURA_REPLICAS must be a whole number"
+}
+
 test_the_certificate_is_kept_across_updates() {
     write || fail "write_overlays failed: $(cat "$WORK_DIR/output")"
     cp "$(overlay local-secrets-http)/reverse-proxy-tls.crt" "$WORK_DIR/first.crt"

utils/write_overlays.sh

@@ -23,9 +23,24 @@ ensure_reverse_proxy_certificate() {
 
 # Writes overlays/{vault,local-secrets}-{http,https}, of which update.sh applies
 # the one REVERSE_PROXY and VAULT_MANAGER select. Reads TURN_DOMAIN,
-# GPU_VENDOR and AI_ENABLED.
+# GPU_VENDOR, AI_ENABLED and HASURA_REPLICAS.
 write_overlays() {
-    local BASE PROTOCOL OVERLAY STREAMING_RESOURCES AI_RESOURCES
+    local BASE PROTOCOL OVERLAY STREAMING_RESOURCES AI_RESOURCES HASURA_REPLICAS_ENTRY
+    # How many Hasuras serve the api (HASURA_REPLICAS, default 1). In the api's
+    # load lab one is busy from about 3,000 users, and a second halved query and
+    # first-data times there. Each keeps its own Postgres pool (the api
+    # metadata's pool_settings), about 1.5 GB more Postgres memory per replica
+    # at that load. Their init containers take turns migrating (api#929).
+    HASURA_REPLICAS_ENTRY=""
+    if [ -n "$HASURA_REPLICAS" ] && [ "$HASURA_REPLICAS" != 1 ]; then
+        if ! [[ "$HASURA_REPLICAS" =~ ^[1-9][0-9]*$ ]]; then
+            echo "HASURA_REPLICAS must be a whole number of replicas, not '$HASURA_REPLICAS'" >&2
+            return 1
+        fi
+        HASURA_REPLICAS_ENTRY="replicas:
+- name: hasura
+  count: $HASURA_REPLICAS"
+    fi
     for BASE in vault local-secrets; do
         for PROTOCOL in "http" "https"; do
             OVERLAY="$PANEL_DIR/overlays/${BASE}-${PROTOCOL}"
@@ -65,6 +80,7 @@ resources:
 $STREAMING_RESOURCES
 $AI_RESOURCES
 $(if [[ "$PROTOCOL" == "https" ]]; then echo "- ../cert-manager"; echo "- https-redirect.yaml"; else echo "- gateway-tls.yaml"; fi)
+$HASURA_REPLICAS_ENTRY
 EOF
             if [ "$PROTOCOL" = "https" ]; then
                 cp "$PANEL_DIR/overlays/http/gateway-https-patch.yaml" "$PANEL_DIR/overlays/http/https-redirect.yaml" "$OVERLAY/"