From ede6e6bdb93026a23efe0c886383b35c76bdf4a7 Mon Sep 17 00:00:00 2001 From: Artem Savchenko Date: Tue, 29 Sep 2026 10:48:21 +0200 Subject: [PATCH 1/2] CI: check that nginx follows services recreated with new IPs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012bF628vAWVF7d1gqS2bQwG Signed-off-by: Artem Savchenko --- .github/workflows/compose-smoke.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/compose-smoke.yaml b/.github/workflows/compose-smoke.yaml index 09b609c..0b0f87b 100644 --- a/.github/workflows/compose-smoke.yaml +++ b/.github/workflows/compose-smoke.yaml @@ -81,6 +81,15 @@ jobs: docker compose restart tests/smoke.sh --verify + - name: Services recreated behind a running nginx + run: | + # recreated containers get new IPs; nginx must follow them without a restart + ip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$(docker compose ps -q "$1")"; } + before="account=$(ip account) transactor=$(ip transactor)" + docker compose up -d --force-recreate --no-deps account transactor + echo "IPs: $before -> account=$(ip account) transactor=$(ip transactor)" + tests/smoke.sh --verify + - name: Report failure if: failure() run: tests/ci-report.sh From 171d2e8b61e8ca4bb95d51d9c390548d9df45e8b Mon Sep 17 00:00:00 2001 From: Artem Savchenko Date: Tue, 29 Sep 2026 10:54:31 +0200 Subject: [PATCH 2/2] nginx: re-resolve service addresses at request time nginx looked up service names once at startup. After 'docker compose restart' containers can come back with different IPs, and nginx kept proxying to the old ones: 502 (or another service) until nginx itself was restarted. Use Docker's DNS (127.0.0.11, valid=10s) and put upstreams in a variable so proxy_pass resolves them per request. 'set' goes before 'rewrite ... break', which would otherwise skip it, and prefixes are stripped with '^/_x/?(.*)$ -> /$1' so /_accounts still maps to / as before. Routing of all paths and query strings is unchanged (checked old vs new side by side). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_012bF628vAWVF7d1gqS2bQwG Signed-off-by: Artem Savchenko --- .huly.nginx | 70 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/.huly.nginx b/.huly.nginx index 1fa76f9..dd32641 100644 --- a/.huly.nginx +++ b/.huly.nginx @@ -3,12 +3,19 @@ server { server_name _; client_max_body_size 100M; + # Resolve service names through Docker's DNS on every request (cached 10s) + # instead of once at startup: after `docker compose restart` containers can + # get new IPs, and a startup-time lookup would keep sending requests to the + # old ones (502). proxy_pass only re-resolves when its target is a variable. + resolver 127.0.0.11 valid=10s ipv6=off; + location / { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_pass http://front:8080; + set $upstream http://front:8080; + proxy_pass $upstream; } location /_accounts { @@ -17,8 +24,9 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - rewrite ^/_accounts(/.*)$ $1 break; - proxy_pass http://account:3000/; + set $upstream http://account:3000; + rewrite ^/_accounts/?(.*)$ /$1 break; + proxy_pass $upstream; } #location /_print { @@ -30,8 +38,9 @@ server { # proxy_http_version 1.1; # proxy_set_header Upgrade $http_upgrade; # proxy_set_header Connection "upgrade"; - # rewrite ^/_print(/.*)$ $1 break; - # proxy_pass http://print:4005/; + # set $upstream http://print:4005; + # rewrite ^/_print/?(.*)$ /$1 break; + # proxy_pass $upstream; #} #location /_aibot { @@ -43,8 +52,9 @@ server { # proxy_http_version 1.1; # proxy_set_header Upgrade $http_upgrade; # proxy_set_header Connection "upgrade"; - # rewrite ^/_aibot(/.*)$ $1 break; - # proxy_pass http://aibot:4010/; + # set $upstream http://aibot:4010; + # rewrite ^/_aibot/?(.*)$ /$1 break; + # proxy_pass $upstream; #} #location /_love { @@ -56,8 +66,9 @@ server { # proxy_http_version 1.1; # proxy_set_header Upgrade $http_upgrade; # proxy_set_header Connection "upgrade"; - # rewrite ^/_love(/.*)$ $1 break; - # proxy_pass http://love:8096/; + # set $upstream http://love:8096; + # rewrite ^/_love/?(.*)$ /$1 break; + # proxy_pass $upstream; #} location /_collaborator { @@ -69,8 +80,9 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - rewrite ^/_collaborator(/.*)$ $1 break; - proxy_pass http://collaborator:3078/; + set $upstream http://collaborator:3078; + rewrite ^/_collaborator/?(.*)$ /$1 break; + proxy_pass $upstream; } location /_transactor { @@ -82,8 +94,9 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - rewrite ^/_transactor(/.*)$ $1 break; - proxy_pass http://transactor:3333/; + set $upstream http://transactor:3333; + rewrite ^/_transactor/?(.*)$ /$1 break; + proxy_pass $upstream; } location ~ ^/eyJ { @@ -95,7 +108,8 @@ server { proxy_http_version 1.1; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection "upgrade"; - proxy_pass http://transactor:3333; + set $upstream http://transactor:3333; + proxy_pass $upstream; } location /_rekoni { @@ -104,8 +118,9 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - rewrite ^/_rekoni(/.*)$ $1 break; - proxy_pass http://rekoni:4004/; + set $upstream http://rekoni:4004; + rewrite ^/_rekoni/?(.*)$ /$1 break; + proxy_pass $upstream; } location /_stats { @@ -114,8 +129,9 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - rewrite ^/_stats(/.*)$ $1 break; - proxy_pass http://stats:4900/; + set $upstream http://stats:4900; + rewrite ^/_stats/?(.*)$ /$1 break; + proxy_pass $upstream; } #location /_pulse { @@ -127,8 +143,9 @@ server { # proxy_http_version 1.1; # proxy_set_header Upgrade $http_upgrade; # proxy_set_header Connection "upgrade"; + # set $upstream http://hulypulse:8099; # rewrite ^/_pulse(.*)$ /ws$1 break; - # proxy_pass http://hulypulse:8099/; + # proxy_pass $upstream; #} #location /_telegram { @@ -137,8 +154,9 @@ server { # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # proxy_set_header X-Forwarded-Proto $scheme; # - # rewrite ^/_telegram(/.*)$ $1 break; - # proxy_pass http://telegram-bot:4020/; + # set $upstream http://telegram-bot:4020; + # rewrite ^/_telegram/?(.*)$ /$1 break; + # proxy_pass $upstream; #} #location /_github { @@ -147,8 +165,9 @@ server { # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; # proxy_set_header X-Forwarded-Proto $scheme; # - # rewrite ^/_github(/.*)$ $1 break; - # proxy_pass http://github:3500/; + # set $upstream http://github:3500; + # rewrite ^/_github/?(.*)$ /$1 break; + # proxy_pass $upstream; #} # location /_export { @@ -160,7 +179,8 @@ server { # proxy_http_version 1.1; # proxy_set_header upgrade $http_upgrade; # proxy_set_header connection "upgrade"; - # rewrite ^/_export(/.*)$ $1 break; - # proxy_pass http://exports:4009/; + # set $upstream http://exports:4009; + # rewrite ^/_export/?(.*)$ /$1 break; + # proxy_pass $upstream; # } }