fix(healthcheck): cold-start readiness gate for lazily-created, unprobed checkers

apisix/healthcheck_manager.lua

@@ -273,6 +273,137 @@ function _M.fetch_node_status(checker, ip, port, hostname)
 end
 
 
+-- Cold-start readiness gate support (PS-12691): checker creation is
+-- otherwise entirely lazy -- waiting_pool is only ever seeded by
+-- fetch_checker(), which is only called from the live request path. An
+-- upstream that receives zero traffic before a readiness probe fires would
+-- never get a checker built at all, so a "has this been probed" gate would
+-- hang forever for an idle-but-critical upstream on a fresh pod. This lets a
+-- caller (the readiness plugin) force that seeding proactively.
+--
+-- Scoped to plain, non-plugin-constructed resource paths (e.g.
+-- "/upstreams/<id>", "/routes/<id>") -- unlike timer_create_checker, this
+-- does not resolve the get_plugin_name()/construct_upstream branch used by
+-- plugins like ai-proxy-multi. Critical upstreams for readiness gating are
+-- configured as standalone apisixUpstreams entries, not inline plugin
+-- config, so this scope is sufficient.
+function _M.ensure_checker(resource_path)
+    if working_pool[resource_path] then
+        -- already created, by traffic or a previous ensure_checker call
+        return true
+    end
+
+    local res_conf = resource.fetch_latest_conf(resource_path)
+    if not res_conf then
+        core.log.error("ensure_checker: resource not found: ", resource_path)
+        return false, "resource not found"
+    end
+
+    local upstream = res_conf.value.upstream or res_conf.value
+    if not upstream.checks then
+        core.log.warn("ensure_checker: resource has no checks configured: ", resource_path)
+        return false, "no checks configured"
+    end
+
+    -- Domain-name nodes are otherwise only resolved by apisix.upstream's
+    -- get_by_id -> parse_domain_in_up, which runs exclusively on the live
+    -- request path (apisix/init.lua). Without this, a checker seeded here
+    -- would start probing under the unresolved domain-string identity, then
+    -- get rebuilt -- wiping its accumulated shm state, including the probe
+    -- count is_resource_probed relies on -- the moment real traffic first
+    -- resolves the domain and bumps _nodes_ver. res_conf is the same shared
+    -- config object get_by_id operates on (both come from
+    -- core.config.fetch_created_obj), so has_domain/dns_nodes are already
+    -- populated by the config watcher's filter callback regardless of
+    -- traffic; resolve here so the checker is built once, under its final
+    -- identity, from the start.
+    if res_conf.has_domain then
+        local resolved, err = upstream_utils.parse_domain_in_up(res_conf)
+        if not resolved then
+            core.log.error("ensure_checker: failed to resolve domain nodes for ",
+                          resource_path, ": ", err)
+            -- fall through with the unresolved config -- a subsequent real
+            -- request or ensure_checker call will retry the resolution
+        else
+            res_conf = resolved
+            upstream = res_conf.value.upstream or res_conf.value
+        end
+    end
+
+    if not upstream.nodes or #upstream.nodes == 0 then
+        return false, "no nodes"
+    end
+
+    local new_version = upstream_utils.version(res_conf.modifiedIndex, upstream._nodes_ver)
+    if waiting_pool[resource_path] ~= new_version then
+        core.log.info("ensure_checker: seeding waiting pool for ", resource_path,
+                      " with version: ", new_version)
+        waiting_pool[resource_path] = new_version
+    end
+    return true
+end
+
+
+-- A single active-check attempt does not guarantee a target's real state is
+-- known: with e.g. unhealthy.http_failures = 2 configured, internal_health
+-- only converges after two consecutive failed attempts. Returns the
+-- worst-case number of consecutive attempts needed to guarantee the real
+-- state has been reached in either direction (healthy or unhealthy), or nil
+-- if the resource has no active check configured at all -- in which case
+-- there is nothing for a readiness gate to wait on.
+local function required_probe_attempts(checks)
+    local active = checks and checks.active
+    if not active then
+        return nil
+    end
+    local unhealthy = active.unhealthy or {}
+    local healthy = active.healthy or {}
+    local max_attempts = 1
+    for _, threshold in ipairs({
+        unhealthy.http_failures,
+        unhealthy.tcp_failures,
+        unhealthy.timeouts,
+        healthy.successes,
+    }) do
+        if threshold and threshold > max_attempts then
+            max_attempts = threshold
+        end
+    end
+    return max_attempts
+end
+
+
+-- Cold-start readiness gate support (PS-12691): true only once every target
+-- of the resource's checker has had enough real active-check attempts for
+-- its state to have actually converged (see resty.healthcheck's
+-- all_targets_probed -- "probed" means attempted, not "healthy", and one
+-- attempt is not always enough, see required_probe_attempts above). false
+-- if there is no live checker yet (ensure_checker not called, or
+-- timer_create_checker hasn't run its next tick yet).
+function _M.is_resource_probed(resource_path)
+    local item = working_pool[resource_path]
+    if not item or not item.checker or item.checker.dead then
+        return false
+    end
+
+    local min_attempts = required_probe_attempts(item.checks)
+    if min_attempts == nil then
+        -- no active check configured: nothing for this gate to wait on.
+        return true
+    end
+
+    if not healthcheck then
+        healthcheck = require("resty.healthcheck")
+    end
+    local ok, err = healthcheck.all_targets_probed(item.checker.name, healthcheck_shdict_name, min_attempts)
+    if ok == nil then
+        core.log.error("is_resource_probed: ", err)
+        return false
+    end
+    return ok
+end
+
+
 local function add_working_pool(resource_path, resource_ver, checker, checks)
     working_pool[resource_path] = {
         version = resource_ver,

hack/kind-repro/Dockerfile

@@ -0,0 +1,101 @@
+#
+# Local repro image: builds APISIX from this repo's source, optionally with
+# the local apache/apisix#13888 patch (hack/patches/lua-resty-healthcheck-13888.patch)
+# applied to the vendored lua-resty-healthcheck-api7 rock.
+#
+# Build stock:   docker build -f hack/kind-repro/Dockerfile -t apisix-repro:stock .
+# Build patched: docker build -f hack/kind-repro/Dockerfile --build-arg APPLY_PATCH=true -t apisix-repro:patched .
+# Build cold-start gate (reconcile + probed-gate patches, mollie-health-check
+# plugin included): docker build -f hack/kind-repro/Dockerfile
+#   --build-arg APPLY_PATCH=true --build-arg APPLY_PROBED_GATE_PATCH=true
+#   -t apisix-repro:coldstart-gate .
+#
+# Adapted from docker/debian-dev/Dockerfile (this repo's own source-build
+# Dockerfile) -- not editing that file directly since it's a tracked,
+# upstream-shared file and this is a throwaway local repro image.
+FROM debian:bullseye-slim AS build
+
+ARG APPLY_PATCH=false
+# Stacks on APPLY_PATCH -- the probed-gate patch's diff assumes the #13888
+# reconcile patch is already applied (see hack/patches/lua-resty-healthcheck-
+# probed-gate.patch's own header). Applying this without APPLY_PATCH=true
+# will fail to apply cleanly; that failure is intentional, not worth guarding.
+ARG APPLY_PROBED_GATE_PATCH=false
+# TEST-ONLY: force real apache/apisix#13888-style dropped worker_events
+# broadcasts by shrinking the resty.events per-worker queue (default 10,240,
+# not exposed as an apisix config knob) down to 2 -- a couple of rapid
+# health-state transitions is then enough to hit queue.lua's real
+# "queue overflow" push() failure, which broker.lua just logs and drops
+# (no retry/redelivery). Never for production images.
+ARG SHRINK_EVENTS_QUEUE=false
+
+ENV DEBIAN_FRONTEND=noninteractive
+ENV ENV_INST_LUADIR=/usr/local/apisix
+
+COPY . /apisix
+WORKDIR /apisix
+
+RUN set -x \
+    && apt-get -y update --fix-missing \
+    && apt-get install -y \
+        make \
+        git \
+        sudo \
+        ca-certificates \
+        curl \
+        gcc \
+        g++ \
+        cmake \
+        libyaml-dev \
+        libxml2-dev \
+        libxslt1-dev \
+        pkg-config \
+        libssl-dev \
+        zlib1g-dev \
+        patch \
+    && rm -rf deps \
+    && make deps \
+    && if [ "$APPLY_PATCH" = "true" ]; then \
+         patch -p3 -d deps/share/lua/5.1/resty < hack/patches/lua-resty-healthcheck-13888.patch; \
+       fi \
+    && if [ "$APPLY_PROBED_GATE_PATCH" = "true" ]; then \
+         patch -p3 -d deps/share/lua/5.1/resty < hack/patches/lua-resty-healthcheck-probed-gate.patch; \
+       fi \
+    && if [ "$SHRINK_EVENTS_QUEUE" = "true" ]; then \
+         sed -i 's/listening = listening,.*/&\n        max_queue_len = 2,/' apisix/events.lua && \
+         grep -n "max_queue_len" apisix/events.lua; \
+       fi \
+    && mkdir -p ${ENV_INST_LUADIR} \
+    && cp -r deps ${ENV_INST_LUADIR} \
+    && make install
+
+FROM debian:bullseye-slim
+
+RUN apt-get -y update --fix-missing \
+    && apt-get install -y libldap2-dev libyaml-0-2 libxml2 libxslt1.1 curl \
+    && apt-get remove --purge --auto-remove -y
+
+COPY --from=build /usr/local/apisix /usr/local/apisix
+COPY --from=build /usr/local/openresty /usr/local/openresty
+COPY --from=build /usr/bin/apisix /usr/bin/apisix
+
+# Mollie's readiness plugin (edge-app), staged into this repo's build context
+# at hack/kind-repro/plugin/ -- harmless if unused (only loaded if a config's
+# `plugins:` list names it).
+COPY hack/kind-repro/plugin/mollie-health-check.lua /usr/local/apisix/apisix/plugins/mollie-health-check.lua
+
+ENV PATH=$PATH:/usr/local/openresty/luajit/bin:/usr/local/openresty/nginx/sbin:/usr/local/openresty/bin
+
+WORKDIR /usr/local/apisix
+
+RUN ln -sf /dev/stdout /usr/local/apisix/logs/access.log \
+    && ln -sf /dev/stderr /usr/local/apisix/logs/error.log
+
+EXPOSE 9080 9443
+
+COPY docker/debian-dev/docker-entrypoint.sh /docker-entrypoint.sh
+RUN chmod +x /docker-entrypoint.sh
+
+ENTRYPOINT ["/docker-entrypoint.sh"]
+CMD ["docker-start"]
+STOPSIGNAL SIGQUIT

hack/kind-repro/coldstart-apisix.yaml

@@ -0,0 +1,46 @@
+routes:
+  -
+    id: pay
+    uri: /pay
+    upstream_id: online-payment-write
+upstreams:
+  -
+    id: online-payment-write
+    type: roundrobin
+    scheme: http
+    pass_host: node
+    checks:
+      active:
+        type: http
+        timeout: 2
+        http_path: /apisix-health-v2-online-payment-write
+        healthy:
+          interval: 3
+          http_statuses: [200, 404]
+          successes: 2
+        unhealthy:
+          interval: 3
+          http_statuses: [500, 502, 503, 504]
+          http_failures: 2
+          tcp_failures: 2
+          timeouts: 2
+      passive:
+        type: http
+        healthy:
+          http_statuses: [200, 201]
+          successes: PASSIVE_SUCCESSES_PLACEHOLDER
+        unhealthy:
+          http_statuses: [500, 502, 503, 504]
+          http_failures: 2
+          tcp_failures: 2
+          timeouts: 2
+    nodes:
+      - host: mock-payments-api
+        port: 80
+        weight: 100
+        priority: 0
+      - host: mock-payproc
+        port: 80
+        weight: 50
+        priority: -1
+#END

hack/kind-repro/coldstart-gate-apisix.yaml

@@ -0,0 +1,55 @@
+routes:
+  -
+    id: pay
+    uri: /pay
+    upstream_id: online-payment-write
+  -
+    id: health-check
+    uris:
+      - /health_check_internal/ready
+      - /health_check_internal/live
+    plugins:
+      mollie-health-check:
+        critical_upstreams: CRITICAL_UPSTREAMS_PLACEHOLDER
+        critical_upstreams_max_wait: 10
+upstreams:
+  -
+    id: online-payment-write
+    type: roundrobin
+    scheme: http
+    pass_host: node
+    checks:
+      active:
+        type: http
+        timeout: 2
+        http_path: /apisix-health-v2-online-payment-write
+        healthy:
+          interval: 3
+          http_statuses: [200, 404]
+          successes: 2
+        unhealthy:
+          interval: 3
+          http_statuses: [500, 502, 503, 504]
+          http_failures: 2
+          tcp_failures: 2
+          timeouts: 2
+      passive:
+        type: http
+        healthy:
+          http_statuses: [200, 201]
+          successes: 0
+        unhealthy:
+          http_statuses: [500, 502, 503, 504]
+          http_failures: 2
+          tcp_failures: 2
+          timeouts: 2
+    nodes:
+      - host: mock-payments-api
+        port: 80
+        weight: 100
+        priority: 0
+      - host: mock-payproc
+        port: 80
+        weight: 50
+        priority: -1
+#END

hack/kind-repro/coldstart-gate-config.yaml

@@ -0,0 +1,11 @@
+deployment:
+  role: data_plane
+  role_data_plane:
+    config_provider: yaml
+nginx_config:
+  worker_processes: 2
+  error_log_level: warn
+apisix:
+  enable_control: true
+plugins:
+  - mollie-health-check