diff --git a/apisix/control/v1.lua b/apisix/control/v1.lua index b224b6838d4b..c73c8db5de47 100644 --- a/apisix/control/v1.lua +++ b/apisix/control/v1.lua @@ -26,6 +26,7 @@ local healthcheck_manager = require("apisix.healthcheck_manager") local get_upstreams = upstream_mod.upstreams local collectgarbage = collectgarbage local ipairs = ipairs +local pairs = pairs local pcall = pcall local setmetatable = setmetatable local str_format = string.format @@ -77,12 +78,14 @@ function _M.schema() end local healthcheck -local function extra_checker_info(value) +-- `value` is anything get_healthchecker_name() accepts: a resource config, or a +-- bare {resource_key = ...} for a checker a plugin owns. +local function get_checker_nodes(value) if not healthcheck then healthcheck = require("resty.healthcheck") end - local name = healthcheck_manager.get_healthchecker_name(value.value) + local name = healthcheck_manager.get_healthchecker_name(value) local nodes, err = healthcheck.get_target_list(name, "upstream-healthcheck") if err then core.log.error("healthcheck.get_target_list failed: ", err) @@ -92,9 +95,14 @@ local function extra_checker_info(value) -- so keep the field a JSON array to report `[]` instead of `{}` then setmetatable(nodes, core.json.array_mt) end + return nodes +end + + +local function extra_checker_info(value) return { name = value.key, - nodes = nodes, + nodes = get_checker_nodes(value.value), } end @@ -108,6 +116,41 @@ local function get_checker_type(checks) end +-- A plugin can run active health checks of its own on nodes that belong to no +-- upstream -- ai-proxy-multi probes every LLM instance and skips the unhealthy +-- ones when it picks a target. Those checkers are keyed by the resource key plus +-- a JSON path, a layout only the plugin knows, so ask the plugin for them +-- instead of guessing. What a checker stands for is the plugin's business too: +-- it names itself in `meta`, reported verbatim. +local function add_plugin_healthcheck_info(infos, value) + local plugins = value.value.plugins + if not plugins then + return + end + + for name, plugin_conf in pairs(plugins) do + -- a disabled plugin never runs, so its checkers never exist; and its + -- config is kept even when check_schema() rejects it, so it must not be + -- handed to the plugin either + if not plugin.check_disable(plugin_conf) then + local plugin_obj = plugin.get(name) + if plugin_obj and plugin_obj.list_healthcheck_targets then + local targets = plugin_obj.list_healthcheck_targets(plugin_conf, value.key) + for _, target in ipairs(targets or {}) do + core.table.insert(infos, { + name = target.resource_path, + plugin = name, + meta = target.meta, + type = get_checker_type(target.checks), + nodes = get_checker_nodes({resource_key = target.resource_path}), + }) + end + end + end + end +end + + local function iter_and_add_healthcheck_info(infos, values) if not values then return @@ -120,6 +163,7 @@ local function iter_and_add_healthcheck_info(infos, values) info.type = get_checker_type(checks) core.table.insert(infos, info) end + add_plugin_healthcheck_info(infos, value) end end @@ -241,15 +285,50 @@ local function iter_and_find_healthcheck_info(values, src_type, src_id) end +-- Every checker a resource owns, in the same entry shape the /v1/healthcheck +-- listing uses. A resource can own more than one -- its upstream plus one per +-- plugin instance -- which the single object returned by +-- /v1/healthcheck/{src_type}/{src_id} cannot express, so this is a sub-resource +-- of its own rather than a new field on that object. A resource with no health +-- check at all is not an error here: it owns an empty set of checkers. +local function iter_and_find_resource_checkers(values, src_type, src_id) + if not values then + return nil, str_format("%s[%s] not found", src_type, src_id) + end + + for _, value in core.config_util.iterate_values(values) do + if value.value.id == src_id then + local infos = core.table.new(1, 0) + local checks = value.value.checks or + (value.value.upstream and value.value.upstream.checks) + if checks then + local info = extra_checker_info(value) + info.type = get_checker_type(checks) + core.table.insert(infos, info) + end + add_plugin_healthcheck_info(infos, value) + -- an empty result must still serialize as [], not {} + return setmetatable(infos, core.json.array_mt) + end + end + + return nil, str_format("%s[%s] not found", src_type, src_id) +end + + function _M.get_health_checker() local uri_segs = core.utils.split_uri(ngx_var.uri) core.log.info("healthcheck uri: ", core.json.delay_encode(uri_segs)) - local src_type, src_id = uri_segs[4], uri_segs[5] + local src_type, src_id, sub_res = uri_segs[4], uri_segs[5], uri_segs[6] if not src_id then return 404, {error_msg = str_format("missing src id for src type %s", src_type)} end + if sub_res and (sub_res ~= "checkers" or uri_segs[7]) then + return 400, {error_msg = str_format("invalid sub resource %s", sub_res)} + end + local values if src_type == "routes" then values = get_routes() @@ -263,6 +342,22 @@ function _M.get_health_checker() return 400, {error_msg = str_format("invalid src type %s", src_type)} end + if sub_res then + local infos, err = iter_and_find_resource_checkers(values, src_type, src_id) + if not infos then + return 404, {error_msg = err} + end + local out, err = try_render_html({stats = infos}) + if out then + core.response.set_header("Content-Type", "text/html") + return 200, out + end + if err then + return 503, {error_msg = err} + end + return 200, infos + end + local info, err = iter_and_find_healthcheck_info(values, src_type, src_id) if not info then return 404, {error_msg = err} diff --git a/apisix/plugin.lua b/apisix/plugin.lua index 6f988fd7d9b1..5a7a405a09ca 100644 --- a/apisix/plugin.lua +++ b/apisix/plugin.lua @@ -162,6 +162,9 @@ local function check_disable(plugin_conf) return plugin_conf._meta.disable end +-- exposed for callers that must not act on a plugin config which never runs, +-- such as the control API reporting the health checkers a plugin owns +_M.check_disable = check_disable local PLUGIN_TYPE_HTTP = 1 local PLUGIN_TYPE_STREAM = 2 diff --git a/apisix/plugins/ai-proxy-multi.lua b/apisix/plugins/ai-proxy-multi.lua index 6ce28e7932a2..19aba41a3fda 100644 --- a/apisix/plugins/ai-proxy-multi.lua +++ b/apisix/plugins/ai-proxy-multi.lua @@ -597,6 +597,16 @@ local function create_server_picker(conf, ups_tab, checkers) end +-- Identity of the health checker of the i-th instance: the parent resource key +-- plus the JSON path of the instance. The health check manager parses it back +-- to reach construct_upstream(), so both ends must agree on this exact layout. +local function instance_resource_path(resource_key, index) + -- json path is 0 indexed so we need to decrement the index + return resource_key .. "#plugins['" .. plugin_name .. "'].instances[" + .. index - 1 .. "]" +end + + local function get_instance_conf(instances, name) for _, ins in ipairs(instances) do if ins.name == name then @@ -616,9 +626,8 @@ local function pick_target(ctx, conf, ups_tab) for i, instance in ipairs(conf.instances) do if instance.checks then resolve_endpoint(instance) - -- json path is 0 indexed so we need to decrement i - local resource_path = conf._meta.parent.resource_key .. - "#plugins['ai-proxy-multi'].instances[" .. i-1 .. "]" + local resource_path = instance_resource_path( + conf._meta.parent.resource_key, i) local resource_version = conf._meta.parent.resource_version if instance._nodes_ver then resource_version = resource_version .. instance._nodes_ver @@ -982,6 +991,28 @@ local function retry_on_error(ctx, conf, code, body) return code end +-- Each instance can run its own active health check, whose checker is keyed by +-- instance_resource_path(). Nothing outside the plugin can guess that layout, so +-- expose it for the control API, which reports the health status of every +-- configured checker (control/v1.lua). `meta` describes the checker to whoever +-- reports it -- what an instance is is the plugin's business, so the control API +-- passes it through instead of knowing about it. +function _M.list_healthcheck_targets(conf, resource_key) + local targets = {} + for i, instance in ipairs(conf.instances or {}) do + if instance.checks then + core.table.insert(targets, { + resource_path = instance_resource_path(resource_key, i), + checks = instance.checks, + meta = {instance = instance.name}, + }) + end + end + + return targets +end + + function _M.construct_upstream(instance) if not instance then return nil, "instance configuration is nil" diff --git a/docs/en/latest/control-api.md b/docs/en/latest/control-api.md index 6f594c402a49..a945b2db93d0 100644 --- a/docs/en/latest/control-api.md +++ b/docs/en/latest/control-api.md @@ -145,6 +145,38 @@ Each of the returned objects contain the following fields: * nodes[i].counter.tcp_failure: tcp connect/read/write failures count. * nodes[i].counter.timeout_failure: timeout count. +A plugin can run active health checks of its own, on nodes that belong to no +upstream. [ai-proxy-multi](./plugins/ai-proxy-multi.md) does this: it probes +every LLM instance and skips the unhealthy ones when it picks a target. Such a +checker is reported with two extra fields: + +```json +{ + "name": "/apisix/routes/1#plugins['ai-proxy-multi'].instances[0]", + "plugin": "ai-proxy-multi", + "meta": { + "instance": "openai" + }, + "type": "http", + "nodes": [ + { + "ip": "52.86.68.46", + "port": 443, + "status": "healthy", + "counter": { + "http_failure": 0, + "success": 2, + "timeout_failure": 0, + "tcp_failure": 0 + } + } + ] +} +``` + +* plugin: name of the plugin owning the health checker. Absent for an upstream health checker. +* meta: filled by the plugin to describe what the checker stands for. Its content is plugin specific; `ai-proxy-multi` reports the instance name. + You can also use `/v1/healthcheck/$src_type/$src_id` to get the health status of specific nodes. For example, `GET /v1/healthcheck/upstreams/1` returns: @@ -193,6 +225,38 @@ If you use browser to access the control API URL, then you will get the HTML out ![Health Check Status Page](https://raw.githubusercontent.com/apache/apisix/master/docs/assets/images/health_check_status_page.png) +### GET /v1/healthcheck/{src_type}/{src_id}/checkers + +Returns every health checker one resource owns, as an array of the entries +described above. A resource can own more than one -- its upstream plus one per +plugin instance -- which `GET /v1/healthcheck/$src_type/$src_id` cannot express, +since it returns a single object. + +For example, `GET /v1/healthcheck/routes/1/checkers` returns: + +```json +[ + { + "name": "/apisix/routes/1", + "type": "http", + "nodes": [...] + }, + { + "name": "/apisix/routes/1#plugins['ai-proxy-multi'].instances[0]", + "plugin": "ai-proxy-multi", + "meta": { + "instance": "openai" + }, + "type": "http", + "nodes": [...] + } +] +``` + +A resource with no health check at all is not an error here: it owns an empty +set of checkers, and the endpoint returns `[]`. `404` is returned only when the +resource itself does not exist. + ### POST /v1/gc Introduced in [v2.8](https://github.com/apache/apisix/releases/tag/2.8). diff --git a/docs/en/latest/plugins/ai-proxy-multi.md b/docs/en/latest/plugins/ai-proxy-multi.md index 4f0dc647ab4e..425771e3024a 100644 --- a/docs/en/latest/plugins/ai-proxy-multi.md +++ b/docs/en/latest/plugins/ai-proxy-multi.md @@ -2630,6 +2630,8 @@ kubectl apply -f ai-proxy-multi-ic.yaml For verification, the behaviours should be consistent with the verification in [active health checks](../tutorials/health-check.md). +The status these checks produce is reported by the [Control API](../control-api.md): `GET /v1/healthcheck` lists one entry per instance, and `GET /v1/healthcheck/routes/{id}/checkers` returns every checker the Route owns. Each entry carries the instance name in `meta.instance`. + ### Include LLM Information in Access Log The following example demonstrates how you can log LLM request related information in the gateway's access log to improve analytics and audit. The following variables are available: diff --git a/docs/zh/latest/control-api.md b/docs/zh/latest/control-api.md index 9040ce0e896c..f7751e7374a3 100644 --- a/docs/zh/latest/control-api.md +++ b/docs/zh/latest/control-api.md @@ -141,6 +141,36 @@ APISIX 中一些插件添加了自己的 control API。如果你对他们感兴 * nodes[i].counter.tcp_failure: TCP 连接或读写的失败计数器。 * nodes[i].counter.timeout_failure: 超时计数器。 +插件也可以对不属于任何上游的节点进行主动健康检查。[ai-proxy-multi](./plugins/ai-proxy-multi.md) +就是这样:它探测每个 LLM 实例,并在选择目标时跳过不健康的实例。这类检查器会额外带上两个字段: + +```json +{ + "name": "/apisix/routes/1#plugins['ai-proxy-multi'].instances[0]", + "plugin": "ai-proxy-multi", + "meta": { + "instance": "openai" + }, + "type": "http", + "nodes": [ + { + "ip": "52.86.68.46", + "port": 443, + "status": "healthy", + "counter": { + "http_failure": 0, + "success": 2, + "timeout_failure": 0, + "tcp_failure": 0 + } + } + ] +} +``` + +* plugin: 拥有该健康检查器的插件名。上游自身的健康检查器没有该字段。 +* meta: 由插件填写,用于描述该检查器代表什么,内容由插件自行定义;`ai-proxy-multi` 在其中报告实例名。 + 用户也可以通过 `/v1/healthcheck/$src_type/$src_id` 来获取指定 health checker 的状态。 例如,`GET /v1/healthcheck/upstreams/1` 返回: @@ -188,6 +218,36 @@ APISIX 中一些插件添加了自己的 control API。如果你对他们感兴 ![Health Check Status Page](https://raw.githubusercontent.com/apache/apisix/master/docs/assets/images/health_check_status_page.png) +### GET /v1/healthcheck/{src_type}/{src_id}/checkers + +以数组形式返回某个资源拥有的全部健康检查器,数组元素与上文描述的 entry 结构一致。 +一个资源可能拥有多个健康检查器——它自身的上游,加上每个插件实例各一个——而 +`GET /v1/healthcheck/$src_type/$src_id` 返回的是单个对象,无法表达这种情况。 + +例如,`GET /v1/healthcheck/routes/1/checkers` 返回: + +```json +[ + { + "name": "/apisix/routes/1", + "type": "http", + "nodes": [...] + }, + { + "name": "/apisix/routes/1#plugins['ai-proxy-multi'].instances[0]", + "plugin": "ai-proxy-multi", + "meta": { + "instance": "openai" + }, + "type": "http", + "nodes": [...] + } +] +``` + +资源没有配置任何健康检查在这里不算错误:它拥有一个空的检查器集合,接口返回 `[]`。 +只有资源本身不存在时才返回 `404`。 + ### POST /v1/gc 引入自 2.8 版本 diff --git a/docs/zh/latest/plugins/ai-proxy-multi.md b/docs/zh/latest/plugins/ai-proxy-multi.md index e5141d0163e5..dd50b025930c 100644 --- a/docs/zh/latest/plugins/ai-proxy-multi.md +++ b/docs/zh/latest/plugins/ai-proxy-multi.md @@ -2416,6 +2416,8 @@ kubectl apply -f ai-proxy-multi-ic.yaml 为了验证,行为应与[主动健康检查](../tutorials/health-check.md)中的验证一致。 +这些健康检查产生的状态可以通过[控制接口](../control-api.md)获取:`GET /v1/healthcheck` 会为每个实例返回一个 entry,`GET /v1/healthcheck/routes/{id}/checkers` 则返回该路由拥有的全部检查器。每个 entry 通过 `meta.instance` 携带实例名。 + ### 发送请求日志到日志记录器 以下示例演示了如何记录请求和响应信息(包括 LLM 模型、令牌和负载),并将其推送到日志记录器。在继续之前,您应该先设置一个日志记录器,例如 Kafka。有关更多信息,请参阅 [`kafka-logger`](./kafka-logger.md)。 diff --git a/t/control/healthcheck-ai-proxy-multi.t b/t/control/healthcheck-ai-proxy-multi.t new file mode 100644 index 000000000000..ed271c0a3e29 --- /dev/null +++ b/t/control/healthcheck-ai-proxy-multi.t @@ -0,0 +1,660 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +use t::APISIX 'no_plan'; + +log_level("info"); +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + + my $http_config = $block->http_config // <<_EOC_; + server { + server_name openai; + listen 127.0.0.1:16724; + + default_type 'application/json'; + + location /v1/chat/completions { + content_by_lua_block { + ngx.say([[{"choices":[{"message":{"content":"ok","role":"assistant"}}]}]]) + } + } + + location /status { + content_by_lua_block { + ngx.say("ok") + } + } + } +_EOC_ + + $block->set_value("http_config", $http_config); +}); + +run_tests; + +__DATA__ + +=== TEST 1: report the active health check status of each ai-proxy-multi instance +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local http = require("resty.http") + + local checks = { + active = { + timeout = 1, + http_path = "/status", + healthy = { + interval = 1, + successes = 1, + }, + unhealthy = { + interval = 1, + http_failures = 1, + tcp_failures = 1, + timeouts = 1, + }, + }, + } + + local route = { + uri = "/ai", + plugins = { + ["ai-proxy-multi"] = { + fallback_strategy = "instance_health_and_rate_limiting", + instances = { + { + name = "openai-healthy", + provider = "openai", + weight = 1, + priority = 1, + auth = {header = {Authorization = "Bearer token"}}, + options = {model = "gpt-4"}, + override = {endpoint = "http://127.0.0.1:16724"}, + checks = checks, + }, + { + name = "openai-unhealthy", + provider = "openai", + weight = 1, + priority = 0, + auth = {header = {Authorization = "Bearer token"}}, + options = {model = "gpt-4"}, + override = {endpoint = "http://127.0.0.1:16725"}, + checks = checks, + }, + }, + ssl_verify = false, + }, + }, + } + + local code, body = t.test("/apisix/admin/routes/1", ngx.HTTP_PUT, + json.encode(route)) + assert(code < 300, body) + + -- the checkers are created on demand, so the route has to be hit once + local httpc = http.new() + local res, err = httpc:request_uri( + "http://127.0.0.1:" .. ngx.var.server_port .. "/ai", + { + method = "POST", + body = json.encode({messages = {{role = "user", content = "hi"}}}), + headers = {["Content-Type"] = "application/json"}, + } + ) + assert(res, err) + assert(res.status == 200, "unexpected status: " .. tostring(res.status)) + + ngx.sleep(3) + + local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET) + assert(code == 200, body) + res = json.decode(res) + + local infos = {} + for _, info in ipairs(res) do + if info.plugin and info.name:find("/routes/1#", 1, true) then + table.insert(infos, info) + end + end + table.sort(infos, function(a, b) return a.name < b.name end) + + for _, info in ipairs(infos) do + ngx.say(info.name, " plugin=", info.plugin, " instance=", info.meta.instance, + " type=", info.type, " nodes=", #info.nodes, + " ", info.nodes[1].ip, ":", info.nodes[1].port, + " ", info.nodes[1].status) + end + } + } +--- response_body +/apisix/routes/1#plugins['ai-proxy-multi'].instances[0] plugin=ai-proxy-multi instance=openai-healthy type=http nodes=1 127.0.0.1:16724 healthy +/apisix/routes/1#plugins['ai-proxy-multi'].instances[1] plugin=ai-proxy-multi instance=openai-unhealthy type=http nodes=1 127.0.0.1:16725 unhealthy +--- timeout: 10 + + + +=== TEST 2: plugin instance checkers coexist with upstream checkers +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local http = require("resty.http") + + local checks = { + active = { + timeout = 1, + http_path = "/status", + healthy = { + interval = 1, + successes = 1, + }, + unhealthy = { + interval = 1, + http_failures = 1, + tcp_failures = 1, + timeouts = 1, + }, + }, + } + + local route = { + uri = "/ai", + plugins = { + ["ai-proxy-multi"] = { + fallback_strategy = "instance_health_and_rate_limiting", + instances = { + { + name = "openai-first", + provider = "openai", + weight = 1, + priority = 1, + auth = {header = {Authorization = "Bearer token"}}, + options = {model = "gpt-4"}, + override = {endpoint = "http://127.0.0.1:16724"}, + checks = checks, + }, + { + name = "openai-second", + provider = "openai", + weight = 1, + priority = 0, + auth = {header = {Authorization = "Bearer token"}}, + options = {model = "gpt-4"}, + override = {endpoint = "http://127.0.0.1:16724"}, + checks = checks, + }, + }, + ssl_verify = false, + }, + }, + } + + local code, body = t.test("/apisix/admin/routes/1", ngx.HTTP_PUT, + json.encode(route)) + assert(code < 300, body) + + code, body = t.test("/apisix/admin/routes/2", ngx.HTTP_PUT, json.encode({ + uri = "/hello", + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1}, + checks = checks, + }, + })) + assert(code < 300, body) + + for _, uri in ipairs({"/ai", "/hello"}) do + local httpc = http.new() + local res, err = httpc:request_uri( + "http://127.0.0.1:" .. ngx.var.server_port .. uri, + { + method = "POST", + body = json.encode({messages = {{role = "user", content = "hi"}}}), + headers = {["Content-Type"] = "application/json"}, + } + ) + assert(res, err) + end + + ngx.sleep(3) + + local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET) + assert(code == 200, body) + res = json.decode(res) + + local infos = {} + for _, info in ipairs(res) do + if info.name:find("/routes/1", 1, true) + or info.name:find("/routes/2", 1, true) then + table.insert(infos, info) + end + end + table.sort(infos, function(a, b) return a.name < b.name end) + + for _, info in ipairs(infos) do + ngx.say(info.name, " plugin=", tostring(info.plugin), + " nodes=", #info.nodes) + end + } + } +--- response_body +/apisix/routes/1#plugins['ai-proxy-multi'].instances[0] plugin=ai-proxy-multi nodes=1 +/apisix/routes/1#plugins['ai-proxy-multi'].instances[1] plugin=ai-proxy-multi nodes=1 +/apisix/routes/2 plugin=nil nodes=1 +--- timeout: 10 + + + +=== TEST 3: disable_upstream_healthcheck stops probing the plugin instances too +--- yaml_config +apisix: + disable_upstream_healthcheck: true +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local http = require("resty.http") + + local httpc = http.new() + local res, err = httpc:request_uri( + "http://127.0.0.1:" .. ngx.var.server_port .. "/ai", + { + method = "POST", + body = json.encode({messages = {{role = "user", content = "hi"}}}), + headers = {["Content-Type"] = "application/json"}, + } + ) + assert(res, err) + + ngx.sleep(3) + + -- the entries stay listed, but no checker was ever created, so every + -- node list is empty -- the same as for a plain upstream checker + local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET) + assert(code == 200, body) + res = json.decode(res) + local probed = 0 + for _, info in ipairs(res) do + probed = probed + #info.nodes + end + ngx.say("probed nodes: ", probed) + + local code, body, res = t.test("/v1/healthcheck/routes/1/checkers", ngx.HTTP_GET) + assert(code == 200, body) + res = json.decode(res) + probed = 0 + for _, info in ipairs(res) do + probed = probed + #info.nodes + end + ngx.say("checkers: ", #res, " probed nodes: ", probed) + } + } +--- response_body +probed nodes: 0 +checkers: 2 probed nodes: 0 +--- timeout: 10 + + + +=== TEST 4: instances are listed before the checkers have probed anything +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + + local checks = { + active = { + timeout = 1, + http_path = "/status", + healthy = { + interval = 1, + successes = 1, + }, + unhealthy = { + interval = 1, + http_failures = 1, + tcp_failures = 1, + timeouts = 1, + }, + }, + } + + local instances = {} + for _, name in ipairs({"openai-a", "openai-b"}) do + table.insert(instances, { + name = name, + provider = "openai", + weight = 1, + auth = {header = {Authorization = "Bearer token"}}, + options = {model = "gpt-4"}, + override = {endpoint = "http://127.0.0.1:16724"}, + checks = checks, + }) + end + + local code, body = t.test("/apisix/admin/routes/3", ngx.HTTP_PUT, json.encode({ + uri = "/ai-untouched", + plugins = { + ["ai-proxy-multi"] = { + instances = instances, + ssl_verify = false, + }, + }, + })) + assert(code < 300, body) + + -- give the route time to reach the worker, but never send a request + ngx.sleep(1) + + local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET) + assert(code == 200, body) + res = json.decode(res) + + local infos = {} + for _, info in ipairs(res) do + if info.name:find("/routes/3#", 1, true) then + table.insert(infos, info) + end + end + table.sort(infos, function(a, b) return a.name < b.name end) + + for _, info in ipairs(infos) do + ngx.say(info.name, " instance=", info.meta.instance, + " type=", info.type, " nodes=", #info.nodes) + end + } + } +--- response_body +/apisix/routes/3#plugins['ai-proxy-multi'].instances[0] instance=openai-a type=http nodes=0 +/apisix/routes/3#plugins['ai-proxy-multi'].instances[1] instance=openai-b type=http nodes=0 +--- timeout: 10 + + + +=== TEST 5: the checkers sub-resource lists every checker of an AI route +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local http = require("resty.http") + + local checks = { + active = { + timeout = 1, + http_path = "/status", + healthy = { + interval = 1, + successes = 1, + }, + unhealthy = { + interval = 1, + http_failures = 1, + tcp_failures = 1, + timeouts = 1, + }, + }, + } + + local code, body = t.test("/apisix/admin/routes/4", ngx.HTTP_PUT, json.encode({ + uri = "/ai4", + plugins = { + ["ai-proxy-multi"] = { + fallback_strategy = "instance_health_and_rate_limiting", + instances = { + { + name = "openai-up", + provider = "openai", + weight = 1, + priority = 1, + auth = {header = {Authorization = "Bearer token"}}, + options = {model = "gpt-4"}, + override = {endpoint = "http://127.0.0.1:16724"}, + checks = checks, + }, + { + name = "openai-down", + provider = "openai", + weight = 1, + priority = 0, + auth = {header = {Authorization = "Bearer token"}}, + options = {model = "gpt-4"}, + override = {endpoint = "http://127.0.0.1:16725"}, + checks = checks, + }, + }, + ssl_verify = false, + }, + }, + })) + assert(code < 300, body) + + local httpc = http.new() + local res, err = httpc:request_uri( + "http://127.0.0.1:" .. ngx.var.server_port .. "/ai4", + { + method = "POST", + body = json.encode({messages = {{role = "user", content = "hi"}}}), + headers = {["Content-Type"] = "application/json"}, + } + ) + assert(res, err) + assert(res.status == 200, "unexpected status: " .. tostring(res.status)) + + ngx.sleep(3) + + local code, body, res = t.test("/v1/healthcheck/routes/4/checkers", ngx.HTTP_GET) + assert(code == 200, body) + res = json.decode(res) + table.sort(res, function(a, b) return a.name < b.name end) + + for _, info in ipairs(res) do + ngx.say(info.name, " plugin=", info.plugin, + " instance=", info.meta.instance, + " type=", info.type, + " ", info.nodes[1].ip, ":", info.nodes[1].port, + " ", info.nodes[1].status) + end + } + } +--- response_body +/apisix/routes/4#plugins['ai-proxy-multi'].instances[0] plugin=ai-proxy-multi instance=openai-up type=http 127.0.0.1:16724 healthy +/apisix/routes/4#plugins['ai-proxy-multi'].instances[1] plugin=ai-proxy-multi instance=openai-down type=http 127.0.0.1:16725 unhealthy +--- timeout: 10 + + + +=== TEST 6: the checkers sub-resource reports a plain upstream checker too +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + local http = require("resty.http") + + local code, body = t.test("/apisix/admin/routes/5", ngx.HTTP_PUT, json.encode({ + uri = "/hello5", + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1}, + checks = { + active = { + timeout = 1, + http_path = "/status", + healthy = { + interval = 1, + successes = 1, + }, + unhealthy = { + interval = 1, + http_failures = 1, + tcp_failures = 1, + timeouts = 1, + }, + }, + }, + }, + })) + assert(code < 300, body) + + local httpc = http.new() + local res, err = httpc:request_uri( + "http://127.0.0.1:" .. ngx.var.server_port .. "/hello5", + {method = "GET"} + ) + assert(res, err) + + ngx.sleep(3) + + local code, body, res = t.test("/v1/healthcheck/routes/5/checkers", ngx.HTTP_GET) + assert(code == 200, body) + res = json.decode(res) + + for _, info in ipairs(res) do + ngx.say(info.name, " plugin=", tostring(info.plugin), + " meta=", tostring(info.meta), + " type=", info.type, " nodes=", #info.nodes) + end + + -- the single-object endpoint keeps returning exactly one checker + local code, body, res = t.test("/v1/healthcheck/routes/5", ngx.HTTP_GET) + assert(code == 200, body) + res = json.decode(res) + ngx.say("single: ", res.name, " nodes=", #res.nodes) + } + } +--- response_body +/apisix/routes/5 plugin=nil meta=nil type=http nodes=1 +single: /apisix/routes/5 nodes=1 +--- timeout: 10 + + + +=== TEST 7: a resource without health checks owns an empty checker set +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + + local code, body = t.test("/apisix/admin/routes/6", ngx.HTTP_PUT, json.encode({ + uri = "/hello6", + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1}, + }, + })) + assert(code < 300, body) + + local function trim(s) + return (tostring(s):gsub("%s+$", "")) + end + + local code, body, res = t.test("/v1/healthcheck/routes/6/checkers", ngx.HTTP_GET) + ngx.say("no checks: ", code, " ", trim(res)) + + -- the single-object endpoint still reports this as an error + local code, body = t.test("/v1/healthcheck/routes/6", ngx.HTTP_GET) + ngx.say("single: ", code, " ", trim(body)) + + local code, body = t.test("/v1/healthcheck/routes/404/checkers", ngx.HTTP_GET) + ngx.say("missing: ", code, " ", trim(body)) + + local code, body = t.test("/v1/healthcheck/routes/6/nodes", ngx.HTTP_GET) + ngx.say("bad sub resource: ", code, " ", trim(body)) + } + } +--- response_body +no checks: 200 [] +single: 404 {"error_msg":"no checker for routes[6]"} +missing: 404 {"error_msg":"routes[404] not found"} +bad sub resource: 400 {"error_msg":"invalid sub resource nodes"} +--- timeout: 10 + + + +=== TEST 8: a disabled plugin owns no checkers +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + local json = require("toolkit.json") + + -- a disabled plugin is kept even when its configuration does not pass + -- check_schema(), so `instances` here is not even an array + local code, body = t.test("/apisix/admin/routes/7", ngx.HTTP_PUT, json.encode({ + uri = "/ai7", + plugins = { + ["ai-proxy-multi"] = { + _meta = {disable = true}, + instances = "not an array", + }, + }, + upstream = { + type = "roundrobin", + nodes = {["127.0.0.1:1980"] = 1}, + }, + })) + assert(code < 300, body) + + ngx.sleep(1) + + local code, body, res = t.test("/v1/healthcheck", ngx.HTTP_GET) + ngx.say("list: ", code) + + local code, body, res = t.test("/v1/healthcheck/routes/7/checkers", ngx.HTTP_GET) + ngx.say("checkers: ", code, " ", (tostring(res):gsub("%s+$", ""))) + } + } +--- response_body +list: 200 +checkers: 200 [] +--- timeout: 10 + + + +=== TEST 9: clean up +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin") + for _, id in ipairs({1, 2, 3, 4, 5, 6, 7}) do + local code, body = t.test("/apisix/admin/routes/" .. id, ngx.HTTP_DELETE) + assert(code < 300, body) + end + ngx.say("passed") + } + } +--- response_body +passed