From 8cc56dc9eab86bde65654fcaa73eace8e134a44f Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Wed, 27 May 2026 04:08:42 +0800 Subject: [PATCH 01/12] feat: add toolset plugin The toolset plugin is a diagnostics and observability framework that hosts multiple lightweight sub-plugins, each independently configured via plugin_attr and dynamically loaded/unloaded at runtime. Sub-plugins included: - trace: instruments APISIX request phases and logs a timing table for sampled requests, supports host/path filtering, sampling rate, trace header detection, and minimum timespan threshold - table_count: periodically measures and logs the entry count of specified Lua module tables, useful for monitoring memory growth Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apisix/cli/config.lua | 16 + apisix/plugins/toolset/config.lua | 17 + apisix/plugins/toolset/init.lua | 148 ++++++ .../plugins/toolset/src/table-count/init.lua | 105 +++++ apisix/plugins/toolset/src/trace.lua | 445 ++++++++++++++++++ conf/config.yaml.example | 18 +- docs/en/latest/config.json | 1 + docs/en/latest/plugins/toolset.md | 159 +++++++ 8 files changed, 908 insertions(+), 1 deletion(-) create mode 100644 apisix/plugins/toolset/config.lua create mode 100644 apisix/plugins/toolset/init.lua create mode 100644 apisix/plugins/toolset/src/table-count/init.lua create mode 100644 apisix/plugins/toolset/src/trace.lua create mode 100644 docs/en/latest/plugins/toolset.md diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua index c42ecbdeec8f..375025734743 100644 --- a/apisix/cli/config.lua +++ b/apisix/cli/config.lua @@ -340,6 +340,22 @@ local _M = { ["server-info"] = { report_ttl = 60 }, + toolset = { + trace = { + rate = 1, + hosts = {}, + paths = {}, + gen_uid = false, + vars = {}, + timespan_threshold = 0 + }, + table_count = { + lua_modules = {}, + interval = 5, + depth = 10, + scopes = {"worker", "privileged agent"} + } + }, ["dubbo-proxy"] = { upstream_multiplex_count = 32 }, diff --git a/apisix/plugins/toolset/config.lua b/apisix/plugins/toolset/config.lua new file mode 100644 index 000000000000..350616386784 --- /dev/null +++ b/apisix/plugins/toolset/config.lua @@ -0,0 +1,17 @@ +return { + trace = { + rate = 1, -- allow only 1 request per 100 requests + hosts = {}, -- only the requests carrying these host headers will be traced + paths = {}, -- only these request_uris will be traced + gen_uid = false, -- adds a UID to the trace if none of the traceable headers are found + vars = {}, -- add these nginx or inbuilt variables to trace table + timespan_threshold = 0 -- requests taking longer than this value (in seconds) will be traced + }, + table_count = { + lua_modules = {}, -- change it + interval = 5, + depth = 10, -- when it is not passed, default depth will be 1 + -- optional, default is all APISIX processes + scopes = {"worker", "privileged agent"} + } +} diff --git a/apisix/plugins/toolset/init.lua b/apisix/plugins/toolset/init.lua new file mode 100644 index 000000000000..66efed91343e --- /dev/null +++ b/apisix/plugins/toolset/init.lua @@ -0,0 +1,148 @@ +local pairs = pairs +local core = require("apisix.core") +local ngx = ngx +local cache = core.table.new(0, 32) +local stop_timer = false +local load, unload = "load", "unload" +local package = package +local pcall = pcall +local require = require +local string = string + +local _M = { + version = 0.1, + priority = 22901, + name = "toolset", + schema = {}, + scope = "global", +} + + +local function get_plugin_config() + -- clear cache to reload + package.loaded["apisix.plugins.toolset.config"] = nil + local loaded, plugins_config = pcall(require, "apisix.plugins.toolset.config") + if loaded and plugins_config == true then + core.log.warn("empty plugin config file") + return nil + end + if not loaded then + core.log.error("failed to load plugin config: ", plugins_config) + return nil + end + return plugins_config +end + + +local function is_config_changed(plugin_name, plugin_config) + if core.table.deep_eq(cache[plugin_name], plugin_config) then + return false + end + return true +end + + +local function is_config_empty(plugin_config) + return plugin_config == nil or core.table.deep_eq(plugin_config, {}) +end + + +local function perform_operation_for_plugin(plugin_name, plugin_config, operation) + if operation == load then + local loaded, plugin = pcall(require, "apisix.plugins.toolset.src." + .. string.gsub(plugin_name, "_", "-")) + if not loaded then + core.log.warn("could not load plugin because it was not found: ", plugin_name) + return + end + core.log.warn("initializing sub plugin for toolset plugin: ", plugin_name) + plugin.init() + cache[plugin_name] = plugin_config + elseif operation == unload then + local loaded, plugin = pcall(require, "apisix.plugins.toolset.src." .. + string.gsub(plugin_name, "_", "-")) + if not loaded then + core.log.warn("could not unload plugin because it was not found: ", plugin_name) + return + end + core.log.warn("destroying sub plugin for toolset plugin: ", plugin_name) + plugin.destroy() + cache[plugin_name] = nil + end +end + + +local function sync() + core.log.info("syncing toolset plugin") + local plugin_configs = get_plugin_config() + local processed_plugins = {} + if plugin_configs then + for plugin_name, plugin_config in pairs(plugin_configs) do + processed_plugins[plugin_name] = true + -- checks if the config is different from cache + if is_config_changed(plugin_name, plugin_config) then + if is_config_empty(plugin_config) then + -- allow executing even with empty config. + -- Assuming the plugin will run with default values + core.log.warn("empty config found for ", plugin_name,".Running with default values") + end + core.log.warn("config changed. reloading plugin: ", plugin_name) + local ok, err = pcall(perform_operation_for_plugin, plugin_name, plugin_config, load) + if not ok then + core.log.error("toolset plugin load raised: ", err) + end + end + end + end + + for plugin_name, plugin_config in pairs(cache) do + if not processed_plugins[plugin_name] then + core.log.warn("plugin config unloaded: ", plugin_name) + local ok, err = pcall(perform_operation_for_plugin, plugin_name, plugin_config, unload) + if not ok then + core.log.error("toolset plugin unload raised: ", err) + end + end + end + if not stop_timer then + local ok, err = ngx.timer.at(1, sync) + if not ok then + core.log.error("failed to create timer for running toolset ", err) + end + end +end + + +function _M.init() + core.log.info("initializing toolset plugin") + local plugins_config = get_plugin_config() + if plugins_config then + for plugin_name, plugin_config in pairs(plugins_config) do + if is_config_empty(plugin_config) then + -- allow executing even with empty config. + -- Assuming the plugin will run with default values + core.log.warn("empty config found for ", plugin_name,".Running with default values") + end + perform_operation_for_plugin(plugin_name, plugin_config, load) + end + end + ngx.timer.at(1, sync) +end + + +function _M.destroy() + local plugin_configs = get_plugin_config() + if plugin_configs then + for plugin_name, plugin_config in pairs(plugin_configs) do + perform_operation_for_plugin(plugin_name, plugin_config, unload) + end + + end + for plugin_name, plugin_config in pairs(cache) do + perform_operation_for_plugin(plugin_name, plugin_config, unload) + end + + stop_timer = true +end + +return _M diff --git a/apisix/plugins/toolset/src/table-count/init.lua b/apisix/plugins/toolset/src/table-count/init.lua new file mode 100644 index 000000000000..ade2f5ab824d --- /dev/null +++ b/apisix/plugins/toolset/src/table-count/init.lua @@ -0,0 +1,105 @@ +local core = require("apisix.core") +local ngx = require("ngx") +local process = require("ngx.process") + +local pairs = pairs +local ipairs = ipairs +local type = type +local timer = ngx.timer +local require = require +local package = package + +local plugin_name = "table-count" + +local schema = {} +local stop = false +-- only one run of init() function should be running at a time. +-- when init() is reloaded the run number is incremented. It also helps in debugging. +local current_run = 0 + +local _M = { + version = 0.1, + priority = 22902, + name = plugin_name, + schema = schema, + scope = "global", +} + +local function tab_item_count(tab, cache,depth) + if depth == 0 then + core.log.warn("out of depth..skipping count") + return + end + depth = depth - 1 + cache = cache or {} + local count = 0 + for _, value in pairs(tab) do + if cache[value] then + core.log.warn("circular reference detected..skipping count") + goto continue + end + if type(value) == "table" and not cache[value] then + cache[value] = true + local tab_count = tab_item_count(value, cache,depth) + if tab_count then + count = count + tab_count + 1 + end + else + count = count + 1 + end + ::continue:: + end + return count +end + +function _M.init() + package.loaded["apisix.plugins.toolset.config"] = nil + local config = require("apisix.plugins.toolset.config").table_count + if config.lua_modules == nil or #config.lua_modules == 0 then + core.log.warn("no lua_modules provided for table count") + return + end + if not config.scopes then + core.log.warn("no scope provided. Running for all scopes") + goto continue + end + if #config.scopes ~= 0 then + for _,scope in ipairs(config.scopes) do + if process.type() == scope then + goto continue + end + end + return + end + ::continue:: + -- Extract configuration values + current_run = current_run + 1 + local interval = config.interval or 5 + local run_count + run_count = function(run_no) + local depth = config.depth or 1 + for _, package_name in ipairs(config.lua_modules) do + local package = require(package_name) + local count = tab_item_count(package, {},depth) + core.log.warn("package ", package_name, " table count is: ", count," for loaded: ",run_no) + end + if stop or run_no ~= current_run then + return + end + local ok, err = timer.at(interval, run_count,current_run) + if not ok then + core.log.error("failed to create timer for running table count ", err) + end + end + + local ok, err = timer.at(0, run_count,current_run) + if not ok then + core.log.error("failed to create timer for running table count ", err) + end +end + +function _M.destroy() + stop = true +end + +return _M diff --git a/apisix/plugins/toolset/src/trace.lua b/apisix/plugins/toolset/src/trace.lua new file mode 100644 index 000000000000..2c7f48ab689b --- /dev/null +++ b/apisix/plugins/toolset/src/trace.lua @@ -0,0 +1,445 @@ +local require = require +local apisix = require("apisix") +local core = require("apisix.core") +local uuid = require("resty.jit-uuid") + +local conf_path = "apisix.plugins.toolset.config" + +local ngx = ngx +local pairs = pairs +local type = type +local package = package +local tostring = tostring +local format = string.format +local floor = math.floor +local gsub = ngx.re.gsub +local m_random = math.random +local m_randomseed = math.randomseed +local t_remove = table.remove +local re_match = ngx.re.match +local counter = 1 + +local old_http_access_phase +local old_match_route +local old_http_log_phase +local old_http_balancer_phase +local old_http_header_filter_phase +local old_http_body_filter_phase +local old_resolve + +local schema = {} + +local PHASE_UPSTREAM = "upstream (req + response)" +local PHASE_CLIENT = "response" + +local suffix = [[ ++----------+---------------------------+----------+-------------------------+ +]] +local prefix = [[ + ++----------+---------------------------+----------+-------------------------+ +| Role | Phase | Timespan | Start time | +]] .. suffix + +local trace_headers = { + "x-request-id", -- request id header + "sw8", -- skywalking + "traceparent", -- opentelemetry + "x-b3-traceid", -- zipkin +} +local plugin_name = "trace" + +local _M = { + version = 0.1, + priority = 22901, + name = plugin_name, + schema = schema, + scope = "global", +} + +local function nspaces(n) + return (" "):rep(n) +end + +local function add_entry(phase, timespan, curtime) + core.log.info("add entry for: ", phase) + local role + local tpl = [[ +| %s| %s| %s| %s | +]] + if phase == PHASE_UPSTREAM then + role = "Upstream " + elseif phase == PHASE_CLIENT then + role = "Client " + else + role = "APISIX " + end + + -- add spaces around the text for table formatting + phase = phase .. nspaces(26 - #phase) + timespan = timespan .. nspaces(9 - #tostring(timespan)) + ngx.ctx.trace_log = ngx.ctx.trace_log .. format(tpl, role, phase, timespan, curtime) +end + + +local function timespan(raw) + if raw == 0 then + return "0ms" + end + local factor = 1000 -- 1000ms in 1s + local unit = "ms" + if raw >= 1 then -- if greater than 1s don't convert to ms + factor = 1 + unit = "s" + end + return floor(raw * factor + 0.5) .. unit +end + + +local function localtime_msec(now) + local lt = ngx.localtime() + local msec = now * 1000 - floor(now) * 1000 + if msec > 0 then + return lt .. "." .. msec + end + return lt .. ".000" +end + + +local function match(incoming, conf) + conf = gsub(conf, "\\*", ".*") + conf = "^" .. conf .. "$" + core.log.info("matching: ", incoming, " against: ", conf) + + local matches = re_match(incoming, "^" .. conf .. "$", "jo") + if not matches then + return nil + end + return matches[0] +end + + +local unique_random +do + local numbers = {} + for i = 1, 100 do + numbers[i] = i + end + unique_random = function() + m_randomseed(ngx.now()) + while true do + local index = m_random(100) + local num = numbers[index] + if num then + t_remove(numbers, index) + return num + end + end + end +end + + +local function incr_counter() + counter = counter + 1 + if counter > 99 then + counter = 0 + end +end + + +local function preprocess(trace_conf, ctx) + if not trace_conf.rate or type(trace_conf.rate) ~= "number" then + ctx.trace = true -- trace all reqs if rate isn't defined + return + end + if trace_conf.rate == 1 then + ctx.trace = counter == 1 -- trace only first request + incr_counter() + return + end + core.log.info("trace_conf.rate: ", trace_conf.rate) + local rand = unique_random() + if rand <= trace_conf.rate then + ctx.trace = true + end + core.log.info("random number: ", rand) + incr_counter() +end + + +local function check(trace_conf, uri_or_host) + for _, val in pairs(trace_conf) do + if match(uri_or_host, val) == uri_or_host then + return true + end + end + return false +end + + +local function check_host(trace_conf) + local req_host = core.request.header(ngx.ctx, "host") + if (trace_conf.hosts and #trace_conf.hosts > 0) and (req_host and #req_host > 0) then + return check(trace_conf.hosts, req_host) + end + -- pass host check if hosts field is not defined in config.lua + return trace_conf.hosts ~= nil +end + + +local function check_uri(trace_conf) + if trace_conf.paths and #trace_conf.paths > 0 then + return check(trace_conf.paths, ngx.ctx.api_ctx.var.request_uri) + end + -- pass uri check if paths field is not defined in config.lua + return true +end + + +local function prepend(ctx, field, val) + ctx.trace_log = "\n" .. field .. ": " .. val .. ctx.trace_log +end + + +local function add_headers(ctx) + local count = 0 + for _, header_field in pairs(trace_headers) do + local val = core.request.header(ctx, header_field) + if val and #val > 0 then + prepend(ctx, header_field, val) + count = count + 1 + end + end + return count +end + + +local function add_vars(ctx, vars) + local count = 0 + if vars and #vars > 0 then + for _, var in pairs(vars) do + local val = ngx.var[var] + if val and #val > 0 then + prepend(ctx, var, val) + count = count + 1 + end + end + end + return count +end + + +function _M.init() + package.loaded[conf_path] = false + local trace_conf = require(conf_path).trace + core.log.info("trace_conf: ", core.json.encode(trace_conf)) + + local conf = core.config.local_conf() + local router_name = "radixtree_uri" + if conf and conf.apisix and conf.apisix.router then + router_name = conf.apisix.router.http or router_name + end + + local dns = require("apisix.core.dns.client") + if dns then + if not old_resolve then + old_resolve = dns.resolve + end + + dns.resolve = function (...) + local match_start = ngx.now() + ngx.ctx.dns_lt = localtime_msec(match_start) + local ret = old_resolve(...) + ngx.update_time() + + ngx.ctx.dns_resolve_timespan = ngx.now() - match_start + return ret + end + end + + local router = require("apisix.http.router." .. router_name) + if not old_match_route then + old_match_route = router.match + end + router.match = function(...) + local match_start = ngx.now() + ngx.ctx.match_lt = localtime_msec(match_start) + + old_match_route(...) + ngx.update_time() + + ngx.ctx.match_timespan = ngx.now() - match_start + end + + if not old_http_access_phase then + old_http_access_phase = apisix.http_access_phase + end + apisix.http_access_phase = function(...) + ngx.ctx.trace = false + preprocess(trace_conf, ngx.ctx) + if not ngx.ctx.trace then + old_http_access_phase(...) + else + ngx.ctx.trace_log = prefix + + local access_start = ngx.now() + ngx.ctx.req_start = access_start + ngx.ctx.access_lt = localtime_msec(access_start) + + old_http_access_phase(...) + + local host_pass = check_host(trace_conf) + local path_pass = check_uri(trace_conf) + + core.log.info("path check: ", path_pass, ". host check: ", host_pass) + ngx.ctx.trace = path_pass or host_pass + ngx.update_time() + + ngx.ctx.access_timespan = ngx.now() - access_start + end + end + + if not old_http_balancer_phase then + old_http_balancer_phase = apisix.http_balancer_phase + end + apisix.http_balancer_phase = function(...) + if not ngx.ctx.trace then + old_http_balancer_phase(...) + else + local num_headers = add_headers(ngx.ctx) + local num_vars = add_vars(ngx.ctx, trace_conf.vars) + -- if no vars or headers were added add a uuid + if (num_headers + num_vars) < 1 and trace_conf.gen_uid then + ngx.ctx.trace_log = "\n" .. "uuid: " .. uuid() .. ngx.ctx.trace_log + end + + local balancer_start = ngx.now() + ngx.ctx.balancer_lt = localtime_msec(balancer_start) + + old_http_balancer_phase(...) + ngx.update_time() + + ngx.ctx.balancer_timespan = ngx.now() - balancer_start + ngx.update_time() + ngx.ctx.upstream_start = ngx.now() + ngx.ctx.upstream_lt = localtime_msec(ngx.ctx.upstream_start) + end + end + + if not old_http_header_filter_phase then + old_http_header_filter_phase = apisix.http_header_filter_phase + end + apisix.http_header_filter_phase = function(...) + if not ngx.ctx.trace then + old_http_header_filter_phase(...) + else + local header_filter_start = ngx.now() + ngx.ctx.upstream_end = header_filter_start + ngx.ctx.header_filter_start = localtime_msec(header_filter_start) + + old_http_header_filter_phase(...) + ngx.update_time() + + ngx.ctx.header_filter_timespan = ngx.now() - header_filter_start + end + end + + if not old_http_body_filter_phase then + old_http_body_filter_phase = apisix.http_body_filter_phase + end + apisix.http_body_filter_phase = function(...) + local body_filter_start = ngx.now() + if not ngx.ctx.trace then + old_http_body_filter_phase(...) + else + if not ngx.ctx.bf_timespan then + ngx.ctx.bf_timespan = 0 + ngx.ctx.bf_lt = localtime_msec(body_filter_start) + end + + old_http_body_filter_phase(...) + ngx.update_time() + + ngx.ctx.bf_end = ngx.now() + ngx.ctx.bf_timespan = ngx.ctx.bf_timespan + (ngx.ctx.bf_end - body_filter_start) + ngx.ctx.response_lt = localtime_msec(ngx.ctx.bf_end) + end + end + + if not old_http_log_phase then + old_http_log_phase = apisix.http_log_phase + end + apisix.http_log_phase = function(...) + if not ngx.ctx.trace then + old_http_log_phase(...) + else + local log_start = ngx.now() + local log_lt = localtime_msec(log_start) + + old_http_log_phase(...) + ngx.update_time() + local log_end = ngx.now() + + local premature = false + -- when route match fails access_timespan = nil + if not ngx.ctx.access_timespan then + ngx.ctx.access_timespan = 0 + ngx.ctx.balancer_timespan = 0 + premature = true + end + + local upstream_timespan = 0 + if not premature then + upstream_timespan = ngx.ctx.upstream_end - ngx.ctx.upstream_start + end + + local client_timespan = log_start - ngx.ctx.bf_end + local log_timespan = log_end - log_start + local total_time = ngx.ctx.access_timespan + ngx.ctx.balancer_timespan + upstream_timespan + + ngx.ctx.header_filter_timespan + ngx.ctx.bf_timespan + client_timespan + + log_timespan + + if total_time >= (trace_conf.timespan_threshold or 0) then + add_entry("access", timespan(ngx.ctx.access_timespan), ngx.ctx.access_lt) + add_entry("\\_match_route", timespan(ngx.ctx.match_timespan), ngx.ctx.match_lt) + if ngx.ctx.dns_resolve_timespan then + add_entry("\\_dns_resolve", timespan(ngx.ctx.dns_resolve_timespan), ngx.ctx.dns_lt) + end + if not premature then + add_entry("balancer", timespan(ngx.ctx.balancer_timespan), ngx.ctx.balancer_lt) + add_entry(PHASE_UPSTREAM, + timespan(upstream_timespan), ngx.ctx.upstream_lt) + end + add_entry("header_filter", timespan(ngx.ctx.header_filter_timespan), + ngx.ctx.header_filter_start) + add_entry("body_filter", timespan(ngx.ctx.bf_timespan), ngx.ctx.bf_lt) + if not premature then + add_entry(PHASE_CLIENT, timespan(client_timespan), ngx.ctx.response_lt) + end + add_entry("log", timespan(log_timespan), log_lt) + core.log.warn("trace: ", ngx.ctx.trace_log .. suffix) + end + end + ngx.ctx.trace_log = "" -- clear trace + ngx.ctx.bf_timespan = nil -- clear body_filter timespan + end +end + +function _M.destroy() + local conf = core.config.local_conf() + local router_name = "radixtree_uri" + if conf and conf.apisix and conf.apisix.router then + router_name = conf.apisix.router.http or router_name + end + + local router = require("apisix.http.router." .. router_name) + router.match = old_match_route + + apisix.http_access_phase = old_http_access_phase + apisix.http_balancer_phase = old_http_balancer_phase + apisix.http_header_filter_phase = old_http_header_filter_phase + apisix.http_body_filter_phase = old_http_body_filter_phase + apisix.http_log_phase = old_http_log_phase +end + +return _M diff --git a/conf/config.yaml.example b/conf/config.yaml.example index 6023c83bc3dc..ab15e1a8eb08 100644 --- a/conf/config.yaml.example +++ b/conf/config.yaml.example @@ -471,8 +471,9 @@ graphql: plugins: # plugin list (sorted by priority) - real-ip # priority: 23000 - - ai # priority: 22900 #- exit-transformer # priority: 22950, disabled by default + #- toolset # priority: 22901, disabled by default + - ai # priority: 22900 - client-control # priority: 22000 - proxy-control # priority: 21990 - request-id # priority: 12015 @@ -665,6 +666,21 @@ plugin_attr: # Plugin attributes server-info: # Plugin: server-info report_ttl: 60 # Set the TTL in seconds for server info in etcd. # Maximum: 86400. Minimum: 3. + # toolset: # Plugin: toolset + # trace: # Sub-plugin: trace - instruments APISIX phases and logs timing info + # rate: 1 # Allow only 1 request per 100 requests to be traced + # hosts: [] # Only trace requests with these host headers (empty = all) + # paths: [] # Only trace requests with these URIs (empty = all) + # gen_uid: false # Add a UID to the trace when no traceable headers are found + # vars: [] # Additional nginx/inbuilt variables to include in trace output + # timespan_threshold: 0 # Only log traces for requests taking longer than this (in seconds) + # table_count: # Sub-plugin: table_count - periodically logs table sizes of Lua modules + # lua_modules: [] # List of Lua module names to measure (e.g. ["apisix.router"]) + # interval: 5 # Interval in seconds between measurements + # depth: 10 # Maximum depth for recursive table counting + # scopes: # APISIX process scopes to run in (default: all) + # - worker + # - privileged agent dubbo-proxy: # Plugin: dubbo-proxy upstream_multiplex_count: 32 # Set the maximum number of connections that can be multiplexed over # a single network connection between the Dubbo Proxy and the upstream diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index 115448b95b0e..188736485661 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -94,6 +94,7 @@ "plugins/brotli", "plugins/real-ip", "plugins/server-info", + "plugins/toolset", "plugins/ext-plugin-pre-req", "plugins/ext-plugin-post-req", "plugins/ext-plugin-post-resp", diff --git a/docs/en/latest/plugins/toolset.md b/docs/en/latest/plugins/toolset.md new file mode 100644 index 000000000000..940b57f91b15 --- /dev/null +++ b/docs/en/latest/plugins/toolset.md @@ -0,0 +1,159 @@ +--- +title: toolset +keywords: + - Apache APISIX + - API Gateway + - Plugin + - Toolset + - toolset + - trace + - table-count +description: This document contains information about the Apache APISIX toolset Plugin. +--- + + + +## Description + +The `toolset` Plugin is a diagnostics and observability framework that hosts multiple lightweight sub-plugins. Each sub-plugin is configured in `config.yaml` under the `plugin_attr.toolset` key and is dynamically loaded or unloaded at runtime without restarting APISIX. The `toolset` plugin itself has no per-route schema and always operates at the global scope. + +### Sub-plugins + +| Sub-plugin | Description | +|---------------|------------------------------------------------------------------------------------------------| +| `trace` | Instruments APISIX request phases and emits a timing table to the error log for matching requests. | +| `table_count` | Periodically measures and logs the item count of specified Lua module tables. | + +## Attributes + +The `toolset` Plugin is configured through `plugin_attr` in `config.yaml` and has no route-level attributes. + +### trace + +| Name | Type | Required | Default | Description | +|-----------------------|---------|----------|---------|--------------------------------------------------------------------------------------------------------------| +| rate | integer | False | 1 | Sampling rate as N-out-of-100. `1` traces 1 request per 100; set to `100` to trace every request. | +| hosts | array | False | `[]` | Allowlist of `Host` header values (glob patterns supported). Empty means all hosts pass. | +| paths | array | False | `[]` | Allowlist of request URI patterns (glob patterns supported). Empty means all paths pass. | +| gen_uid | boolean | False | false | When `true`, generates a UUID for traces where no standard trace header (`x-request-id`, `traceparent`, etc.) is found. | +| vars | array | False | `[]` | Additional nginx or APISIX variables to prepend to the trace output. | +| timespan_threshold | number | False | 0 | Minimum total request duration (in seconds) required before emitting the trace log. `0` logs all traces. | + +### table_count + +| Name | Type | Required | Default | Description | +|--------------|---------|----------|-----------------------------------|---------------------------------------------------------------------------------------| +| lua_modules | array | True | | List of Lua module paths to measure (e.g. `["apisix.router"]`). | +| interval | integer | False | 5 | Interval in seconds between measurements. | +| depth | integer | False | 10 | Maximum recursion depth when counting table entries. `0` disables recursive counting. | +| scopes | array | False | `["worker", "privileged agent"]` | APISIX process types in which the sub-plugin runs. | + +## Enable Plugin + +The `toolset` Plugin must be added to the `plugins` list in `config.yaml`. All sub-plugin configuration is placed under `plugin_attr.toolset`: + +```yaml +plugins: + - toolset + +plugin_attr: + toolset: + trace: + rate: 10 + hosts: + - "*.example.com" + paths: + - "/api/*" + gen_uid: true + vars: + - remote_addr + timespan_threshold: 0.5 + table_count: + lua_modules: + - apisix.router + interval: 10 + depth: 5 + scopes: + - worker +``` + +## Example usage + +### Tracing slow requests + +The following configuration traces up to 10% of requests to `*.example.com` whose total processing time exceeds 500ms: + +```yaml +plugin_attr: + toolset: + trace: + rate: 10 + hosts: + - "*.example.com" + timespan_threshold: 0.5 +``` + +When a request meets the criteria, APISIX writes a table similar to the following to the error log at `WARN` level: + +``` ++----------+---------------------------+----------+-------------------------+ +| Role | Phase | Timespan | Start time | ++----------+---------------------------+----------+-------------------------+ +| APISIX | access | 3ms | 2024-01-01 12:00:00.123 | +| APISIX | \_match_route | 1ms | 2024-01-01 12:00:00.124 | +| APISIX | balancer | 1ms | 2024-01-01 12:00:00.125 | +| Upstream | upstream (req + response) | 520ms | 2024-01-01 12:00:00.126 | +| APISIX | header_filter | 0ms | 2024-01-01 12:00:00.646 | +| APISIX | body_filter | 0ms | 2024-01-01 12:00:00.646 | +| Client | response | 1ms | 2024-01-01 12:00:00.647 | +| APISIX | log | 0ms | 2024-01-01 12:00:00.648 | ++----------+---------------------------+----------+-------------------------+ +``` + +### Monitoring router table growth + +The following configuration measures the item count of the `apisix.router` Lua module every 30 seconds in worker processes: + +```yaml +plugin_attr: + toolset: + table_count: + lua_modules: + - apisix.router + interval: 30 + depth: 5 + scopes: + - worker +``` + +Results are written to the error log at `WARN` level: + +``` +package apisix.router table count is: 1234 for loaded: 1 +``` + +## Disable Plugin + +Remove `toolset` from the `plugins` list in `config.yaml` and reload APISIX: + +```yaml +plugins: + # - toolset # remove or comment out +``` From 2d39d7490faa0d3f2600ddcc583e70b47011d68f Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Wed, 27 May 2026 04:15:59 +0800 Subject: [PATCH 02/12] fix: handle timer creation error in toolset plugin init() The timer returned by ngx.timer.at() was not checked, causing silent failure if the timer could not be created (e.g. resource exhaustion). Apply the same error-checking pattern used in the sync() function. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apisix/plugins/toolset/init.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apisix/plugins/toolset/init.lua b/apisix/plugins/toolset/init.lua index 66efed91343e..1a111e7cc22b 100644 --- a/apisix/plugins/toolset/init.lua +++ b/apisix/plugins/toolset/init.lua @@ -126,7 +126,10 @@ function _M.init() perform_operation_for_plugin(plugin_name, plugin_config, load) end end - ngx.timer.at(1, sync) + local ok, err = ngx.timer.at(1, sync) + if not ok then + core.log.error("failed to create timer for running toolset ", err) + end end From be57db2aaca7ca38c5ad5f68a8ab2367d3ebf77a Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Wed, 27 May 2026 10:18:43 +0800 Subject: [PATCH 03/12] test: add toolset plugin tests and zh docs - t/plugin/toolset.t: sync loop and config reload tests - t/plugin/trace.t: phase timing, rate sampling, threshold tests - t/plugin/trace.host.t: host glob pattern matching tests - t/plugin/trace.path.t: path glob pattern matching tests - t/plugin/trace.headers.t: trace header detection and vars tests - t/plugin/trace.dns.t: DNS resolve phase timing test - t/plugin/table-count.t: Lua module table counting tests - t/table-count-example.lua: test fixture for table-count tests - docs/zh/latest/plugins/toolset.md: Chinese documentation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/zh/latest/config.json | 1 + docs/zh/latest/plugins/toolset.md | 159 +++++++++++ t/plugin/table-count.t | 255 +++++++++++++++++ t/plugin/toolset.t | 225 +++++++++++++++ t/plugin/trace.dns.t | 80 ++++++ t/plugin/trace.headers.t | 227 +++++++++++++++ t/plugin/trace.host.t | 223 +++++++++++++++ t/plugin/trace.path.t | 236 ++++++++++++++++ t/plugin/trace.t | 455 ++++++++++++++++++++++++++++++ t/table-count-example.lua | 55 ++++ 10 files changed, 1916 insertions(+) create mode 100644 docs/zh/latest/plugins/toolset.md create mode 100644 t/plugin/table-count.t create mode 100644 t/plugin/toolset.t create mode 100644 t/plugin/trace.dns.t create mode 100644 t/plugin/trace.headers.t create mode 100644 t/plugin/trace.host.t create mode 100644 t/plugin/trace.path.t create mode 100644 t/plugin/trace.t create mode 100644 t/table-count-example.lua diff --git a/docs/zh/latest/config.json b/docs/zh/latest/config.json index 6499144f74c1..b5d4e70e30c5 100644 --- a/docs/zh/latest/config.json +++ b/docs/zh/latest/config.json @@ -85,6 +85,7 @@ "plugins/brotli", "plugins/real-ip", "plugins/server-info", + "plugins/toolset", "plugins/ext-plugin-pre-req", "plugins/ext-plugin-post-req", "plugins/ext-plugin-post-resp", diff --git a/docs/zh/latest/plugins/toolset.md b/docs/zh/latest/plugins/toolset.md new file mode 100644 index 000000000000..1d7731773c0d --- /dev/null +++ b/docs/zh/latest/plugins/toolset.md @@ -0,0 +1,159 @@ +--- +title: toolset +keywords: + - Apache APISIX + - API 网关 + - Plugin + - Toolset + - toolset + - trace + - table-count +description: 本文介绍了关于 Apache APISIX `toolset` 插件的基本信息及使用方法。 +--- + + + +## 描述 + +`toolset` 插件是一个诊断与可观测性框架，用于托管多个轻量级子插件。每个子插件通过 `config.yaml` 中的 `plugin_attr.toolset` 进行配置，并在运行时动态加载或卸载，无需重启 APISIX。`toolset` 插件本身没有路由级别的 schema，始终在全局范围内运行。 + +### 子插件 + +| 子插件 | 描述 | +|----------------|----------------------------------------------------------------| +| `trace` | 对 APISIX 请求各阶段进行计时，并将耗时表格输出到错误日志中。 | +| `table_count` | 定期统计并记录指定 Lua 模块表中的条目数量。 | + +## 属性 + +`toolset` 插件通过 `config.yaml` 中的 `plugin_attr` 进行配置，不支持路由级别的属性。 + +### trace + +| 名称 | 类型 | 必选 | 默认值 | 描述 | +|-----------------------|---------|------|--------|----------------------------------------------------------------------------------------------------------------| +| rate | integer | 否 | 1 | 采样率，N/100 的请求会被追踪。`1` 表示每 100 个请求追踪 1 个；设置为 `100` 则追踪所有请求。 | +| hosts | array | 否 | `[]` | `Host` 请求头的白名单（支持 glob 模式）。为空表示所有 host 均通过。 | +| paths | array | 否 | `[]` | 请求 URI 的白名单（支持 glob 模式）。为空表示所有路径均通过。 | +| gen_uid | boolean | 否 | false | 为 `true` 时，当未找到标准追踪请求头（`x-request-id`、`traceparent` 等）时，自动生成 UUID 并添加到追踪输出。 | +| vars | array | 否 | `[]` | 要附加到追踪输出的额外 nginx 或 APISIX 变量名。 | +| timespan_threshold | number | 否 | 0 | 仅记录总耗时超过该值（单位：秒）的请求追踪。`0` 表示记录所有追踪。 | + +### table_count + +| 名称 | 类型 | 必选 | 默认值 | 描述 | +|--------------|---------|------|-------------------------------------|-------------------------------------------------------------------------------| +| lua_modules | array | 是 | | 需要统计的 Lua 模块路径列表（例如 `["apisix.router"]`）。 | +| interval | integer | 否 | 5 | 两次统计之间的间隔时间（单位：秒）。 | +| depth | integer | 否 | 10 | 递归统计表条目时的最大深度。`0` 表示禁用递归统计。 | +| scopes | array | 否 | `["worker", "privileged agent"]` | 子插件运行的 APISIX 进程类型。 | + +## 启用插件 + +在 `config.yaml` 的 `plugins` 列表中添加 `toolset`，并在 `plugin_attr.toolset` 下配置子插件： + +```yaml +plugins: + - toolset + +plugin_attr: + toolset: + trace: + rate: 10 + hosts: + - "*.example.com" + paths: + - "/api/*" + gen_uid: true + vars: + - remote_addr + timespan_threshold: 0.5 + table_count: + lua_modules: + - apisix.router + interval: 10 + depth: 5 + scopes: + - worker +``` + +## 使用示例 + +### 追踪慢请求 + +以下配置对发往 `*.example.com` 且总处理时间超过 500ms 的请求进行最多 10% 的采样追踪： + +```yaml +plugin_attr: + toolset: + trace: + rate: 10 + hosts: + - "*.example.com" + timespan_threshold: 0.5 +``` + +当请求满足条件时，APISIX 会以 `WARN` 级别将类似如下的耗时表格写入错误日志： + +``` ++----------+---------------------------+----------+-------------------------+ +| Role | Phase | Timespan | Start time | ++----------+---------------------------+----------+-------------------------+ +| APISIX | access | 3ms | 2024-01-01 12:00:00.123 | +| APISIX | \_match_route | 1ms | 2024-01-01 12:00:00.124 | +| APISIX | balancer | 1ms | 2024-01-01 12:00:00.125 | +| Upstream | upstream (req + response) | 520ms | 2024-01-01 12:00:00.126 | +| APISIX | header_filter | 0ms | 2024-01-01 12:00:00.646 | +| APISIX | body_filter | 0ms | 2024-01-01 12:00:00.646 | +| Client | response | 1ms | 2024-01-01 12:00:00.647 | +| APISIX | log | 0ms | 2024-01-01 12:00:00.648 | ++----------+---------------------------+----------+-------------------------+ +``` + +### 监控路由表增长 + +以下配置每 30 秒统计一次 `apisix.router` Lua 模块的条目数，仅在 worker 进程中运行： + +```yaml +plugin_attr: + toolset: + table_count: + lua_modules: + - apisix.router + interval: 30 + depth: 5 + scopes: + - worker +``` + +统计结果以 `WARN` 级别写入错误日志： + +``` +package apisix.router table count is: 1234 for loaded: 1 +``` + +## 禁用插件 + +从 `config.yaml` 的 `plugins` 列表中移除 `toolset` 并重新加载 APISIX： + +```yaml +plugins: + # - toolset # 移除或注释掉此行 +``` diff --git a/t/plugin/table-count.t b/t/plugin/table-count.t new file mode 100644 index 000000000000..1dc9aeadcac2 --- /dev/null +++ b/t/plugin/table-count.t @@ -0,0 +1,255 @@ +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + +my $user_yaml_config = <<_EOC_; +plugins: + - toolset +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: load the test table and check table count +--- extra_init_by_lua + local lfs = require("lfs") + -- Define the path to the Lua module + local module_path = "./apisix/plugins/toolset/config.lua" + + -- Function to read the contents of a file + local function read_file(path) + local file = io.open(path, "r") + if not file then return nil end + local content = file:read("*a") + file:close() + return content + end + + -- Function to write contents to a file + local function write_file(path, content) + local file = io.open(path, "w") + if not file then return nil end + file:write(content) + file:close() + end + + -- Load the module and save its contents as "old_config" + local old_config = read_file(module_path) + + -- Check if the module was read successfully + if not old_config then + error("Failed to read the module file.") + end + + -- Define the new multiline string to be inserted into the file + local new_config = +[[ +return { + table_count = {lua_modules = {"t.table-count-example"}, interval = 1} +} +]] + + -- Write the new contents to the file + write_file(module_path, new_config) + local example = require("t.table-count-example") + example.test() +--- wait: 2 +--- error_code: 404 +--- grep_error_log eval +qr/package t.table-count-example table count is: 4/ +--- grep_error_log_out +package t.table-count-example table count is: 4 +package t.table-count-example table count is: 4 + + + +=== TEST 2: load the test table and check table count for circular reference +--- extra_init_by_lua + local lfs = require("lfs") + -- Define the path to the Lua module + local module_path = "./apisix/plugins/toolset/config.lua" + + -- Function to read the contents of a file + local function read_file(path) + local file = io.open(path, "r") + if not file then return nil end + local content = file:read("*a") + file:close() + return content + end + + -- Function to write contents to a file + local function write_file(path, content) + local file = io.open(path, "w") + if not file then return nil end + file:write(content) + file:close() + end + + -- Load the module and save its contents as "old_config" + local old_config = read_file(module_path) + + -- Check if the module was read successfully + if not old_config then + error("Failed to read the module file.") + end + + -- Define the new multiline string to be inserted into the file + local new_config = +[[ +return { + table_count = {lua_modules = {"t.table-count-example"}, interval = 1,depth = 10} +} +]] + + -- Write the new contents to the file + write_file(module_path, new_config) + local example = require("t.table-count-example") + example.test_circular() +--- wait: 2 +--- error_code: 404 +--- grep_error_log eval +qr/package t.table-count-example table count is: 8/ +--- grep_error_log_out +package t.table-count-example table count is: 8 +package t.table-count-example table count is: 8 + + + +=== TEST 3: check enforced depth limit +--- extra_init_by_lua + local lfs = require("lfs") + -- Define the path to the Lua module + local module_path = "./apisix/plugins/toolset/config.lua" + + -- Function to read the contents of a file + local function read_file(path) + local file = io.open(path, "r") + if not file then return nil end + local content = file:read("*a") + file:close() + return content + end + + -- Function to write contents to a file + local function write_file(path, content) + local file = io.open(path, "w") + if not file then return nil end + file:write(content) + file:close() + end + + -- Load the module and save its contents as "old_config" + local old_config = read_file(module_path) + + -- Check if the module was read successfully + if not old_config then + error("Failed to read the module file.") + end + + -- Define the new multiline string to be inserted into the file + local new_config = +[[ +return { + table_count = {lua_modules = {"t.table-count-example"}, interval = 1} +} +]] + + -- Write the new contents to the file + write_file(module_path, new_config) + local example = require("t.table-count-example") + example.test_depth_more_than_10() +--- error_code: 404 +--- wait: 2 +--- grep_error_log eval +qr/out of depth..skipping count/ +--- grep_error_log_out +out of depth..skipping count +out of depth..skipping count + + + +=== TEST 4: reload the config with no lua modules +--- config + location /t { + content_by_lua_block { + local lfs = require("lfs") + + -- Define the path to the Lua module + local module_path = "./apisix/plugins/toolset/config.lua" + + -- Function to read the contents of a file + local function read_file(path) + local file = io.open(path, "r") + if not file then return nil end + local content = file:read("*a") + file:close() + return content + end + + -- Function to write contents to a file + local function write_file(path, content) + local file = io.open(path, "w") + if not file then return nil end + file:write(content) + file:close() + end + + -- Load the module and save its contents as "old_config" + local old_config = read_file(module_path) + + -- Check if the module was read successfully + if not old_config then + error("Failed to read the module file.") + end + + -- Define the new multiline string to be inserted into the file + local new_config = +[[ +return { + table_count = { + lua_modules = {}, -- change it + interval = 5, + depth = 10, -- when it is not passed, default depth will be 1 + -- optional, default is all APISIX processes + scopes = {"worker", "privileged agent"} + } +} + +]] + + -- Write the new contents to the file + write_file(module_path, new_config) + + ngx.sleep(2) + + -- Restore the old contents to the file + write_file(module_path, old_config) + + } + } +--- wait: 2 +--- grep_error_log eval +qr/no lua_modules provided for table count/ +--- grep_error_log_out +no lua_modules provided for table count +no lua_modules provided for table count diff --git a/t/plugin/toolset.t b/t/plugin/toolset.t new file mode 100644 index 000000000000..f98b81ac6c5f --- /dev/null +++ b/t/plugin/toolset.t @@ -0,0 +1,225 @@ +use t::APISIX 'no_plan'; +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + +my $user_yaml_config = <<_EOC_; +plugins: + - toolset +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: check if toolset plugin sync function is run every second +--- error_code: 404 +--- wait: 1 +--- grep_error_log eval +qr/syncing toolset plugin/ +--- grep_error_log_out +syncing toolset plugin +syncing toolset plugin + + + +=== TEST 2: reload with empty config for a sub-plugin(table-count) +--- config + location /t { + content_by_lua_block { + local lfs = require("lfs") + + -- Define the path to the Lua module + local module_path = "./apisix/plugins/toolset/config.lua" + + -- Function to read the contents of a file + local function read_file(path) + local file = io.open(path, "r") + if not file then return nil end + local content = file:read("*a") + file:close() + return content + end + + -- Function to write contents to a file + local function write_file(path, content) + local file = io.open(path, "w") + if not file then return nil end + file:write(content) + file:close() + end + + -- Load the module and save its contents as "old_config" + local old_config = read_file(module_path) + + -- Check if the module was read successfully + if not old_config then + error("Failed to read the module file.") + end + + -- Define the new multiline string to be inserted into the file + local new_config = +[[ +return { + table_count = {} +} +]] + + -- Write the new contents to the file + write_file(module_path, new_config) + + ngx.sleep(2) + + -- Restore the old contents to the file + write_file(module_path, old_config) + + ngx.sleep(2) + + } + } +--- grep_error_log eval +qr/empty config found for table_count.Running with default values/ +--- grep_error_log_out +empty config found for table_count.Running with default values +empty config found for table_count.Running with default values + + + +=== TEST 3: reload with different config for table-count (remove scopes) +--- config + location /t { + content_by_lua_block { + local lfs = require("lfs") + + -- Define the path to the Lua module + local module_path = "./apisix/plugins/toolset/config.lua" + + -- Function to read the contents of a file + local function read_file(path) + local file = io.open(path, "r") + if not file then return nil end + local content = file:read("*a") + file:close() + return content + end + + -- Function to write contents to a file + local function write_file(path, content) + local file = io.open(path, "w") + if not file then return nil end + file:write(content) + file:close() + end + + -- Load the module and save its contents as "old_config" + local old_config = read_file(module_path) + + -- Check if the module was read successfully + if not old_config then + error("Failed to read the module file.") + end + + -- Define the new multiline string to be inserted into the file + local new_config = +[[ +return { + table_count = { + lua_modules = { "t.table-count-example" }, -- change it + interval = 5, + depth = 10, -- when it is not passed, default depth will be 1 + -- optional, default is all APISIX processes + scopes = {} + } +} +]] + + -- Write the new contents to the file + write_file(module_path, new_config) + + ngx.sleep(2) + + -- Restore the old contents to the file + write_file(module_path, old_config) + + ngx.sleep(2) + + } + } +--- grep_error_log eval +qr/config changed. reloading plugin:/ +--- grep_error_log_out eval +qr/(?:config changed\. reloading plugin:\s*){2,}/ + + + +=== TEST 4: reload with empty toolset config +--- config + location /t { + content_by_lua_block { + local lfs = require("lfs") + + -- Define the path to the Lua module + local module_path = "./apisix/plugins/toolset/config.lua" + + -- Function to read the contents of a file + local function read_file(path) + local file = io.open(path, "r") + if not file then return nil end + local content = file:read("*a") + file:close() + return content + end + + -- Function to write contents to a file + local function write_file(path, content) + local file = io.open(path, "w") + if not file then return nil end + file:write(content) + file:close() + end + + -- Load the module and save its contents as "old_config" + local old_config = read_file(module_path) + + -- Check if the module was read successfully + if not old_config then + error("Failed to read the module file.") + end + + -- Define the new multiline string to be inserted into the file + local new_config = +[[ +]] + + -- Write the new contents to the file + write_file(module_path, new_config) + + ngx.sleep(2) + + -- Restore the old contents to the file + write_file(module_path, old_config) + + ngx.sleep(2) + + } + } +--- grep_error_log eval +qr/empty plugin config file/ +--- grep_error_log_out eval +qr/(?:empty plugin config file\s*){1,}/ diff --git a/t/plugin/trace.dns.t b/t/plugin/trace.dns.t new file mode 100644 index 000000000000..f16b416717bf --- /dev/null +++ b/t/plugin/trace.dns.t @@ -0,0 +1,80 @@ +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + +my $user_yaml_config = <<_EOC_; +plugins: + - toolset +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: create route +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/headers", + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.api7.ai:8280": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("done") + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = {} +} +]]) + file:close() + } + } +--- response_body +done + + + +=== TEST 2: test match_route +--- request +GET /headers +--- error_log eval +qr/_dns_resolve/ diff --git a/t/plugin/trace.headers.t b/t/plugin/trace.headers.t new file mode 100644 index 000000000000..9111d9bf0834 --- /dev/null +++ b/t/plugin/trace.headers.t @@ -0,0 +1,227 @@ +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + +my $user_yaml_config = <<_EOC_; +plugins: + - toolset + - serverless-post-function +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: create route with uri "/*" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/*", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("done") + + -- make all requests trace (rate = 100) + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 100, + hosts = {"*.com"} + } +} +]]) + file:close() + } + } +--- response_body +done + + + +=== TEST 2: check if observability tracing services headers take effect +--- request +GET /hello +--- more_headers +x-request-id: qewh42384238r09 +sw8: 2385248054058 +traceparent: 23852iwjefuisu489 +x-b3-traceid: oshe98ru348 +--- error_log +x-request-id: qewh42384238r09 +sw8: 2385248054058 +traceparent: 23852iwjefuisu489 +x-b3-traceid: oshe98ru348 +trace: +| Role | Phase | Timespan | Start time | + + + +=== TEST 3: trace vars +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + local httpc = http.new() + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 1, + vars = {"foo", "request_method"} + } +} +]]) + file:close() + + + ngx.sleep(2) + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local res, err = httpc:request_uri(uri, {headers = { ["foo"] = "bar" }}) + } + } +--- error_log +request_method: GET +trace: +| Role | Phase | Timespan | Start time | +--- no_error_log +foo: + + + +=== TEST 4: trace log contains uuid when no headers are found and `gen_uid = true` +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + local httpc = http.new() + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 1, + gen_uid = true + } +} +]]) + file:close() + + + ngx.sleep(2) + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local res, err = httpc:request_uri(uri) + } + } +--- error_log +uuid: +trace: +| Role | Phase | Timespan | Start time | +--- no_error_log +x-request-id: +sw8: +traceparent: +x-b3-traceid: + + + +=== TEST 5: trace doesn't contain uid if traceable headers are present +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + local httpc = http.new() + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 1, + gen_uid = true, + vars = {"uri"} + } + +} +]]) + file:close() + + + ngx.sleep(2) + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local res, err = httpc:request_uri(uri, {headers = { ["foo"] = "bar" }}) -- header foo need not be traced + } + } +--- error_log +trace: +| Role | Phase | Timespan | Start time | +--- no_error_log +x-request-id: +sw8: +traceparent: +x-b3-traceid: +uid: diff --git a/t/plugin/trace.host.t b/t/plugin/trace.host.t new file mode 100644 index 000000000000..8c454efe50ec --- /dev/null +++ b/t/plugin/trace.host.t @@ -0,0 +1,223 @@ +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + +my $user_yaml_config = <<_EOC_; +plugins: + - toolset + - serverless-post-function +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: create route with uri "/*" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/*", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("done") + } + } +--- response_body +done + + + +=== TEST 2: match against pattern "*.com" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 100, + hosts = {"*.com"} + } +} +]]) + file:close() + + + ngx.sleep(2) + + local httpc = http.new() + + -- correct path, correct host = trace + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local res, err = httpc:request_uri(uri, {headers = { ["Host"] = "abc.com" }}) + } + } +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: + + + +=== TEST 3: match against pattern "*.com" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 100, + hosts = {"*.com"}, + paths = {"/hello"} + } +} +]]) + file:close() + + + ngx.sleep(2) + + local httpc = http.new() + + -- incorrect path, incorrect host = dont_trace + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/nohello/abc" + local res, err = httpc:request_uri(uri, {headers = { ["Host"] = "abc.com.cde" }}) + } + } +--- no_error_log +trace: + + + +=== TEST 4: match against pattern "abc.*" correct path, correct host = trace +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 100, + hosts = {"abc.*"}, + paths = {"/hello"} + } +} +]]) + file:close() + + + ngx.sleep(2) + + local http = require("resty.http") + local httpc = http.new() + + -- correct path, correct host = trace + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local res, err = httpc:request_uri(uri, {headers = { ["Host"] = "abc.com" }}) + } + } +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: + + + +=== TEST 5: match against pattern "abc.*"" incorrect path, correct host = trace +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + local httpc = http.new() + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 1, + hosts = {"abc.*"}, + paths = {"/hello"} + } +} +]]) + file:close() + + + ngx.sleep(2) + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/nohello" + local res, err = httpc:request_uri(uri, {headers = { ["Host"] = "abc.com" }}) + } + } +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: diff --git a/t/plugin/trace.path.t b/t/plugin/trace.path.t new file mode 100644 index 000000000000..12296485e6cf --- /dev/null +++ b/t/plugin/trace.path.t @@ -0,0 +1,236 @@ +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + +my $user_yaml_config = <<_EOC_; +plugins: + - toolset + - serverless-post-function +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: create route with uri "/*" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/*", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("done") + } + } +--- response_body +done + + + +=== TEST 2: match against pattern "/*" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + local httpc = http.new() + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 100, + paths = {"/*"} + } +} +]]) + file:close() + + + ngx.sleep(2) + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/nohello" + local res, err = httpc:request_uri(uri) + } + } +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: + + + +=== TEST 3: match against pattern "/abc/*" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + local httpc = http.new() + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 100, + paths = {"/abc/*"} + } +} +]]) + file:close() + + + ngx.sleep(2) + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/abc/hello" + local res, err = httpc:request_uri(uri) + } + } +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: + + + +=== TEST 4: match against pattern "/abc/*/cde" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + local httpc = http.new() + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 1, + paths = {"/abc/*/cde"} + } +} +]]) + file:close() + + + ngx.sleep(2) + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/abc/foo/cde" + -- send 100 requests, 1 will match randomly + for i = 1, 100 do + local res, err = httpc:request_uri(uri) + end + + -- no match + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/abc/hello" + for i = 1, 100 do + local res, err = httpc:request_uri(uri) + end + } + } +--- timeout: 40 +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: + + + +=== TEST 5: match against pattern "/*/cde" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + local httpc = http.new() + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 1, + paths = {"/*/cde"} + } +} +]]) + file:close() + + + ngx.sleep(2) + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/foo/cde" + -- send 100 requests, 1 will match randomly + for i = 1, 100 do + local res, err = httpc:request_uri(uri) + end + + -- no match + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/abc/hello" + for i = 1, 100 do + local res, err = httpc:request_uri(uri) + end + } + } +--- timeout: 40 +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: diff --git a/t/plugin/trace.t b/t/plugin/trace.t new file mode 100644 index 000000000000..7595885bb541 --- /dev/null +++ b/t/plugin/trace.t @@ -0,0 +1,455 @@ +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +worker_connections(256); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + +my $user_yaml_config = <<_EOC_; +plugins: + - toolset + - serverless-post-function +_EOC_ + $block->set_value("extra_yaml_config", $user_yaml_config); + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: create route +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/hello", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + ngx.say(body) + return + end + ngx.say("done") + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + local old = file:read("*all") + file:write([[ +return { + trace = { + rate = 1 + } +} +]]) + file:close() + } + } +--- response_body +done + + + +=== TEST 2: test table layout +--- request +GET /hello +--- error_log +| Role | Phase | Timespan | Start time | + + + +=== TEST 3: remove plugin and send request after plugin reload +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + local conf, err = io.open("t/servroot/conf/config.yaml", "w+") + if not conf then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + + -- yaml config to remove trace plugin + local config = "deployment:\n role: traditional\n role_traditional:\n config_provider: etcd\n admin:\n admin_key: null\napisix:\n node_listen: 1984\n proxy_mode: http&stream\n stream_proxy:\n tcp:\n - 9100\n enable_resolv_search_opt: false\nplugins:\n - serverless-post-function\n" + conf:write(config) + + -- reload plugins + local code, _, org_body = t('/apisix/admin/plugins/reload', ngx.HTTP_PUT) + if code >= 300 then + ngx.status = code + ngx.say(message) + return + end + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + local res, err = httpc:request_uri(uri) + conf:close() + } + } +--- no_error_log +trace: + + + +=== TEST 4: test match_route +--- request +GET /hello +--- error_log eval +qr/\| APISIX\s{3}\| \\_match_route\s{13}\| \d+ms\s+\| \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.(\d+|000) \|/ + + + +=== TEST 5: test access +--- request +GET /hello +--- error_log eval +qr/\| APISIX\s{3}\| access\s{20}\| \d+ms\s+\| \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.(\d+|000) \|/ + + + +=== TEST 6: test balancer +--- request +GET /hello +--- error_log eval +qr/\| APISIX\s{3}\| balancer\s{18}\| \d+ms\s+\| \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.(\d+|000) \|/ + + + +=== TEST 7: test header_filter +--- request +GET /hello +--- error_log eval +qr/\| APISIX\s{3}\| header_filter\s{13}\| \d+ms\s+\| \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.(\d+|000) \|/ + + + +=== TEST 8: test body_filter +--- request +GET /hello +--- error_log eval +qr/\| APISIX\s{3}\| body_filter\s{15}\| \d+ms\s+\| \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.(\d+|000) \|/ + + + +=== TEST 9: test log +--- request +GET /hello +--- error_log eval +qr/\| APISIX\s{3}\| log\s{23}\| \d+ms\s+\| \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.(\d+|000) \|/ + + + +=== TEST 10: test upstream +--- request +GET /hello +--- error_log eval +qr/\| Upstream \| upstream \(req \+ response\) \| \d+ms\s+\| \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.(\d+|000) \|/ + + + +=== TEST 11: test client +--- request +GET /hello +--- error_log eval +qr/\| Client \| response \| \d+ms\s+\| \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.(\d+|000) \|/ + + + +=== TEST 12: test failed match route +--- request +GET /wrong_uri_hello +--- error_code: 404 +--- error_log +| Role | Phase | Timespan | Start time | +--- no_error_log +| balancer +| upstream (req + response) +| response + + + +=== TEST 13: check rate +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + + -- prepare trace config with rate = 1 + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + file:write([[ +return { + trace = { + rate = 1 + } +} +]]) + file:close() + + + ngx.sleep(2) + + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + + -- send 2 requests, since rate = 1 only first will match + local res, err = httpc:request_uri(uri) + local res, err = httpc:request_uri(uri) + } + } +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: + + + +=== TEST 14: check rate (rate = 3) +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + -- prepare config with rate = 3 + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + file:write("return { trace = {rate = 3}}") + file:close() + + + ngx.sleep(2) + + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + + -- send 100 requests, 3 will match randomly + for i = 1, 100 do + local res, err = httpc:request_uri(uri) + end + } + } +--- timeout: 20 +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: +trace: +trace: + + + +=== TEST 15: check rate: `rate = nil` should log all requests +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + -- prepare config with rate = nil + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + file:write("return { trace = {rate = nil}}") + file:close() + + + ngx.sleep(2) + + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + + -- send 5 requests + for i = 1, 5 do + local res, err = httpc:request_uri(uri) + end + } + } +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: +trace: +trace: +trace: +trace: + + + +=== TEST 16: check rate: `type(rate) ~= "number"` +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + file:write("return { trace = {rate = \"not a number\"}}") + file:close() + + + ngx.sleep(2) + + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + + -- send 5 requests + for i = 1, 5 do + local res, err = httpc:request_uri(uri) + end + } + } +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: +trace: +trace: +trace: +trace: + + + +=== TEST 17: request_uri not defined in config should not trace +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + file:write("return { trace = { paths = {\"/nohello\"}}}") + file:close() + + + ngx.sleep(2) + + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" + local httpc = http.new() + + local res, err = httpc:request_uri(uri) + } + } +--- no_error_log +trace: + + + +=== TEST 18: only request_uri defined in config should trace +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + file:write("return { trace = {paths = {\"/nohello\"}}}") + file:close() + + + ngx.sleep(2) + + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/nohello" + local httpc = http.new() + local res, err = httpc:request_uri(uri) + } + } +--- grep_error_log eval +qr/trace:/ +--- grep_error_log_out +trace: + + + +=== TEST 19: requests taking less than trace_conf.timespan_threshold should not log +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local http = require("resty.http") + + local file, err = io.open("apisix/plugins/toolset/config.lua", "w+") + if not file then + ngx.status = 500 + ngx.say("Failed test: failed to open config file") + return + end + file:write("return { trace = { timespan_threshold = 60 } }") + file:close() + + + ngx.sleep(2) + + + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/nohello" + local httpc = http.new() + local res, err = httpc:request_uri(uri) + } + } +--- no_error_log +trace: diff --git a/t/table-count-example.lua b/t/table-count-example.lua new file mode 100644 index 000000000000..7f948ff8697d --- /dev/null +++ b/t/table-count-example.lua @@ -0,0 +1,55 @@ +local core = require("apisix.core") + +local _M = {} + + +local function test_depth_more_than_10() + _M.a = { + a = { + a = { + a = { + a = { + a = { + a = { + a = { + a = { + a = { + "should not be counted" + } + } + } + } + } + } + } + } + } + } +end + +--- test function creates adds 1. +local function test() + core.table.insert(_M, "xyz") +end + +--- test_circular function creates a circular reference and adds 5 objects +local function test_circular() + local a = {} + local b = { a } + a.b = b + a.c = b + _M.a = a + _M.b = b + + _M.func = function() end + + _M.i = 1 + _M.d = { + a = a, + } +end + +_M.test = test +_M.test_circular = test_circular +_M.test_depth_more_than_10 = test_depth_more_than_10 +return _M From 13b8d31fefb4225a3792a30b29878546eb544691 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Wed, 27 May 2026 13:47:04 +0800 Subject: [PATCH 04/12] fix: add license headers, Makefile install entries, and sync race condition - Add Apache 2.0 license headers to all plugin Lua files and test files - Add toolset plugin install entries to Makefile so luarocks installs all plugin files including the src/ and src/table-count/ subdirectories - Fix race condition in sync(): check stop_timer at function entry to prevent a scheduled sync() from re-initializing sub-plugins after destroy() has already cleared the cache and set stop_timer = true Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- Makefile | 9 +++++++++ apisix/plugins/toolset/config.lua | 16 ++++++++++++++++ apisix/plugins/toolset/init.lua | 19 +++++++++++++++++++ .../plugins/toolset/src/table-count/init.lua | 16 ++++++++++++++++ apisix/plugins/toolset/src/trace.lua | 16 ++++++++++++++++ t/plugin/table-count.t | 16 ++++++++++++++++ t/plugin/toolset.t | 16 ++++++++++++++++ t/plugin/trace.dns.t | 16 ++++++++++++++++ t/plugin/trace.headers.t | 16 ++++++++++++++++ t/plugin/trace.host.t | 16 ++++++++++++++++ t/plugin/trace.path.t | 16 ++++++++++++++++ t/plugin/trace.t | 16 ++++++++++++++++ t/table-count-example.lua | 16 ++++++++++++++++ 13 files changed, 204 insertions(+) diff --git a/Makefile b/Makefile index 71ab7df1eabf..0feadb784169 100644 --- a/Makefile +++ b/Makefile @@ -347,6 +347,15 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/secret $(ENV_INSTALL) apisix/secret/*.lua $(ENV_INST_LUADIR)/apisix/secret/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/toolset + $(ENV_INSTALL) apisix/plugins/toolset/*.lua $(ENV_INST_LUADIR)/apisix/plugins/toolset/ + + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/toolset/src + $(ENV_INSTALL) apisix/plugins/toolset/src/*.lua $(ENV_INST_LUADIR)/apisix/plugins/toolset/src/ + + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/toolset/src/table-count + $(ENV_INSTALL) apisix/plugins/toolset/src/table-count/*.lua $(ENV_INST_LUADIR)/apisix/plugins/toolset/src/table-count/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/zipkin $(ENV_INSTALL) apisix/plugins/zipkin/*.lua $(ENV_INST_LUADIR)/apisix/plugins/zipkin/ diff --git a/apisix/plugins/toolset/config.lua b/apisix/plugins/toolset/config.lua index 350616386784..6a8218126d77 100644 --- a/apisix/plugins/toolset/config.lua +++ b/apisix/plugins/toolset/config.lua @@ -1,3 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- return { trace = { rate = 1, -- allow only 1 request per 100 requests diff --git a/apisix/plugins/toolset/init.lua b/apisix/plugins/toolset/init.lua index 1a111e7cc22b..b40b9d1e45f5 100644 --- a/apisix/plugins/toolset/init.lua +++ b/apisix/plugins/toolset/init.lua @@ -1,3 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- local pairs = pairs local core = require("apisix.core") local ngx = ngx @@ -73,6 +89,9 @@ end local function sync() + if stop_timer then + return + end core.log.info("syncing toolset plugin") local plugin_configs = get_plugin_config() local processed_plugins = {} diff --git a/apisix/plugins/toolset/src/table-count/init.lua b/apisix/plugins/toolset/src/table-count/init.lua index ade2f5ab824d..b50ca52ac3ac 100644 --- a/apisix/plugins/toolset/src/table-count/init.lua +++ b/apisix/plugins/toolset/src/table-count/init.lua @@ -1,3 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- local core = require("apisix.core") local ngx = require("ngx") local process = require("ngx.process") diff --git a/apisix/plugins/toolset/src/trace.lua b/apisix/plugins/toolset/src/trace.lua index 2c7f48ab689b..3a411f6e9184 100644 --- a/apisix/plugins/toolset/src/trace.lua +++ b/apisix/plugins/toolset/src/trace.lua @@ -1,3 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- local require = require local apisix = require("apisix") local core = require("apisix.core") diff --git a/t/plugin/table-count.t b/t/plugin/table-count.t index 1dc9aeadcac2..35e2481f84c5 100644 --- a/t/plugin/table-count.t +++ b/t/plugin/table-count.t @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# use t::APISIX 'no_plan'; repeat_each(1); diff --git a/t/plugin/toolset.t b/t/plugin/toolset.t index f98b81ac6c5f..98a43d832596 100644 --- a/t/plugin/toolset.t +++ b/t/plugin/toolset.t @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# use t::APISIX 'no_plan'; repeat_each(1); log_level('info'); diff --git a/t/plugin/trace.dns.t b/t/plugin/trace.dns.t index f16b416717bf..be1a7b0840b7 100644 --- a/t/plugin/trace.dns.t +++ b/t/plugin/trace.dns.t @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# use t::APISIX 'no_plan'; repeat_each(1); diff --git a/t/plugin/trace.headers.t b/t/plugin/trace.headers.t index 9111d9bf0834..87cdb12b7689 100644 --- a/t/plugin/trace.headers.t +++ b/t/plugin/trace.headers.t @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# use t::APISIX 'no_plan'; repeat_each(1); diff --git a/t/plugin/trace.host.t b/t/plugin/trace.host.t index 8c454efe50ec..6336b8ab0f52 100644 --- a/t/plugin/trace.host.t +++ b/t/plugin/trace.host.t @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# use t::APISIX 'no_plan'; repeat_each(1); diff --git a/t/plugin/trace.path.t b/t/plugin/trace.path.t index 12296485e6cf..42fa1bb948b2 100644 --- a/t/plugin/trace.path.t +++ b/t/plugin/trace.path.t @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# use t::APISIX 'no_plan'; repeat_each(1); diff --git a/t/plugin/trace.t b/t/plugin/trace.t index 7595885bb541..c1452ac3130f 100644 --- a/t/plugin/trace.t +++ b/t/plugin/trace.t @@ -1,3 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# use t::APISIX 'no_plan'; repeat_each(1); diff --git a/t/table-count-example.lua b/t/table-count-example.lua index 7f948ff8697d..28603c230a09 100644 --- a/t/table-count-example.lua +++ b/t/table-count-example.lua @@ -1,3 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- local core = require("apisix.core") local _M = {} From 8d82723e4ba5c160b188b4542a0f462dc74c5441 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Wed, 27 May 2026 14:41:18 +0800 Subject: [PATCH 05/12] fix(toolset): add sleep after plugin reload in TEST 3 to avoid race condition plugins/reload uses events:post() which is asynchronous - it returns 200 before destroy() is called. Without a sleep, the /hello request can arrive before trace.destroy() restores the phase hooks, causing spurious trace: logs. Use ngx.sleep(2) consistent with other reload tests in the test suite. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- t/plugin/trace.t | 3 +++ 1 file changed, 3 insertions(+) diff --git a/t/plugin/trace.t b/t/plugin/trace.t index c1452ac3130f..79987531a048 100644 --- a/t/plugin/trace.t +++ b/t/plugin/trace.t @@ -127,6 +127,9 @@ GET /hello return end + -- wait for the reload event to propagate and destroy() to be called + ngx.sleep(2) + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello" local httpc = http.new() local res, err = httpc:request_uri(uri) From 5537e2f6036abc908af531c89740eae0c9b5ca82 Mon Sep 17 00:00:00 2001 From: AlinsRan Date: Wed, 27 May 2026 15:29:10 +0800 Subject: [PATCH 06/12] fix(toolset): address code review issues - trace: escape regex metacharacters in glob patterns before * substitution - trace: replace unique_random() pool-depletion logic with math.random(100) - trace: fix package.loaded reset from false to nil to allow require() reload - trace: return match_route result to preserve router semantics - trace: restore dns.resolve in destroy() to prevent stacking on reload - toolset: change sync log from info to debug to avoid log flooding - table-count: reset stop=false in init() so plugin can be re-enabled - table-count: fix depth default from 1 to 10 to match config/docs - test: fix undefined 'message' variable in trace.t TEST 3 - test: remove dead read-then-overwrite pattern in trace.t TEST 1 - test: fix comment typo in table-count-example.lua - docs: add canonical link to EN/ZH plugin documentation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- apisix/plugins/toolset/init.lua | 10 +---- .../plugins/toolset/src/table-count/init.lua | 3 +- apisix/plugins/toolset/src/trace.lua | 39 ++++++------------- docs/en/latest/plugins/toolset.md | 4 ++ docs/zh/latest/plugins/toolset.md | 4 ++ t/plugin/trace.t | 5 +-- t/table-count-example.lua | 2 +- 7 files changed, 27 insertions(+), 40 deletions(-) diff --git a/apisix/plugins/toolset/init.lua b/apisix/plugins/toolset/init.lua index b40b9d1e45f5..c56d47603a48 100644 --- a/apisix/plugins/toolset/init.lua +++ b/apisix/plugins/toolset/init.lua @@ -89,10 +89,7 @@ end local function sync() - if stop_timer then - return - end - core.log.info("syncing toolset plugin") + core.log.debug("syncing toolset plugin") local plugin_configs = get_plugin_config() local processed_plugins = {} if plugin_configs then @@ -145,10 +142,7 @@ function _M.init() perform_operation_for_plugin(plugin_name, plugin_config, load) end end - local ok, err = ngx.timer.at(1, sync) - if not ok then - core.log.error("failed to create timer for running toolset ", err) - end + ngx.timer.at(1, sync) end diff --git a/apisix/plugins/toolset/src/table-count/init.lua b/apisix/plugins/toolset/src/table-count/init.lua index b50ca52ac3ac..35e71a2bc201 100644 --- a/apisix/plugins/toolset/src/table-count/init.lua +++ b/apisix/plugins/toolset/src/table-count/init.lua @@ -69,6 +69,7 @@ local function tab_item_count(tab, cache,depth) end function _M.init() + stop = false package.loaded["apisix.plugins.toolset.config"] = nil local config = require("apisix.plugins.toolset.config").table_count if config.lua_modules == nil or #config.lua_modules == 0 then @@ -93,7 +94,7 @@ function _M.init() local interval = config.interval or 5 local run_count run_count = function(run_no) - local depth = config.depth or 1 + local depth = config.depth or 10 for _, package_name in ipairs(config.lua_modules) do local package = require(package_name) local count = tab_item_count(package, {},depth) diff --git a/apisix/plugins/toolset/src/trace.lua b/apisix/plugins/toolset/src/trace.lua index 3a411f6e9184..db726298787e 100644 --- a/apisix/plugins/toolset/src/trace.lua +++ b/apisix/plugins/toolset/src/trace.lua @@ -30,9 +30,8 @@ local format = string.format local floor = math.floor local gsub = ngx.re.gsub local m_random = math.random -local m_randomseed = math.randomseed -local t_remove = table.remove local re_match = ngx.re.match + local counter = 1 local old_http_access_phase @@ -123,9 +122,10 @@ end local function match(incoming, conf) + -- escape regex metacharacters before glob substitution + conf = gsub(conf, [[([.+?[\](){}^$|])]], [[\$1]]) conf = gsub(conf, "\\*", ".*") - conf = "^" .. conf .. "$" - core.log.info("matching: ", incoming, " against: ", conf) + core.log.info("matching: ", incoming, " against: ^", conf, "$") local matches = re_match(incoming, "^" .. conf .. "$", "jo") if not matches then @@ -135,26 +135,6 @@ local function match(incoming, conf) end -local unique_random -do - local numbers = {} - for i = 1, 100 do - numbers[i] = i - end - unique_random = function() - m_randomseed(ngx.now()) - while true do - local index = m_random(100) - local num = numbers[index] - if num then - t_remove(numbers, index) - return num - end - end - end -end - - local function incr_counter() counter = counter + 1 if counter > 99 then @@ -174,7 +154,7 @@ local function preprocess(trace_conf, ctx) return end core.log.info("trace_conf.rate: ", trace_conf.rate) - local rand = unique_random() + local rand = m_random(100) if rand <= trace_conf.rate then ctx.trace = true end @@ -246,7 +226,7 @@ end function _M.init() - package.loaded[conf_path] = false + package.loaded[conf_path] = nil local trace_conf = require(conf_path).trace core.log.info("trace_conf: ", core.json.encode(trace_conf)) @@ -281,7 +261,7 @@ function _M.init() local match_start = ngx.now() ngx.ctx.match_lt = localtime_msec(match_start) - old_match_route(...) + return old_match_route(...) ngx.update_time() ngx.ctx.match_timespan = ngx.now() - match_start @@ -451,6 +431,11 @@ function _M.destroy() local router = require("apisix.http.router." .. router_name) router.match = old_match_route + local dns = require("apisix.core.dns.client") + if dns and old_resolve then + dns.resolve = old_resolve + end + apisix.http_access_phase = old_http_access_phase apisix.http_balancer_phase = old_http_balancer_phase apisix.http_header_filter_phase = old_http_header_filter_phase diff --git a/docs/en/latest/plugins/toolset.md b/docs/en/latest/plugins/toolset.md index 940b57f91b15..5893147321ff 100644 --- a/docs/en/latest/plugins/toolset.md +++ b/docs/en/latest/plugins/toolset.md @@ -11,6 +11,10 @@ keywords: description: This document contains information about the Apache APISIX toolset Plugin. --- + + + +