feat: add toolset plugin

Makefile

@@ -347,6 +347,15 @@ install: runtime
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/secret
 	$(ENV_INSTALL) apisix/secret/*.lua $(ENV_INST_LUADIR)/apisix/secret/
 
+	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/toolset
+	$(ENV_INSTALL) apisix/plugins/toolset/*.lua $(ENV_INST_LUADIR)/apisix/plugins/toolset/
+
+	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/toolset/src
+	$(ENV_INSTALL) apisix/plugins/toolset/src/*.lua $(ENV_INST_LUADIR)/apisix/plugins/toolset/src/
+
+	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/toolset/src/table-count
+	$(ENV_INSTALL) apisix/plugins/toolset/src/table-count/*.lua $(ENV_INST_LUADIR)/apisix/plugins/toolset/src/table-count/
+
 	$(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/zipkin
 	$(ENV_INSTALL) apisix/plugins/zipkin/*.lua $(ENV_INST_LUADIR)/apisix/plugins/zipkin/
 

apisix/cli/config.lua

@@ -340,6 +340,22 @@ local _M = {
     ["server-info"] = {
       report_ttl = 60
     },
+    toolset = {
+      trace = {
+        rate = 1,
+        hosts = {},
+        paths = {},
+        gen_uid = false,
+        vars = {},
+        timespan_threshold = 0
+      },
+      table_count = {
+        lua_modules = {},
+        interval = 5,
+        depth = 10,
+        scopes = {"worker", "privileged agent"}
+      }
+    },
     ["dubbo-proxy"] = {
       upstream_multiplex_count = 32
     },

apisix/plugins/toolset/config.lua

@@ -0,0 +1,33 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+return {
+    trace = {
+        rate = 1, -- allow only 1 request per 100 requests
+        hosts = {}, -- only the requests carrying these host headers will be traced
+        paths = {}, -- only these request_uris will be traced
+        gen_uid = false, -- adds a UID to the trace if none of the traceable headers are found
+        vars = {}, -- add these nginx or inbuilt variables to trace table
+        timespan_threshold = 0 -- requests taking longer than this value (in seconds) will be traced
+    },
+    table_count = {
+        lua_modules = {}, -- change it
+        interval = 5,
+        depth = 10, -- when it is not passed, default depth will be 1
+        -- optional, default is all APISIX processes
+        scopes = {"worker", "privileged agent"}
+    }
+}

apisix/plugins/toolset/init.lua

@@ -0,0 +1,170 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local pairs = pairs
+local core = require("apisix.core")
+local ngx = ngx
+local cache = core.table.new(0, 32)
+local stop_timer = false
+local load, unload = "load", "unload"
+local package = package
+local pcall = pcall
+local require = require
+local string = string
+
+local _M = {
+  version = 0.1,
+  priority = 22901,
+  name = "toolset",
+  schema = {},
+  scope = "global",
+}
+
+
+local function get_plugin_config()
+  -- clear cache to reload
+  package.loaded["apisix.plugins.toolset.config"] = nil
+  local loaded, plugins_config = pcall(require, "apisix.plugins.toolset.config")
+  if loaded and plugins_config == true then
+    core.log.warn("empty plugin config file")
+    return nil
+  end
+  if not loaded then
+    core.log.error("failed to load plugin config: ", plugins_config)
+    return nil
+  end
+  return plugins_config
+end
+
+
+local function is_config_changed(plugin_name, plugin_config)
+  if core.table.deep_eq(cache[plugin_name], plugin_config) then
+    return false
+  end
+  return true
+end
+
+
+local function is_config_empty(plugin_config)
+  return plugin_config == nil or core.table.deep_eq(plugin_config, {})
+end
+
+
+local function perform_operation_for_plugin(plugin_name, plugin_config, operation)
+  if operation == load then
+    local loaded, plugin = pcall(require, "apisix.plugins.toolset.src."
+                           .. string.gsub(plugin_name, "_", "-"))
+    if not loaded then
+      core.log.warn("could not load plugin because it was not found: ", plugin_name)
+      return
+    end
+    core.log.warn("initializing sub plugin for toolset plugin: ", plugin_name)
+    plugin.init()
+    cache[plugin_name] = plugin_config
+  elseif operation == unload then
+    local loaded, plugin = pcall(require, "apisix.plugins.toolset.src." ..
+                                 string.gsub(plugin_name, "_", "-"))
+    if not loaded then
+      core.log.warn("could not unload plugin because it was not found: ", plugin_name)
+      return
+    end
+    core.log.warn("destroying sub plugin for toolset plugin: ", plugin_name)
+    plugin.destroy()
+    cache[plugin_name] = nil
+  end
+end
+
+
+local function sync()
+  if stop_timer then
+    return
+  end
+  core.log.info("syncing toolset plugin")
+  local plugin_configs = get_plugin_config()
+  local processed_plugins = {}
+  if plugin_configs then
+    for plugin_name, plugin_config in pairs(plugin_configs) do
+      processed_plugins[plugin_name] = true
+      -- checks if the config is different from cache
+      if is_config_changed(plugin_name, plugin_config) then
+          if is_config_empty(plugin_config) then
+            -- allow executing even with empty config.
+            -- Assuming the plugin will run with default values
+            core.log.warn("empty config found for ", plugin_name,".Running with default values")
+          end
+          core.log.warn("config changed. reloading plugin: ", plugin_name)
+          local ok, err = pcall(perform_operation_for_plugin, plugin_name, plugin_config, load)
+          if not ok then
+            core.log.error("toolset plugin load raised: ", err)
+          end
+      end
+    end
+  end
+
+  for plugin_name, plugin_config in pairs(cache) do
+    if not processed_plugins[plugin_name] then
+      core.log.warn("plugin config unloaded: ", plugin_name)
+      local ok, err = pcall(perform_operation_for_plugin, plugin_name, plugin_config, unload)
+      if not ok then
+        core.log.error("toolset plugin unload raised: ", err)
+      end
+    end
+  end
+  if not stop_timer then
+    local ok, err = ngx.timer.at(1, sync)
+    if not ok then
+      core.log.error("failed to create timer for running toolset ", err)
+    end
+  end
+end
+
+
+function _M.init()
+    core.log.info("initializing toolset plugin")
+    local plugins_config = get_plugin_config()
+    if plugins_config then
+      for plugin_name, plugin_config in pairs(plugins_config) do
+        if is_config_empty(plugin_config) then
+          -- allow executing even with empty config.
+          -- Assuming the plugin will run with default values
+          core.log.warn("empty config found for ", plugin_name,".Running with default values")
+        end
+        perform_operation_for_plugin(plugin_name, plugin_config, load)
+      end
+    end
+    local ok, err = ngx.timer.at(1, sync)
+    if not ok then
+      core.log.error("failed to create timer for running toolset ", err)
+    end
+end
+
+
+function _M.destroy()
+  local plugin_configs = get_plugin_config()
+  if plugin_configs then
+    for plugin_name, plugin_config in pairs(plugin_configs) do
+      perform_operation_for_plugin(plugin_name, plugin_config, unload)
+    end
+
+  end
+  for plugin_name, plugin_config in pairs(cache) do
+    perform_operation_for_plugin(plugin_name, plugin_config, unload)
+  end
+
+  stop_timer = true
+end
+
+return _M

apisix/plugins/toolset/src/table-count/init.lua

@@ -0,0 +1,121 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements.  See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License.  You may obtain a copy of the License at
+--
+--     http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+local core = require("apisix.core")
+local ngx = require("ngx")
+local process = require("ngx.process")
+
+local pairs = pairs
+local ipairs = ipairs
+local type = type
+local timer = ngx.timer
+local require = require
+local package = package
+
+local plugin_name = "table-count"
+
+local schema = {}
+local stop = false
+-- only one run of init() function should be running at a time.
+-- when init() is reloaded the run number is incremented. It also helps in debugging.
+local current_run = 0
+
+local _M = {
+  version = 0.1,
+  priority = 22902,
+  name = plugin_name,
+  schema = schema,
+  scope = "global",
+}
+
+local function tab_item_count(tab, cache,depth)
+  if depth == 0 then
+    core.log.warn("out of depth..skipping count")
+    return
+  end
+  depth = depth - 1
+  cache = cache or {}
+  local count = 0
+  for _, value in pairs(tab) do
+    if cache[value] then
+      core.log.warn("circular reference detected..skipping count")
+      goto continue
+    end
+    if type(value) == "table" and not cache[value] then
+      cache[value] = true
+      local tab_count = tab_item_count(value, cache,depth)
+      if tab_count then
+        count = count + tab_count + 1
+      end
+    else
+      count = count + 1
+    end
+    ::continue::
+  end
+  return count
+end
+
+function _M.init()
+  package.loaded["apisix.plugins.toolset.config"] = nil
+  local config = require("apisix.plugins.toolset.config").table_count
+  if config.lua_modules == nil or #config.lua_modules == 0 then
+    core.log.warn("no lua_modules provided for table count")
+    return
+  end
+  if not config.scopes then
+    core.log.warn("no scope provided. Running for all scopes")
+    goto continue
+  end
+  if #config.scopes ~= 0 then
+    for _,scope in ipairs(config.scopes) do
+      if process.type() == scope then
+        goto continue
+      end
+    end
+    return
+  end
+  ::continue::
+  -- Extract configuration values
+  current_run = current_run + 1
+  local interval = config.interval or 5
+  local run_count
+  run_count = function(run_no)
+    local depth = config.depth or 1
+    for _, package_name in ipairs(config.lua_modules) do
+      local package = require(package_name)
+      local count = tab_item_count(package, {},depth)
+      core.log.warn("package ", package_name, " table count is: ", count," for loaded: ",run_no)
+    end
+    if stop or run_no ~= current_run then
+      return
+    end
+    local ok, err = timer.at(interval, run_count,current_run)
+    if not ok then
+      core.log.error("failed to create timer for running table count ", err)
+    end
+  end
+
+  local ok, err = timer.at(0, run_count,current_run)
+  if not ok then
+    core.log.error("failed to create timer for running table count ", err)
+  end
+end
+
+function _M.destroy()
+  stop = true
+end
+
+return _M