Removed checks plugins concept from ntopng

This commit is contained in:
Matteo Biscosi 2022-02-08 18:11:40 +01:00
parent 616293aaf1
commit 1b3a1c97c8
5 changed files with 10 additions and 665 deletions

View file

@ -23,15 +23,8 @@ local do_trace = false
-- How deep the recursive plugins search should go into subdirectories
local MAX_RECURSION = 2
plugins_utils.COMMUNITY_SOURCE_DIR = os_utils.fixPath(dirs.scriptdir .. "/plugins")
plugins_utils.PRO_SOURCE_DIR = os_utils.fixPath(dirs.installdir .. "/pro/scripts/pro_plugins")
plugins_utils.ENTERPRISE_M_SOURCE_DIR = os_utils.fixPath(dirs.installdir .. "/pro/scripts/enterprise_m_plugins")
plugins_utils.ENTERPRISE_L_SOURCE_DIR = os_utils.fixPath(dirs.installdir .. "/pro/scripts/enterprise_l_plugins")
local PLUGIN_RELATIVE_PATHS = {
menu_items = "menu_items",
metadata = "plugins_metadata",
modules = "modules",
menu_items = "menu_items",
}
local RUNTIME_PATHS = {}
local METADATA = nil
@ -39,7 +32,7 @@ local METADATA = nil
-- ##############################################
-- The runtime path can change when the user reloads the plugins.
-- We need to cache this into the same vm to ensure that all the lua
-- We need to cache this into the same vm to ensure that all thea
-- scripts into this vm use the same directory.
local cached_runtime_dir = nil
@ -51,147 +44,6 @@ function plugins_utils.getRuntimePath()
return(cached_runtime_dir)
end
local function getMetadataPath()
return(os_utils.fixPath(plugins_utils.getRuntimePath() .. "/"..PLUGIN_RELATIVE_PATHS.metadata..".lua"))
end
-- ##############################################
local function clearInternalState()
RUNTIME_PATHS = {}
METADATA = nil
cached_runtime_dir = nil
-- Tell lua to forget about require-d metadata. This is necessary as plugins may have been swapped betwenn plugins0/ and plugins1/.
-- However, as PLUGIN_RELATIVE_PATHS.metadata is the same, lua would not reload it unless it's entry in package.loaded is reset.
package.loaded[PLUGIN_RELATIVE_PATHS.metadata] = nil
end
-- ##############################################
-- @brief Recursively search for plugins starting from `source_dir`
-- @param edition A string indicating the plugin edition. One of `community`, `pro`, `enterprise_m` or `enterprise_l`
-- @param source_dir The path of the directory to start the plugin search from
-- @param max_recursion Maximum number of recursive calls to this function
-- @param plugins A lua table with all the plugins found
-- @param plugins_with_deps A lua table with all the plugins found which have other plugins as dependencies
local function recursivePluginsSearch(edition, source_dir, max_recursion, plugins, plugins_with_deps)
-- Prepend the current `source_dir` to the Lua path - this is necessary for doing the require
lua_path_utils.package_path_prepend(source_dir)
local source_dir_contents = ntop.readdir(source_dir)
for plugin_name in pairs(source_dir_contents) do
local plugin_dir = os_utils.fixPath(source_dir .. "/" .. plugin_name)
local plugin_info = os_utils.fixPath(plugin_dir .. "/manifest.lua")
if ntop.exists(plugin_info) then
-- If there's a manifest, we are in a plugin directory
local req_name = string.format("%s.manifest", plugin_name)
local metadata = require(req_name)
local mandatory_fields = {"title", "description", "author"}
if not metadata then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Could not load manifest.lua in '%s'", plugin_name))
goto continue
end
for _, field in pairs(mandatory_fields) do
if not metadata[field] then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Missing mandatory field '%s' in manifest.lua of '%s'", field, plugin_name))
goto continue
end
end
if metadata.disabled then
-- The plugin is disabled, skip it
goto continue
end
-- Augument information
metadata.path = plugin_dir
metadata.key = plugin_name
metadata.edition = edition
if not table.empty(metadata.dependencies) then
plugins_with_deps[plugin_name] = metadata
else
plugins[plugin_name] = metadata
end
elseif ntop.isdir(plugin_dir) then
if max_recursion > 0 then
-- Recursively see if this is a directory containing other plugins
recursivePluginsSearch(edition, plugin_dir, max_recursion - 1, plugins, plugins_with_deps)
else
-- Maximum recursion hit. must stop
traceError(TRACE_INFO, TRACE_CONSOLE, string.format("Unable to load '%s'. Too many recursion levels.", plugin_dir))
end
end
::continue::
end
end
-- ##############################################
-- @brief Lists the all available plugins
-- @returns a sorted table with plugins as values.
-- @notes Plugins must be loaded based according to the sort order to honor dependencies
local function listPlugins(community_plugins_only)
local plugins = {}
local plugins_with_deps = {}
local rv = {}
local source_dirs = {{"community", plugins_utils.COMMUNITY_SOURCE_DIR}}
if not community_plugins_only and ntop.isPro() then
source_dirs[#source_dirs + 1] = {"pro", plugins_utils.PRO_SOURCE_DIR}
if ntop.isEnterpriseM() then
source_dirs[#source_dirs + 1] = {"enterprise_m", plugins_utils.ENTERPRISE_M_SOURCE_DIR}
end
if ntop.isEnterpriseL() then
source_dirs[#source_dirs + 1] = {"enterprise_l", plugins_utils.ENTERPRISE_L_SOURCE_DIR}
end
end
for _, source_conf in ipairs(source_dirs) do
local edition = source_conf[1]
local source_dir = source_conf[2]
recursivePluginsSearch(edition, source_dir, MAX_RECURSION, plugins, plugins_with_deps)
end
-- Add plugins without dependencies to the result
for _, plugin_metadata in pairs(plugins) do
rv[#rv + 1] = plugin_metadata
end
-- Check basic dependencies.
-- No recursion is supported (e.g. dependency on a plugin which has dependencies itself)
for plugin_name, metadata in pairs(plugins_with_deps) do
local satisfied = true
for _, dep_name in pairs(metadata.dependencies) do
if not plugins[dep_name] then
satisfied = false
if do_trace then
io.write(string.format("Skipping plugin '%s' with unmet depedendency ('%s')\n", plugin_name, dep_name))
end
break
end
end
if satisfied then
plugins[plugin_name] = metadata
rv[#rv + 1] = metadata
end
end
return(rv)
end
-- ##############################################
local function init_runtime_paths()
@ -294,377 +146,26 @@ end
-- ##############################################
local function load_plugin_ts_schemas(plugin)
local src_path = os_utils.fixPath(plugin.path .. "/ts_schemas")
local ts_path = os_utils.fixPath(RUNTIME_PATHS.ts_schemas .. "/" .. plugin.key)
if ntop.exists(src_path) then
ntop.mkdir(ts_path)
return(
file_utils.recursive_copy(src_path, ts_path)
)
end
return(true)
end
-- ##############################################
local function load_plugin_i18n(locales, default_locale, plugin)
local locales_dir = os_utils.fixPath(plugin.path .. "/locales")
local locales_path = ntop.readdir(locales_dir)
if table.empty(locales_path) then
return(true)
end
-- Ensure that the plugin localization will not override any existing
-- key
if default_locale[plugin.key] then
traceError(TRACE_WARNING, TRACE_CONSOLE, string.format(
"Plugin name %s overlaps with an existing i18n key. Please rename the plugin.", plugin.key))
return(false)
end
for fname in pairs(locales_path) do
if string.ends(fname, ".lua") then
local full_path = os_utils.fixPath(locales_dir .. "/" .. fname)
local locale = persistence.load(full_path)
if locale then
locales[fname] = locales[fname] or {}
locales[fname][plugin.key] = locale
if do_trace then
io.write("\ti18n: " .. fname .. "\n")
end
else
return(false)
end
end
end
return(true)
end
-- ##############################################
local function load_plugin_lint(plugin)
local lint_path = os_utils.fixPath(plugin.path .. "/http_lint.lua")
if(ntop.exists(lint_path)) then
if(not file_utils.copy_file(nil, lint_path,
os_utils.fixPath(RUNTIME_PATHS.http_lint .. "/" .. plugin.key .. ".lua"))) then
return(false)
end
end
return(true)
end
-- ##############################################
local function load_plugin_checks(paths_to_plugin, plugin)
local scripts_path = os_utils.fixPath(plugin.path .. "/checks")
local paths_map = {}
local extn = ".lua"
local rv = (
file_utils.recursive_copy(os_utils.fixPath(scripts_path .. "/interface"), RUNTIME_PATHS.interface_scripts, paths_map, extn) and
file_utils.recursive_copy(os_utils.fixPath(scripts_path .. "/host"), RUNTIME_PATHS.host_scripts, paths_map, extn) and
file_utils.recursive_copy(os_utils.fixPath(scripts_path .. "/network"), RUNTIME_PATHS.network_scripts, paths_map, extn) and
file_utils.recursive_copy(os_utils.fixPath(scripts_path .. "/flow"), RUNTIME_PATHS.flow_scripts, paths_map, extn) and
file_utils.recursive_copy(os_utils.fixPath(scripts_path .. "/syslog"), RUNTIME_PATHS.syslog, paths_map, extn) and
file_utils.recursive_copy(os_utils.fixPath(scripts_path .. "/snmp_device"), RUNTIME_PATHS.snmp_scripts, paths_map, extn) and
file_utils.recursive_copy(os_utils.fixPath(scripts_path .. "/system"), RUNTIME_PATHS.system_scripts, paths_map, extn)
)
for runtime_path, source_path in pairs(paths_map) do
-- Ensure that the script does not have errors
local res = load_plugin_file(runtime_path)
if(res == nil) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Skipping bad user script '%s' in plugin '%s'", source_path, plugin.key))
os.remove(runtime_path)
else
paths_to_plugin[runtime_path] = {
source_path = source_path,
plugin = plugin,
}
end
end
return(rv)
end
-- ##############################################
local function load_plugin_alert_endpoints(plugin)
local endpoints_path = os_utils.fixPath(plugin.path .. "/alert_endpoints")
local endpoints_template_path = os_utils.fixPath(plugin.path .. "/templates")
if not ntop.exists(endpoints_path) then
-- No alert endpoints for this plugin
return true
end
for fname in pairs(ntop.readdir(endpoints_path)) do
if fname:ends(".lua") then
-- Execute the alert endpoint and call its method onLoad, if present
local fname_path = os_utils.fixPath(endpoints_path .. "/" .. fname)
local endpoint = load_plugin_file(fname_path)
if not endpoint then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Unable to load endpoint '%s'", fname))
return false
end
-- Check for configuration templates existence
if endpoint.endpoint_template and endpoint.endpoint_template.template_name then
-- Stop if the template doesn't exist
if not ntop.exists(os_utils.fixPath(endpoints_template_path.."/"..endpoint.endpoint_template.template_name)) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Missing conf template '%s' in '%s' for endpoint '%s'",
endpoint.endpoint_template.template_name,
endpoints_template_path,
fname))
return false
end
end
-- Check for recipient templates existence
if endpoint.recipient_template and endpoint.recipient_template.template_name then
-- Return if the recipient template doesn't exist
if not ntop.exists(os_utils.fixPath(endpoints_template_path.."/"..endpoint.recipient_template.template_name)) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Missing recipient template '%s' in '%s' for endpoint '%s'",
endpoint.recipient_template.template_name,
endpoints_template_path,
fname))
return false
end
end
if not file_utils.copy_file(fname, endpoints_path, RUNTIME_PATHS.alert_endpoints) then
return false
end
if endpoint and endpoint.onLoad then
endpoint.onLoad()
end
end
end
return true
end
-- ##############################################
local function load_plugin_web_gui(plugin)
local gui_dir = os_utils.fixPath(plugin.path .. "/web_gui")
for fname in pairs(ntop.readdir(gui_dir)) do
if(fname == "menu.lua") then
local full_path = os_utils.fixPath(gui_dir .. "/" .. fname)
local menu_entry = load_plugin_file(full_path)
if(menu_entry) then
if(menu_entry.label == nil) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Missing menu entry 'label' in %s (menu.lua)", plugin.key))
return(false)
elseif(menu_entry.script == nil) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Missing menu entry 'script' in %s (menu.lua)", plugin.key))
return(false)
else
-- Check that the menu entry exists
local script_path = os_utils.fixPath(gui_dir .. "/" .. menu_entry.script)
if(not ntop.exists(script_path)) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Menu entry script path '%s' does not exists in %s", script_path, plugin.key))
return(false)
end
if(not file_utils.copy_file(nil, full_path,
os_utils.fixPath(RUNTIME_PATHS.menu_items .. "/" .. plugin.key .. ".lua"))) then
return(false)
end
end
end
else
if not file_utils.copy_file(fname, gui_dir, RUNTIME_PATHS.web_gui) then
return(false)
end
end
end
return(true)
end
-- ##############################################
-- A plugin can specify additional directories to load with the "data_dirs"
-- field in its manifest.lua . The plugin can then retrieve the runtime path
-- by using the plugins_utils.getPluginDataDir() api
local function load_plugin_data_dirs(plugin)
for _, dir in pairs(plugin.data_dirs or {}) do
local data_dir = os_utils.fixPath(plugin.path .. "/" .. dir)
if ntop.exists(data_dir) then
local dest_path = os_utils.fixPath(RUNTIME_PATHS.plugins_data .. "/" .. plugin.key .. "/" .. dir)
ntop.mkdir(dest_path)
file_utils.recursive_copy(data_dir, dest_path)
end
end
return(true)
end
-- ##############################################
local function load_plugin_other(plugin)
local templates_dir = os_utils.fixPath(plugin.path .. "/templates")
local modules_dir = os_utils.fixPath(plugin.path .. "/modules")
local httpdocs_dir = os_utils.fixPath(plugin.path .. "/httpdocs")
local rv = true
if ntop.exists(templates_dir) then
local path = plugins_utils.getPluginTemplatesDir(plugin.key)
ntop.mkdir(path)
rv = rv and file_utils.recursive_copy(templates_dir, path)
end
if ntop.exists(modules_dir) then
local path = os_utils.fixPath(RUNTIME_PATHS.modules.. "/" ..plugin.key)
ntop.mkdir(path)
rv = rv and file_utils.recursive_copy(modules_dir, path)
end
if ntop.exists(httpdocs_dir) then
local path = os_utils.fixPath(RUNTIME_PATHS.httpdocs.. "/" ..plugin.key)
ntop.mkdir(path)
rv = rv and file_utils.recursive_copy(httpdocs_dir, path)
end
return(rv)
end
-- ##############################################
-- @brief Loads the ntopng plugins into a single directory tree.
-- @notes This should be called at startup. It clears and populates the
-- shadow_dir first, then swaps it with the current_dir. This prevents
-- other threads to see intermediate states and half-populated directories.
function plugins_utils.loadPlugins(community_plugins_only)
local locales_utils = require("locales_utils")
local plugins = listPlugins(community_plugins_only)
local loaded_plugins = {}
local locales = {}
function plugins_utils.loadPlugins()
local path_map = {}
local en_locale = locales_utils.readDefaultLocale()
local current_dir = ntop.getCurrentPluginsDir()
local shadow_dir = ntop.getShadowPluginsDir()
-- Clean up the shadow directory
ntop.rmdir(shadow_dir)
-- Use the shadow directory as the new base
clearInternalState()
cached_runtime_dir = shadow_dir
-- Init runtime Path
init_runtime_paths()
-- Ensure that the directory is writable
ntop.mkdir(shadow_dir)
local test_file = os_utils.fixPath(shadow_dir .. "/test")
local outfile, err = io.open(test_file, "w")
if(outfile) then
outfile:close()
end
if(outfile == nil) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Cannot write to the plugins directory: %s. Plugins will not be loaded!",
err or shadow_dir))
clearInternalState()
return(false)
end
os.remove(test_file)
for _, path in pairs(RUNTIME_PATHS) do
ntop.mkdir(path)
end
-- Load plugin alert definitions, i.e., definitions found under <plugin_name>/alert_definitions
-- alert definitions MUST be loaded before flow status definitions as, flow status definitions,
-- may depend on alert definitions
for _, plugin in ipairs(plugins) do
load_plugin_definitions(plugin)
end
-- Make sure to invalidate the (possibly) already required alert_consts which depends on alert definitions.
-- By invalidating the module, we make sure all the newly loaded alert definitions will be picked up by any
-- subsequent `require "alert_consts"`
package.loaded["alert_consts"] = nil
-- Load the plugins following the dependecies order
for _, plugin in ipairs(plugins) do
if community_plugins_only and plugin.edition ~= "community" then
goto continue
end
-- Ensure that the depencies has been loaded as well
for _, dep in pairs(plugin.dependencies or {}) do
if not loaded_plugins[dep] then
traceError(TRACE_WARNING, TRACE_CONSOLE, string.format("Skipping plugin %s due to missing dependency '%s'", plugin.key, dep))
goto continue
end
end
if do_trace then
io.write(string.format("Loading plugin %s [edition: %s]\n", plugin.key, plugin.edition))
end
if load_plugin_i18n(locales, en_locale, plugin) and
load_plugin_lint(plugin) and
load_plugin_ts_schemas(plugin) and
load_plugin_web_gui(plugin) and
load_plugin_data_dirs(plugin) and
load_plugin_other(plugin) and
load_plugin_checks(path_map, plugin) and
load_plugin_alert_endpoints(plugin) then
loaded_plugins[plugin.key] = plugin
else
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Errors occurred while processing plugin '%s'", plugin.key))
end
::continue::
end
-- Save the locales
for fname, plugins_locales in pairs(locales) do
local locale_path = os_utils.fixPath(RUNTIME_PATHS.locales .. "/" .. fname)
persistence.store(locale_path, plugins_locales)
ntop.setDefaultFilePermissions(locale_path)
end
-- Save loaded plugins metadata
-- See load_metadata()
local plugins_metadata = {
plugins = loaded_plugins,
path_map = path_map,
}
persistence.store(getMetadataPath(), plugins_metadata)
ntop.setDefaultFilePermissions(getMetadataPath())
-- Swap the active plugins directory with the shadow
clearInternalState()
ntop.swapPluginsDir()
-- Remove the list of system scripts enabled, re-added from the checks.lua file
deleteCachePattern("ntonpng.cache.checks.available_system_modules.*")
-- Reload checks with their configurations
@ -760,89 +261,12 @@ end
-- ##############################################
function plugins_utils.getUrl(script)
return(plugins_utils.getMonitorUrl(script))
end
-- ##############################################
function plugins_utils.timeseriesCreationEnabled()
return areSystemTimeseriesEnabled()
end
-- ##############################################
local function load_metadata()
if not METADATA then
local runtime_path = plugins_utils.getRuntimePath()
lua_path_utils.package_path_prepend(runtime_path)
-- Do the require via pcall to avoid Lua generating an exception.
-- Print an error and a stacktrace when the require fails.
local status
status, METADATA = pcall(require, PLUGIN_RELATIVE_PATHS.metadata)
if not status then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Could not load plugins metadata file '%s'", PLUGIN_RELATIVE_PATHS.metadata))
-- Prints the traceback using multiple traceError to make sure it ends up completely as error in logging systems
local tb = debug.traceback()
for s in tb:gmatch("[^\r\n]+") do
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("%s", s))
end
end
end
end
-- ##############################################
-- @brief Retrieve the original source path of a user script
-- @param script_path the runtime path of the user script
-- @return the user script source path
function plugins_utils.getUserScriptSourcePath(script_path)
load_metadata()
if(not METADATA) then
return(nil)
end
local info = METADATA.path_map[script_path]
if info then
return(info.source_path)
end
end
-- ##############################################
-- @brief Retrieve the runtime data directory of the plugin, which is specified in the "data_dirs" directive of the plugin manifest.lua
-- @param plugin_key the plugin name
-- @param subdir an optional subdirectory of the datadir
-- @return the runtime directory path
function plugins_utils.getPluginDataDir(plugin_key, subdir)
init_runtime_paths()
local path = RUNTIME_PATHS.plugins_data .. "/" .. plugin_key
if subdir then
path = path .. "/" .. subdir
end
return os_utils.fixPath(path)
end
-- ##############################################
-- @brief Get the httpdocs directory of the plugin. This can be used to access
-- javascript, css and similar files
function plugins_utils.getHttpdocsDir(plugin_name)
local dir = ternary(ntop.isPlugins0Dir(), "plugins0_httpdocs", "plugins1_httpdocs")
-- See url_rewrite_patterns in HTTPserver.cpp
return(os_utils.fixPath(ntop.getHttpPrefix() .. "/".. dir .."/" .. plugin_name))
end
-- ##############################################
-- @brief Retrieve the runtime templates directory of the plugin
-- @param plugin_name the plugin name
-- @return the runtime directory path
@ -856,39 +280,6 @@ end
-- ##############################################
-- @brief Retrieve the plugin associated with the user script
-- @param script_path the runtime path of the user script
-- @return the associated plugin
function plugins_utils.getUserScriptPlugin(script_path)
load_metadata()
if(not METADATA) then
return(nil)
end
local info = METADATA.path_map[script_path]
if info then
return(info.plugin)
end
end
-- ##############################################
-- @brief Retrieve metadata of the loaded plugins
-- @return the loaded plugins metadata
function plugins_utils.getLoadedPlugins()
load_metadata()
if(not METADATA) then
return({})
end
return(METADATA.plugins)
end
-- ##############################################
-- Descending sort by priority
local function endpoint_sorter(a, b)
if((a.prio ~= nil) and (b.prio == nil)) then
@ -945,19 +336,9 @@ function plugins_utils.getLoadedAlertEndpoints()
-- Community endpoints
rv = get_available_notification("/scripts/lua/modules/notifications/endpoints/", rv)
-- Pro endpoints
-- Pro, Enterprise M and Enterprise L endpoints
if ntop.isPro() then
rv = get_available_notification("/pro/scripts/lua/notifications/endpoints/", rv)
-- Enterprise M endpoints
if ntop.isEnterpriseM() then
rv = get_available_notification("/pro/scripts/lua/enterprise/enterprise_m/notifications/endpoints/", rv)
-- Enterprise L endpoints
if ntop.isEnterpriseL() then
rv = get_available_notification("/pro/scripts/lua/enterprise/enterprise_l/notifications/endpoints/", rv)
end
end
end
-- Sort by priority (higher priority first)
@ -1065,15 +446,4 @@ end
-- ##############################################
-- @brief Deletes the plugins runtime directories. This is usually called
-- in boot.lua to start fresh.
function plugins_utils.cleanup()
ntop.rmdir(os_utils.fixPath(dirs.workingdir .. "/plugins"))
ntop.rmdir(ntop.getCurrentPluginsDir())
ntop.rmdir(ntop.getShadowPluginsDir())
clearInternalState()
end
-- ##############################################
return(plugins_utils)