Fixes circular dependency and lightened some modules

This commit is contained in:
Matteo Biscosi 2024-02-21 10:34:57 +00:00
parent de03d66a73
commit ef27b0b058
14 changed files with 1929 additions and 2045 deletions

View file

@ -7,6 +7,7 @@ package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path
-- Import the classes library.
local classes = require "classes"
local alert_severities = require "alert_severities"
local alert_granularities = require "alert_granularities"
-- ##############################################
@ -32,7 +33,6 @@ end
-- Those information could be set using standard functions
-- like set_score() or set_granularity
function Alert:set_info(params)
local alert_consts = require "alert_consts"
local script = params.check
if (not self.score or self.score == 0) and (script) then
self.score = ntop.mapSeverityToScore(script.severity.severity_id or 0 --[[ no score ]] )
@ -46,7 +46,7 @@ function Alert:set_info(params)
self.subtype = params.entity_info.name or ""
end
self.granularity = alert_consts.alerts_granularities[params.granularity]
self.granularity = alert_granularities[params.granularity]
end
-- ##############################################
@ -173,9 +173,7 @@ end
-- ##############################################
function Alert:set_granularity(granularity)
local alert_consts = require "alert_consts"
self.granularity = alert_consts.alerts_granularities[granularity]
self.granularity = alert_granularities[granularity]
if (self.granularity == nil) then
print("[ERROR] Unknown granularity\n")
tprint(granularity)

View file

@ -25,6 +25,7 @@ local alert_granularities = require "alert_granularities"
local alert_severities = require "alert_severities"
local alert_categories = require "alert_categories"
local alert_entities = require "alert_entities"
local consts = require "consts"
-- ##############################################
@ -41,7 +42,7 @@ if (ntop.isPro()) then
-- NOTE: import snmp_utils below to avoid import cycles
end
alert_consts.SEPARATOR = ';'
alert_consts.SEPARATOR = consts.SEPARATOR
-- NOTE: sqlite can handle about 10-50 alerts/sec
alert_consts.MAX_NUM_QUEUED_ALERTS_PER_MODULE = 1024 -- should match ALERTS_MANAGER_MAX_ENTITY_ALERTS
alert_consts.MAX_NUM_QUEUED_ALERTS_PER_RECIPIENT = 4096
@ -328,7 +329,6 @@ local function loadAlertsDefs()
traceError(TRACE_WARNING, TRACE_CONSOLE, "second.lua is loading alert_consts.lua. This will slow it down!")
end
end
local defs_dirs = alert_consts.getDefinititionDirs()
for _, defs_dir in pairs(defs_dirs) do
@ -705,7 +705,6 @@ function alert_consts.sec2granularity(seconds)
end
return key
end
-- Load definitions now
loadAlertsDefs()
initMappings()

View file

@ -664,99 +664,6 @@ end
-- ##############################################
-- Processes queued alerts and returns the information necessary to store them.
-- Alerts are only enqueued by AlertsQueue in C. From lua, the alerts_api
-- can be called directly as slow operations will be postponed
local function processStoreAlertFromQueue(alert)
local entity_info = nil
local type_info = nil
interface.select(tostring(alert.ifid))
if (alert.alert_id == "misconfigured_dhcp_range") then
local router_info = {
host = alert.router_ip,
vlan = alert.vlan_id
}
entity_info = alerts_api.hostAlertEntity(alert.client_ip, alert.vlan_id)
type_info = alert_consts.alert_types.alert_ip_outsite_dhcp_range.new(router_info, alert.mac_address,
alert.client_mac, alert.sender_mac)
type_info:set_score_warning()
type_info:set_subtype(string.format("%s_%s_%s", hostinfo2hostkey(router_info), alert.client_mac,
alert.sender_mac))
elseif (alert.alert_id == "mac_ip_association_change") then
local name = getDeviceName(alert.new_mac)
entity_info = alerts_api.macEntity(alert.new_mac)
type_info = alert_consts.alert_types.alert_mac_ip_association_change.new(name, alert.ip, alert.old_mac,
alert.new_mac)
type_info:set_score(100)
type_info:set_subtype(string.format("%s_%s_%s", alert.ip, alert.old_mac, alert.new_mac))
elseif (alert.alert_id == "login_failed") then
entity_info = alerts_api.userEntity(alert.user)
type_info = alert_consts.alert_types.alert_login_failed.new()
type_info:set_score_warning()
elseif (alert.alert_id == "broadcast_domain_too_large") then
entity_info = alerts_api.macEntity(alert.src_mac)
type_info = alert_consts.alert_types.alert_broadcast_domain_too_large.new(alert.src_mac, alert.dst_mac,
alert.vlan_id, alert.spa, alert.tpa)
type_info:set_score_warning()
type_info:set_subtype(string.format("%u_%s_%s_%s_%s", alert.vlan_id, alert.src_mac, alert.spa, alert.dst_mac,
alert.tpa))
elseif ((alert.alert_id == "user_activity") and (alert.scope == "login")) then
entity_info = alerts_api.userEntity(alert.user)
type_info = alert_consts.alert_types.alert_user_activity.new("login", nil, nil, nil, "authorized")
type_info:set_score_notice()
type_info:set_subtype("login//")
elseif (alert.alert_id == "nfq_flushed") then
entity_info = alerts_api.interfaceAlertEntity(alert.ifid)
type_info = alert_consts.alert_types.alert_nfq_flushed.new(getInterfaceName(alert.ifid), alert.pct, alert.tot,
alert.dropped)
type_info:set_score_error()
else
traceError(TRACE_ERROR, TRACE_CONSOLE, "Unknown alert type " .. (alert.alert_id or ""))
end
local category = alert_consts.get_category_by_id(alert.alert_category or 0)
type_info:set_category(category)
return entity_info, type_info
end
-- ##############################################
-- @brief Process notifications arriving from the internal C queue
-- Such notifications are transformed into stored alerts
function alert_utils.process_notifications_from_c_queue()
local budget = 1024 -- maximum 1024 alerts per call
local budget_used = 0
-- Check for alerts pushed by the datapath to an internal queue (from C)
-- and store them (push them to the SQLite and Notification queues).
-- NOTE: this is executed in a system VM, with no interfaces references
while budget_used <= budget do
local alert = ntop.popInternalAlerts()
if alert == nil then
break
end
if (verbose) then
tprint(alert)
end
local entity_info, type_info = processStoreAlertFromQueue(alert)
if type_info and entity_info then
type_info:store(entity_info)
end
budget_used = budget_used + 1
end
end
-- ##############################################
local function notify_ntopng_status(started)
local info = ntop.getInfo()
local score = 10

View file

@ -14,7 +14,6 @@ local alert_entities = require "alert_entities"
local alert_consts = require "alert_consts"
local recipients = require "recipients"
local alert_entity_builders = require "alert_entity_builders"
local alert_management = require "alert_management"
local do_trace = false
local alerts_api = {}
@ -420,9 +419,9 @@ function alerts_api.release(entity_info, type_info, when, cur_alerts)
if (entity_info.alert_entity.entity_id == alert_consts.alertEntity("interface")) then
if (interface.checkContext(entity_info.entity_val) == false) then
-- alertErrorTraceback("Invalid interface context detected for entity id " ..
-- entity_info.alert_entity.entity_id)
-- tprint(entity_info)
-- alertErrorTraceback("Invalid interface context detected for entity id " ..
-- entity_info.alert_entity.entity_id)
-- tprint(entity_info)
return (false)
else
released = interface.releaseTriggeredAlert(table.unpack(params))
@ -464,6 +463,7 @@ end
-- ##############################################
function alerts_api.releaseAllAlerts()
local alert_management = require "alert_management"
local alerts = interface.getEngagedAlerts()
alert_management.releaseEntityAlerts(nil, alerts)
end
@ -622,8 +622,4 @@ end
-- ##############################################
if (trace_script_duration ~= nil) then
io.write(debug.getinfo(1, 'S').source .. " executed in " .. (os.clock() - clock_start) * 1000 .. " ms\n")
end
return (alerts_api)

View file

@ -1,248 +1,255 @@
--
-- (C) 2019-24 - ntop.org
--
require "ntop_utils"
-- ###########################################
-- NOTE: '~= "0"' is used for prefs which are enabled by default
function areInterfaceTimeseriesEnabled(ifid)
return((ntop.getPref("ntopng.prefs.interface_rrd_creation") ~= "0"))
return ((ntop.getPref("ntopng.prefs.interface_rrd_creation") ~= "0"))
end
-- ###########################################
function areInterfaceL7TimeseriesEnabled(ifid)
return(areInterfaceTimeseriesEnabled(ifid) and
(ntop.getPref("ntopng.prefs.interface_ndpi_timeseries_creation") ~= "per_category"))
return (areInterfaceTimeseriesEnabled(ifid) and
(ntop.getPref("ntopng.prefs.interface_ndpi_timeseries_creation") ~= "per_category"))
end
-- ###########################################
function areInterfaceCategoriesTimeseriesEnabled(ifid)
local rv = ntop.getPref("ntopng.prefs.interface_ndpi_timeseries_creation")
local rv = ntop.getPref("ntopng.prefs.interface_ndpi_timeseries_creation")
-- note: categories are disabled by default
return(areInterfaceTimeseriesEnabled(ifid) and
((rv == "per_category") or (rv == "both")))
-- note: categories are disabled by default
return (areInterfaceTimeseriesEnabled(ifid) and ((rv == "per_category") or (rv == "both")))
end
-- ###########################################
function areHostTimeseriesEnabled(ifid)
local rv = ntop.getPref("ntopng.prefs.hosts_ts_creation")
if isEmptyString(rv) then rv = "light" end
local rv = ntop.getPref("ntopng.prefs.hosts_ts_creation")
if isEmptyString(rv) then
rv = "light"
end
return((rv == "light") or (rv == "full"))
return ((rv == "light") or (rv == "full"))
end
-- ###########################################
function areHostL7TimeseriesEnabled(ifid)
local rv = ntop.getPref("ntopng.prefs.host_ndpi_timeseries_creation")
local rv = ntop.getPref("ntopng.prefs.host_ndpi_timeseries_creation")
-- note: host protocols are disabled by default
return((ntop.getPref("ntopng.prefs.hosts_ts_creation") == "full") and
((rv == "per_protocol") or (rv == "both")))
-- note: host protocols are disabled by default
return ((ntop.getPref("ntopng.prefs.hosts_ts_creation") == "full") and ((rv == "per_protocol") or (rv == "both")))
end
-- ###########################################
function areHostCategoriesTimeseriesEnabled(ifid)
local rv = ntop.getPref("ntopng.prefs.host_ndpi_timeseries_creation")
local rv = ntop.getPref("ntopng.prefs.host_ndpi_timeseries_creation")
-- note: host protocols are disabled by default
return((ntop.getPref("ntopng.prefs.hosts_ts_creation") == "full") and
((rv == "per_category") or (rv == "both")))
-- note: host protocols are disabled by default
return ((ntop.getPref("ntopng.prefs.hosts_ts_creation") == "full") and ((rv == "per_category") or (rv == "both")))
end
-- ###########################################
function areSystemTimeseriesEnabled()
return(ntop.getPref("ntopng.prefs.system_probes_timeseries") ~= "0")
return (ntop.getPref("ntopng.prefs.system_probes_timeseries") ~= "0")
end
-- ###########################################
function areHostPoolsTimeseriesEnabled(ifid)
return(ntop.isPro() and (ntop.getPref("ntopng.prefs.host_pools_rrd_creation") == "1"))
return (ntop.isPro() and (ntop.getPref("ntopng.prefs.host_pools_rrd_creation") == "1"))
end
-- ###########################################
function areASTimeseriesEnabled(ifid)
return(ntop.getPref("ntopng.prefs.asn_rrd_creation") == "1")
return (ntop.getPref("ntopng.prefs.asn_rrd_creation") == "1")
end
-- ###########################################
function areInternalTimeseriesEnabled(ifid)
-- NOTE: no separate preference so far
return(areSystemTimeseriesEnabled())
-- NOTE: no separate preference so far
return (areSystemTimeseriesEnabled())
end
-- ###########################################
function areCountryTimeseriesEnabled(ifid)
return((ntop.getPref("ntopng.prefs.country_rrd_creation") == "1"))
return ((ntop.getPref("ntopng.prefs.country_rrd_creation") == "1"))
end
-- ###########################################
function areOSTimeseriesEnabled(ifid)
return((ntop.getPref("ntopng.prefs.os_rrd_creation") == "1"))
return ((ntop.getPref("ntopng.prefs.os_rrd_creation") == "1"))
end
-- ###########################################
function areVlanTimeseriesEnabled(ifid)
return(ntop.getPref("ntopng.prefs.vlan_rrd_creation") == "1")
return (ntop.getPref("ntopng.prefs.vlan_rrd_creation") == "1")
end
-- ###########################################
function areMacsTimeseriesEnabled(ifid)
return(ntop.getPref("ntopng.prefs.l2_device_rrd_creation") == "1")
return (ntop.getPref("ntopng.prefs.l2_device_rrd_creation") == "1")
end
-- ###########################################
function areContainersTimeseriesEnabled(ifid)
-- NOTE: no separate preference so far
return(true)
-- NOTE: no separate preference so far
return (true)
end
-- ###########################################
function areSnmpTimeseriesEnabled(device, port_idx)
return(ntop.getPref("ntopng.prefs.snmp_devices_rrd_creation") == "1")
return (ntop.getPref("ntopng.prefs.snmp_devices_rrd_creation") == "1")
end
-- ###########################################
function areFlowdevTimeseriesEnabled(ifid, device)
return(ntop.getPref("ntopng.prefs.flow_device_port_rrd_creation") == "1")
return (ntop.getPref("ntopng.prefs.flow_device_port_rrd_creation") == "1")
end
-- ###########################################
function areAlertsEnabled()
if(__alert_enabled == nil) then
-- Not too nice as changes will be read periodically as new VMs are reloaded
-- but at least we avoid breaking up the performance
__alert_enabled = (ntop.getPref("ntopng.prefs.disable_alerts_generation") ~= "1")
end
if (__alert_enabled == nil) then
-- Not too nice as changes will be read periodically as new VMs are reloaded
-- but at least we avoid breaking up the performance
__alert_enabled = (ntop.getPref("ntopng.prefs.disable_alerts_generation") ~= "1")
end
return (__alert_enabled)
return (__alert_enabled)
end
-- ##########################################
function get5MinTSConfig()
local config = {}
local config = {}
config.host_ts_creation = ntop.getPref("ntopng.prefs.hosts_ts_creation")
config.host_ndpi_timeseries_creation = ntop.getPref("ntopng.prefs.host_ndpi_timeseries_creation")
config.l2_device_rrd_creation = ntop.getPref("ntopng.prefs.l2_device_rrd_creation")
config.l2_device_ndpi_timeseries_creation = ntop.getPref("ntopng.prefs.l2_device_ndpi_timeseries_creation")
config.flow_devices_rrd_creation = ntop.getPref("ntopng.prefs.flow_device_port_rrd_creation")
config.host_pools_rrd_creation = ntop.getPref("ntopng.prefs.host_pools_rrd_creation")
config.snmp_devices_rrd_creation = ntop.getPref("ntopng.prefs.snmp_devices_rrd_creation")
config.asn_rrd_creation = ntop.getPref("ntopng.prefs.asn_rrd_creation")
config.obs_point_rrd_creation = ntop.getPref("ntopng.prefs.observation_points_rrd_creation")
config.country_rrd_creation = ntop.getPref("ntopng.prefs.country_rrd_creation")
config.os_rrd_creation = ntop.getPref("ntopng.prefs.os_rrd_creation")
config.vlan_rrd_creation = ntop.getPref("ntopng.prefs.vlan_rrd_creation")
config.ndpi_flows_timeseries_creation = ntop.getPref("ntopng.prefs.ndpi_flows_rrd_creation")
config.interface_ndpi_timeseries_creation = ntop.getPref("ntopng.prefs.interface_ndpi_timeseries_creation")
config.host_ts_creation = ntop.getPref("ntopng.prefs.hosts_ts_creation")
config.host_ndpi_timeseries_creation = ntop.getPref("ntopng.prefs.host_ndpi_timeseries_creation")
config.l2_device_rrd_creation = ntop.getPref("ntopng.prefs.l2_device_rrd_creation")
config.l2_device_ndpi_timeseries_creation = ntop.getPref("ntopng.prefs.l2_device_ndpi_timeseries_creation")
config.flow_devices_rrd_creation = ntop.getPref("ntopng.prefs.flow_device_port_rrd_creation")
config.host_pools_rrd_creation = ntop.getPref("ntopng.prefs.host_pools_rrd_creation")
config.snmp_devices_rrd_creation = ntop.getPref("ntopng.prefs.snmp_devices_rrd_creation")
config.asn_rrd_creation = ntop.getPref("ntopng.prefs.asn_rrd_creation")
config.obs_point_rrd_creation = ntop.getPref("ntopng.prefs.observation_points_rrd_creation")
config.country_rrd_creation = ntop.getPref("ntopng.prefs.country_rrd_creation")
config.os_rrd_creation = ntop.getPref("ntopng.prefs.os_rrd_creation")
config.vlan_rrd_creation = ntop.getPref("ntopng.prefs.vlan_rrd_creation")
config.ndpi_flows_timeseries_creation = ntop.getPref("ntopng.prefs.ndpi_flows_rrd_creation")
config.interface_ndpi_timeseries_creation = ntop.getPref("ntopng.prefs.interface_ndpi_timeseries_creation")
-- ########################################################
-- Populate some defaults
if (tostring(config.flow_devices_rrd_creation) == "1" and ntop.isEnterpriseM() == false) then
config.flow_devices_rrd_creation = "0"
end
-- ########################################################
-- Populate some defaults
if (tostring(config.flow_devices_rrd_creation) == "1" and ntop.isEnterpriseM() == false) then
config.flow_devices_rrd_creation = "0"
end
if (tostring(config.snmp_devices_rrd_creation) == "1" and not (ntop.isEnterpriseM() or ntop.isnEdgeEnterprise())) then
config.snmp_devices_rrd_creation = "0"
end
if (tostring(config.snmp_devices_rrd_creation) == "1" and not (ntop.isEnterpriseM() or ntop.isnEdgeEnterprise())) then
config.snmp_devices_rrd_creation = "0"
end
-- Local hosts RRD creation is on, with no nDPI rrd creation
if isEmptyString(config.host_ts_creation) then
config.host_ts_creation = "light"
end
if isEmptyString(config.host_ndpi_timeseries_creation) then
config.host_ndpi_timeseries_creation = "none"
end
-- Local hosts RRD creation is on, with no nDPI rrd creation
if isEmptyString(config.host_ts_creation) then
config.host_ts_creation = "light"
end
if isEmptyString(config.host_ndpi_timeseries_creation) then
config.host_ndpi_timeseries_creation = "none"
end
-- Devices RRD creation is OFF, as OFF is the nDPI rrd creation
if isEmptyString(config.l2_device_rrd_creation) then
config.l2_device_rrd_creation = "0"
end
if isEmptyString(config.l2_device_ndpi_timeseries_creation) then
config.l2_device_ndpi_timeseries_creation = "none"
end
-- Devices RRD creation is OFF, as OFF is the nDPI rrd creation
if isEmptyString(config.l2_device_rrd_creation) then
config.l2_device_rrd_creation = "0"
end
if isEmptyString(config.l2_device_ndpi_timeseries_creation) then
config.l2_device_ndpi_timeseries_creation = "none"
end
-- Interface RRD creation is on, with per-protocol nDPI, Pref used by Observation Points
if isEmptyString(config.interface_ndpi_timeseries_creation) then
config.interface_ndpi_timeseries_creation = "per_protocol"
end
-- Interface RRD creation is on, with per-protocol nDPI, Pref used by Observation Points
if isEmptyString(config.interface_ndpi_timeseries_creation) then
config.interface_ndpi_timeseries_creation = "per_protocol"
end
return config
return config
end
-- ###########################################
function getMinTSConfig()
local config = {}
local prefs = ntop.getPrefs() -- runtime ntopng preferences
local config = {}
local prefs = ntop.getPrefs() -- runtime ntopng preferences
config.interface_ndpi_timeseries_creation = ntop.getPref("ntopng.prefs.interface_ndpi_timeseries_creation")
config.ndpi_flows_timeseries_creation = ntop.getPref("ntopng.prefs.ndpi_flows_rrd_creation")
config.internals_rrd_creation = ntop.getPref("ntopng.prefs.internals_rrd_creation") == "1"
config.is_dump_flows_enabled = ntop.getPrefs()["is_dump_flows_enabled"]
config.interface_ndpi_timeseries_creation = ntop.getPref("ntopng.prefs.interface_ndpi_timeseries_creation")
config.ndpi_flows_timeseries_creation = ntop.getPref("ntopng.prefs.ndpi_flows_rrd_creation")
config.internals_rrd_creation = ntop.getPref("ntopng.prefs.internals_rrd_creation") == "1"
config.is_dump_flows_enabled = ntop.getPrefs()["is_dump_flows_enabled"]
-- Interface RRD creation is on, with per-protocol nDPI
if isEmptyString(config.interface_ndpi_timeseries_creation) then
config.interface_ndpi_timeseries_creation = "per_protocol"
end
-- Interface RRD creation is on, with per-protocol nDPI
if isEmptyString(config.interface_ndpi_timeseries_creation) then
config.interface_ndpi_timeseries_creation = "per_protocol"
end
return config
return config
end
-- ##############################################
-- Get from redis the throughput type bps or pps
function getThroughputType()
local throughput_type = ntop.getCache("ntopng.prefs.thpt_content")
if throughput_type == "" then throughput_type = "bps" end
return throughput_type
local throughput_type = ntop.getCache("ntopng.prefs.thpt_content")
if throughput_type == "" then
throughput_type = "bps"
end
return throughput_type
end
-- ##############################################
function hasClickHouseSupport()
local auth = require "auth"
local auth = require "auth"
if not (ntop.isPro() or ntop.isnEdgeEnterprise())
or ntop.isWindows() then
return false
end
if not (ntop.isPro() or ntop.isnEdgeEnterprise()) or ntop.isWindows() then
return false
end
-- Don't allow nIndex for unauthorized users
if not auth.has_capability(auth.capabilities.historical_flows) then
return false
end
-- Don't allow nIndex for unauthorized users
if not auth.has_capability(auth.capabilities.historical_flows) then
return false
end
-- TODO optimize
if prefs == nil then
prefs = ntop.getPrefs()
end
-- TODO optimize
if prefs == nil then
prefs = ntop.getPrefs()
end
if prefs.is_dump_flows_to_clickhouse_enabled then
return true
end
if prefs.is_dump_flows_to_clickhouse_enabled then
return true
end
return false
end
return false
end
-- ##############################################
-- NOTE: global nindex support may be enabled but some disable on some interfaces
function interfaceHasClickHouseSupport()
require "check_redis_prefs"
return (hasClickHouseSupport() and ntop.getPrefs()["is_dump_flows_to_clickhouse_enabled"])
end

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,7 @@
--
-- (C) 2020-24 - ntop.org
--
local consts = {}
consts.SEPARATOR = ';'

View file

@ -673,29 +673,6 @@ function mac2label(mac)
return(mac)
end
-- ##############################################
-- Mac Addresses --
local specialMACs = {
"01:00:0C",
"01:80:C2",
"01:00:5E",
"01:0C:CD",
"01:1B:19",
"FF:FF",
"33:33"
}
function isSpecialMac(mac)
for _,key in pairs(specialMACs) do
if(string.contains(mac, key)) then
return true
end
end
return false
end
-- ##############################################
@ -740,21 +717,6 @@ end
-- ##############################################
function flow2hostinfo(host_info, host_type)
local host_name
local res = interface.getHostMinInfo(host_info[host_type .. ".ip"])
if((res == nil) or (res["name"] == nil)) then
host_name = host_info[host_type .. ".ip"]
else
host_name = res["name"]
end
return({host = host_info[host_type .. ".ip"], vlan = host_info[host_type .. ".vlan"], name = host_name})
end
-- ##############################################
function isHostKey(key)
local info = split(key,"@")
-- Check format
@ -1180,17 +1142,6 @@ end
-- ###########################################
-- @brief Deletes all the cache/prefs keys matching the pattern
function deleteCachePattern(pattern)
local keys = ntop.getKeysCache(pattern)
for key in pairs(keys or {}) do
ntop.delCache(key)
end
end
-- ###########################################
-- version is major.minor.veryminor
function version2int(v)
if(v == nil) then return(0) end

View file

@ -2,16 +2,9 @@
-- (C) 2014-24 - ntop.org
--
if(pragma_once_lua_utils_generic == true) then
-- io.write(debug.traceback().."\n")
-- avoid multiple inclusions
return
end
pragma_once_lua_utils_generic = true
local clock_start = os.clock()
-- This require is okay, it just adds up a couple of utilities
require "string_utils"
require "check_redis_prefs"
-- GENERIC UTILS
-- split
@ -33,16 +26,6 @@ function split(s, delimiter)
return result;
end
-- startswith
function startswith(s, char)
return string.sub(s, 1, string.len(s)) == char
end
-- endswith
function endswith(s, char)
return string.sub(s, -#char) == char
end
-- strsplit
function strsplit(s, delimiter)
result = {};
@ -81,21 +64,6 @@ end
-- ###############################################
-- removes trailing/leading spaces
function trimString(s)
return (s:gsub("^%s*(.-)%s*$", "%1"))
end
-- ###############################################
-- removes all spaces
function trimSpace(what)
if(what == nil) then return("") end
return(string.gsub(string.gsub(what, "%s+", ""), "+%s", ""))
end
-- ###############################################
-- TODO: improve this function
function jsonencode(what)
what = string.gsub(what, '"', "'")
@ -389,50 +357,6 @@ end
-- ##############################################
-- Note: Regexs are applied by default. Pass plain=true to disable them.
function string.contains(str, start, is_plain)
if type(str) ~= 'string' or type(start) ~= 'string' or isEmptyString(str) or isEmptyString(start) then
return false
end
local i, _ = string.find(str, start, 1, is_plain)
return(i ~= nil)
end
-- ##############################################
function string.containsIgnoreCase(str, start, is_plain)
return string.contains(string.lower(str), string.lower(start), is_plain)
end
-- ##############################################
function shortenString(name, max_len)
local ellipsis = "\u{2026}" -- The unicode ellipsis (takes less space than three separate dots)
if(name == nil) then return("") end
if max_len == nil then
max_len = ntop.getPref("ntopng.prefs.max_ui_strlen")
max_len = tonumber(max_len)
if(max_len == nil) then max_len = 24 end
end
-- Error, max_len is not a number, print an error and return the name
if not tonumber(max_len) then
traceError(TRACE_DEBUG, TRACE_CONSOLE, "Length parameter is not a number.")
tprint(debug.traceback())
return name
end
if(string.len(name) < max_len + 1 --[[ The space taken by the ellipsis --]]) then
return(name)
else
return(string.sub(name, 1, max_len)..ellipsis)
end
end
-- ##############################################
function convertDate(vardate)
local m,d,y,h,i,s = string.match(vardate, '(%d+)/(%d+)/(%d+) (%d+):(%d+):(%d+)')
local key = ntop.getPref('ntopng.user.' .. _SESSION["user"] .. '.date_format')
@ -529,15 +453,6 @@ function printInterfaceIndex(idx)
end
end
-- ##############################################
-- NOTE: global nindex support may be enabled but some disable on some interfaces
function interfaceHasClickHouseSupport()
require "check_redis_prefs"
return(hasClickHouseSupport() and ntop.getPrefs()["is_dump_flows_to_clickhouse_enabled"])
end
-- ###########################################
function swapKeysValues(tbl)

View file

@ -13,6 +13,8 @@ end
pragma_once_ntop_utils = true
-- This is required because it simply adds the tracing functions, like traceError
require "lua_trace"
local clock_start = os.clock()

View file

@ -1,7 +1,6 @@
--
-- (C) 2019-24 - ntop.org
--
-- Includes
local dirs = ntop.getDirs()
package.path = dirs.installdir .. "/scripts/lua/modules/notifications/?.lua;" .. package.path
@ -10,6 +9,8 @@ local os_utils = require("os_utils")
local template_utils = require("template_utils")
local lua_path_utils = require("lua_path_utils")
require "lua_trace"
require "lua_utils_generic"
require "check_redis_prefs"
-- ##############################################
@ -37,11 +38,11 @@ local cached_runtime_dir = nil
-- @brief Return the path of the scripts
function script_manager.getRuntimePath()
if(not cached_runtime_dir) then
cached_runtime_dir = ntop.getCurrentScriptsDir()
end
if (not cached_runtime_dir) then
cached_runtime_dir = ntop.getCurrentScriptsDir()
end
return(cached_runtime_dir)
return (cached_runtime_dir)
end
-- ##############################################
@ -50,156 +51,132 @@ end
local runtime_path = script_manager.getRuntimePath()
local MENU_ITEMS_PATH = "menu_items"
local RUNTIME_PATHS = {
-- Definitions
alert_definitions = os_utils.fixPath(runtime_path .. "/alert_definitions"),
check_definitions = os_utils.fixPath(runtime_path .. "/check_definitions"),
-- Definitions
alert_definitions = os_utils.fixPath(runtime_path .. "/alert_definitions"),
check_definitions = os_utils.fixPath(runtime_path .. "/check_definitions"),
-- Locales
locales = os_utils.fixPath(runtime_path .. "/locales"),
-- Locales
locales = os_utils.fixPath(runtime_path .. "/locales"),
-- Timeseries
ts_schemas = os_utils.fixPath(runtime_path .. "/ts_schemas"),
-- Timeseries
ts_schemas = os_utils.fixPath(runtime_path .. "/ts_schemas"),
-- Web Gui
web_gui = os_utils.fixPath(runtime_path) .. "/scripts",
menu_items = os_utils.fixPath(runtime_path.."/"..MENU_ITEMS_PATH),
-- Web Gui
web_gui = os_utils.fixPath(runtime_path) .. "/scripts",
menu_items = os_utils.fixPath(runtime_path .. "/" .. MENU_ITEMS_PATH),
-- Alert endpoints
alert_endpoints = os_utils.fixPath(runtime_path) .. "/alert_endpoints",
-- Alert endpoints
alert_endpoints = os_utils.fixPath(runtime_path) .. "/alert_endpoints",
-- HTTP lint
http_lint = os_utils.fixPath(runtime_path) .. "/http_lint",
-- HTTP lint
http_lint = os_utils.fixPath(runtime_path) .. "/http_lint",
-- TODO: rename scripts_data
-- Scripts Data Directories
scripts_data = os_utils.fixPath(runtime_path) .. "/scripts_data",
-- TODO: rename scripts_data
-- Scripts Data Directories
scripts_data = os_utils.fixPath(runtime_path) .. "/scripts_data",
-- Templates
templates = os_utils.fixPath(runtime_path) .. "/templates",
-- Lua Modules
modules = os_utils.fixPath(runtime_path) .. "/modules",
-- Client Side Files
httpdocs = os_utils.fixPath(runtime_path) .. "/httpdocs",
-- Templates
templates = os_utils.fixPath(runtime_path) .. "/templates",
-- Callbacks
interface_scripts = os_utils.fixPath(runtime_path .. "/callbacks/interface/interface"),
host_scripts = os_utils.fixPath(runtime_path .. "/callbacks/interface/host"),
network_scripts = os_utils.fixPath(runtime_path .. "/callbacks/interface/network"),
flow_scripts = os_utils.fixPath(runtime_path .. "/callbacks/interface/flow"),
syslog = os_utils.fixPath(runtime_path .. "/callbacks/system/syslog"),
snmp_scripts = os_utils.fixPath(runtime_path .. "/callbacks/system/snmp_device"),
system_scripts = os_utils.fixPath(runtime_path .. "/callbacks/system/system"),
active_monitoring_scripts = os_utils.fixPath(runtime_path .. "/callbacks/system/active_monitoring"),
-- Lua Modules
modules = os_utils.fixPath(runtime_path) .. "/modules",
-- Client Side Files
httpdocs = os_utils.fixPath(runtime_path) .. "/httpdocs",
-- Callbacks
interface_scripts = os_utils.fixPath(runtime_path .. "/callbacks/interface/interface"),
host_scripts = os_utils.fixPath(runtime_path .. "/callbacks/interface/host"),
network_scripts = os_utils.fixPath(runtime_path .. "/callbacks/interface/network"),
flow_scripts = os_utils.fixPath(runtime_path .. "/callbacks/interface/flow"),
syslog = os_utils.fixPath(runtime_path .. "/callbacks/system/syslog"),
snmp_scripts = os_utils.fixPath(runtime_path .. "/callbacks/system/snmp_device"),
system_scripts = os_utils.fixPath(runtime_path .. "/callbacks/system/system"),
active_monitoring_scripts = os_utils.fixPath(runtime_path .. "/callbacks/system/active_monitoring")
}
-- ##############################################
-- @brief Loads the ntopng scripts into a single directory.
function script_manager.loadScripts()
for _, path in pairs(RUNTIME_PATHS) do
ntop.mkdir(path)
end
-- Make sure to invalidate the (possibly) already required alert_consts which depends on alert definitions.
-- By invalidating the module, we make sure all the newly loaded alert definitions will be picked up by any
-- subsequent `require "alert_consts"`
package.loaded["alert_consts"] = nil
-- Remove the list of system scripts enabled, re-added from the checks.lua file
deleteCachePattern("ntonpng.cache.checks.available_system_modules.*")
-- Reload checks with their configurations
local checks = require "checks"
checks.initDefaultConfig()
checks.loadUnloadUserScripts(true --[[ load --]])
return(true)
end
-- ##############################################
-- @brief Loads the timeseries schemas.
function script_manager.loadSchemas(granularity)
lua_path_utils.package_path_prepend(RUNTIME_PATHS.ts_schemas)
lua_path_utils.package_path_prepend(RUNTIME_PATHS.ts_schemas)
for ts_name in pairs(ntop.readdir(RUNTIME_PATHS.ts_schemas)) do
local ts_dir = os_utils.fixPath(RUNTIME_PATHS.ts_schemas .. "/" .. ts_name)
local files_to_load = {}
for ts_name in pairs(ntop.readdir(RUNTIME_PATHS.ts_schemas)) do
local ts_dir = os_utils.fixPath(RUNTIME_PATHS.ts_schemas .. "/" .. ts_name)
local files_to_load = {}
if(granularity ~= nil) then
-- Only load schemas for the specified granularity
local ts_granularity_file = granularity..".lua"
local ts_granularity_path = os_utils.fixPath(ts_dir.."/"..ts_granularity_file)
if (granularity ~= nil) then
-- Only load schemas for the specified granularity
local ts_granularity_file = granularity .. ".lua"
local ts_granularity_path = os_utils.fixPath(ts_dir .. "/" .. ts_granularity_file)
if ntop.exists(ts_granularity_path) then
files_to_load = { ts_granularity_file }
if ntop.exists(ts_granularity_path) then
files_to_load = {ts_granularity_file}
end
else
-- Load all granularities
files_to_load = ntop.readdir(ts_dir)
end
else
-- Load all granularities
files_to_load = ntop.readdir(ts_dir)
end
for _, fname in pairs(files_to_load) do
if fname:ends(".lua") then
local fgran = string.sub(fname, 1, string.len(fname) - 4)
-- Ts schemas are required using the dot notation in the
-- require string name. Dots are used to navigate the base directory, RUNTIME_PATHS.ts_schemas,
-- which has been prepended to the path.
-- Examples:
-- require(active_monitoring.hour)
-- require(active_monitoring.5mins)
-- require(active_monitoring.min)
-- require(score.min)
-- require(influxdb_monitor.5mins)
local req_name = string.format("%s.%s", ts_name, fgran)
require(req_name)
for _, fname in pairs(files_to_load) do
if fname:ends(".lua") then
local fgran = string.sub(fname, 1, string.len(fname) - 4)
-- Ts schemas are required using the dot notation in the
-- require string name. Dots are used to navigate the base directory, RUNTIME_PATHS.ts_schemas,
-- which has been prepended to the path.
-- Examples:
-- require(active_monitoring.hour)
-- require(active_monitoring.5mins)
-- require(active_monitoring.min)
-- require(score.min)
-- require(influxdb_monitor.5mins)
local req_name = string.format("%s.%s", ts_name, fgran)
require(req_name)
end
end
end
end
end
end
-- ##############################################
-- @brief Return the menu entries
function script_manager.getMenuEntries()
local menu = {}
local entries_data = {}
local menu = {}
local entries_data = {}
lua_path_utils.package_path_prepend(script_manager.getRuntimePath())
lua_path_utils.package_path_prepend(script_manager.getRuntimePath())
for fname in pairs(ntop.readdir(RUNTIME_PATHS.menu_items)) do
local req_name = string.format("%s.%s", MENU_ITEMS_PATH, fname)
local menu_entry = require(req_name)
for fname in pairs(ntop.readdir(RUNTIME_PATHS.menu_items)) do
local req_name = string.format("%s.%s", MENU_ITEMS_PATH, fname)
local menu_entry = require(req_name)
if(menu_entry and ((not menu_entry.is_shown) or menu_entry.is_shown())) then
-- Don't add any getHttpPrefix to the url here, it's the caller that
-- can potentially add it
menu_entry.url = "/scripts/" .. menu_entry.script
menu[fname] = menu_entry
if (menu_entry and ((not menu_entry.is_shown) or menu_entry.is_shown())) then
-- Don't add any getHttpPrefix to the url here, it's the caller that
-- can potentially add it
menu_entry.url = "/scripts/" .. menu_entry.script
menu[fname] = menu_entry
if menu_entry.menu_entry then
entries_data[menu_entry.menu_entry.key] = menu_entry.menu_entry
end
if menu_entry.menu_entry then
entries_data[menu_entry.menu_entry.key] = menu_entry.menu_entry
end
end
end
end
return menu, entries_data
return menu, entries_data
end
-- ##############################################
-- @brief Return monitor pages URL (e.g. /lua/monitor/redis_monitor.lua)
function script_manager.getMonitorUrl(script)
return(ntop.getHttpPrefix() .. "/lua/monitor/" .. script)
return (ntop.getHttpPrefix() .. "/lua/monitor/" .. script)
end
-- ##############################################
-- @brief Checks if the system timeseries are enabled
function script_manager.systemTimeseriesEnabled()
return areSystemTimeseriesEnabled()
return areSystemTimeseriesEnabled()
end
-- ##############################################
@ -208,8 +185,8 @@ end
-- @param script_name the script name
-- @return the runtime directory path
function script_manager.getScriptTemplatesDir(script_name)
local path = dirs.installdir .. "/httpdocs/templates/pages/" .. (script_name or '')
return os_utils.fixPath(path)
local path = dirs.installdir .. "/httpdocs/templates/pages/" .. (script_name or '')
return os_utils.fixPath(path)
end
-- ##############################################
@ -217,47 +194,47 @@ end
-- @brief Return the list of available endpoint/recipients,
-- named even 'notification'
local function get_available_notification(path, rv)
-- Get Endpoints files, like discord.lua, slack.lua ecc.
local base_path = os_utils.fixPath(dirs.installdir .. path)
lua_path_utils.package_path_prepend(base_path)
-- Get Endpoints files, like discord.lua, slack.lua ecc.
local base_path = os_utils.fixPath(dirs.installdir .. path)
lua_path_utils.package_path_prepend(base_path)
for fname in pairs(ntop.readdir(base_path)) do
if fname:ends(".lua") then
local full_path = os_utils.fixPath(base_path .. "/" .. fname)
local key = string.sub(fname, 1, string.len(fname) - 4)
for fname in pairs(ntop.readdir(base_path)) do
if fname:ends(".lua") then
local full_path = os_utils.fixPath(base_path .. "/" .. fname)
local key = string.sub(fname, 1, string.len(fname) - 4)
-- Check if the endpoint has a valid function to handle the notification
local endpoint = require(key)
if(endpoint) then
if((type(endpoint.isAvailable) ~= "function") or endpoint.isAvailable()) then
endpoint.full_path = full_path
endpoint.key = key
-- Check if the endpoint has a valid function to handle the notification
local endpoint = require(key)
if (endpoint) then
if ((type(endpoint.isAvailable) ~= "function") or endpoint.isAvailable()) then
endpoint.full_path = full_path
endpoint.key = key
rv[#rv + 1] = endpoint
rv[#rv + 1] = endpoint
end
else
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Could not load alert endpoint '%s'", full_path))
end
end
else
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Could not load alert endpoint '%s'", full_path))
end
end
end
return rv
return rv
end
-- ##############################################
-- @brief Sorter used to sort endpoints by priority
local function endpoint_sorter(a, b)
if((a.prio ~= nil) and (b.prio == nil)) then
return(true)
elseif((a.prio == nil) and (b.prio ~= nil)) then
return(false)
elseif(a.prio ~= b.prio) then
return(a.prio > b.prio)
end
if ((a.prio ~= nil) and (b.prio == nil)) then
return (true)
elseif ((a.prio == nil) and (b.prio ~= nil)) then
return (false)
elseif (a.prio ~= b.prio) then
return (a.prio > b.prio)
end
-- Use the endpoint key to fix a defined sort order
return(a.key > b.key)
-- Use the endpoint key to fix a defined sort order
return (a.key > b.key)
end
-- ##############################################
@ -265,93 +242,101 @@ end
-- @brief Get the available alert endpoints
-- @return a sorted table, in order of priority, for the alert endpoints
function script_manager.getLoadedAlertEndpoints()
local rv = {}
local rv = {}
-- Community endpoints
rv = get_available_notification("/scripts/lua/modules/notifications/endpoints/", rv)
-- Community endpoints
rv = get_available_notification("/scripts/lua/modules/notifications/endpoints/", rv)
-- Pro, Enterprise M and Enterprise L endpoints
if ntop.isPro() then
rv = get_available_notification("/pro/scripts/lua/notifications/endpoints/", rv)
end
-- Pro, Enterprise M and Enterprise L endpoints
if ntop.isPro() then
rv = get_available_notification("/pro/scripts/lua/notifications/endpoints/", rv)
end
-- Sort by priority (higher priority first)
table.sort(rv, endpoint_sorter)
-- Sort by priority (higher priority first)
table.sort(rv, endpoint_sorter)
return(rv)
return (rv)
end
-- ##############################################
-- @brief Extends the http_lint using all the lint available
function script_manager.extendLintParams(http_lint, params)
lua_path_utils.package_path_prepend(RUNTIME_PATHS.http_lint)
for fname in pairs(ntop.readdir(RUNTIME_PATHS.http_lint)) do
local key = string.sub(fname, 1, string.len(fname) - 4)
local lint = require(key)
lua_path_utils.package_path_prepend(RUNTIME_PATHS.http_lint)
if(lint == nil) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Could not load lint for '%s'", key))
goto continue
for fname in pairs(ntop.readdir(RUNTIME_PATHS.http_lint)) do
local key = string.sub(fname, 1, string.len(fname) - 4)
local lint = require(key)
if (lint == nil) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Could not load lint for '%s'", key))
goto continue
end
if (lint.getAdditionalParameters == nil) then
traceError(TRACE_ERROR, TRACE_CONSOLE,
string.format("Missing mandatory function 'getAdditionalParameters' in '%s'", key))
goto continue
end
local rv = lint.getAdditionalParameters(http_lint)
if (type(rv) ~= "table") then
traceError(TRACE_ERROR, TRACE_CONSOLE,
string.format("function 'getAdditionalParameters' in '%s' returned a non-table value", key))
goto continue
end
for k, v in pairs(rv) do
params[k] = v
end
::continue::
end
if(lint.getAdditionalParameters == nil) then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Missing mandatory function 'getAdditionalParameters' in '%s'", key))
goto continue
end
local rv = lint.getAdditionalParameters(http_lint)
if(type(rv) ~= "table") then
traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("function 'getAdditionalParameters' in '%s' returned a non-table value", key))
goto continue
end
for k, v in pairs(rv) do
params[k] = v
end
::continue::
end
end
-- ##############################################
-- @brief Render an html template located into the templates directory
function script_manager.renderNotificationTemplate(script_name, template_file, context)
-- Locate the template file into the script directory, e.g. httpdocs/templates/pages/notifications/webhook_endpoint.template
local full_path = script_manager.getScriptTemplatesDir("notifications/" .. script_name .. "/" .. template_file)
-- Locate the template file into the script directory, e.g. httpdocs/templates/pages/notifications/webhook_endpoint.template
local full_path = script_manager.getScriptTemplatesDir("notifications/" .. script_name .. "/" .. template_file)
-- If no template is found attempt to locate the template under the modules
if not ntop.exists(full_path) then
full_path = os_utils.fixPath(dirs.installdir .. "/scripts/lua/modules/check_templates/"..template_file)
end
-- If no template is found attempt to locate the template under the modules
if not ntop.exists(full_path) then
full_path = os_utils.fixPath(dirs.installdir .. "/scripts/lua/modules/check_templates/" .. template_file)
end
return template_utils.gen(full_path, context, true --[[ using full path ]])
return template_utils.gen(full_path, context, true --[[ using full path ]] )
end
-- ##############################################
-- @brief Load an alert template
function script_manager.loadTemplate(script_name, template_file)
-- Checking the standard templates path, '/httpdocs/templates/pages/'
local script_template_path = script_manager.getScriptTemplatesDir(script_name)
local template_path = os_utils.fixPath(script_template_path.."/"..template_file..".lua")
local req = nil
-- Checking the standard templates path, '/httpdocs/templates/pages/'
local script_template_path = script_manager.getScriptTemplatesDir(script_name)
local template_path = os_utils.fixPath(script_template_path .. "/" .. template_file .. ".lua")
local req = nil
-- Templates not found
if ntop.exists(template_path) then
-- Do the necessary require
lua_path_utils.package_path_prepend(RUNTIME_PATHS.templates)
-- Templates not found
if ntop.exists(template_path) then
-- Do the necessary require
lua_path_utils.package_path_prepend(RUNTIME_PATHS.templates)
local req_name = string.format("%s.%s", script_name, template_file)
req = require(req_name)
end
local req_name = string.format("%s.%s", script_name, template_file)
req = require(req_name)
end
return req
return req
end
-- ##############################################
return(script_manager)
function script_manager.getAllRuntimePaths()
return RUNTIME_PATHS
end
-- ##############################################
return (script_manager)

View file

@ -0,0 +1,78 @@
--
-- (C) 2014-24 - ntop.org
--
-- ###############################################
-- removes trailing/leading spaces
function trimString(s)
return (s:gsub("^%s*(.-)%s*$", "%1"))
end
-- ###############################################
-- removes all spaces
function trimSpace(what)
if (what == nil) then
return ("")
end
return (string.gsub(string.gsub(what, "%s+", ""), "+%s", ""))
end
-- ##############################################
-- Note: Regexs are applied by default. Pass plain=true to disable them.
function string.contains(str, start, is_plain)
if type(str) ~= 'string' or type(start) ~= 'string' or isEmptyString(str) or isEmptyString(start) then
return false
end
local i, _ = string.find(str, start, 1, is_plain)
return (i ~= nil)
end
-- ##############################################
function shortenString(name, max_len)
local ellipsis = "\u{2026}" -- The unicode ellipsis (takes less space than three separate dots)
if (name == nil) then
return ("")
end
if max_len == nil then
max_len = ntop.getPref("ntopng.prefs.max_ui_strlen")
max_len = tonumber(max_len)
if (max_len == nil) then
max_len = 24
end
end
-- Error, max_len is not a number, print an error and return the name
if not tonumber(max_len) then
traceError(TRACE_DEBUG, TRACE_CONSOLE, "Length parameter is not a number.")
tprint(debug.traceback())
return name
end
if (string.len(name) < max_len + 1 --[[ The space taken by the ellipsis --]] ) then
return (name)
else
return (string.sub(name, 1, max_len) .. ellipsis)
end
end
-- ##############################################
function string.containsIgnoreCase(str, start, is_plain)
return string.contains(string.lower(str), string.lower(start), is_plain)
end
-- startswith
function startswith(s, char)
return string.sub(s, 1, string.len(s)) == char
end
-- endswith
function endswith(s, char)
return string.sub(s, -#char) == char
end

View file

@ -11,13 +11,14 @@ local host_pools = require "host_pools"
local dscp_consts = require "dscp_consts"
local country_codes = require "country_codes"
local alert_category_utils = require "alert_category_utils"
local consts = require "consts"
local snmp_filter_options_cache
local tag_utils = {}
-- Operator Separator in query strings
tag_utils.SEPARATOR = alert_consts.SEPARATOR
tag_utils.SEPARATOR = consts.SEPARATOR
-- #####################################