Implements flow callbacks and alerts in C++

scripts/lua/modules/user_scripts.lua

@@ -61,7 +61,7 @@ local REQUEST_PERIODIC_USER_SCRIPTS_RUN_KEY = "ntopng.cache.ifid_%i.user_scripts
 local NON_TRAFFIC_ELEMENT_CONF_KEY = "all"
 local NON_TRAFFIC_ELEMENT_ENTITY = "no_entity"
 local ALL_HOOKS_CONFIG_KEY = "all"
-local CONFIGSET_KEY = "ntopng.prefs.user_scripts.configset_v3"
+local CONFIGSET_KEY = "ntopng.prefs.user_scripts.configset_v3" -- Keep in sync with ntop_defines.h FLOW_CALLBACKS_CONFIG
 user_scripts.DEFAULT_CONFIGSET_ID = 0
 
 -- NOTE: the subdir id must be unique
@@ -184,51 +184,26 @@ local available_subdirs = {
       -- User script execution filters (field names are those that arrive from the C Flow.cpp)
       filter = {
 	 -- Default fields populated automatically when creating filters
-	 default_fields   = {"srv_addr", "srv_port", "l7_proto", "proto" },
+	 default_fields   = { "srv_addr", },
 	 -- All possible filter fields
 	 available_fields = {
 	    cli_addr = {
 	       lint = http_lint.validateNetwork,
 	       match = function(context, val)
-		  local client_ip = flow.getClientIp()
-		  -- Attempt exact match
-		  if client_ip == val then return true end
-		  -- Attempt IPv4 network match
-		  local network, netmask = ipv4_utils.cidr_2_addr(val)
-		  if network and netmask then return ipv4_utils.includes(network, netmask, client_ip) end
-		  -- No match
-		  return false
+		  -- NO match, match is done in C++
 	       end,
 	       sqlite = function(val)
 		  -- Keep in sync with SQLite database schema declared in AlertsManager.cpp
-		  return string.format("cli_addr = '%s'", val)
+ 		  return string.format("cli_addr = '%s'", val)
 	       end,
 	       find = function(alert, alert_json, filter, val)
 		  return (alert[filter] and (alert[filter] == val))
 	       end,
 	    },
-	    cli_port = {
-	       lint = http_lint.validatePort,
-	       match = function(context, val) return flow.getClientPort() == tonumber(val) end,
-	       sqlite = function(val)
-		  -- Keep in sync with SQLite database schema declared in AlertsManager.cpp
-		  return string.format("cli_port = %u", val)
-	       end,
-	       find = function(alert, alert_json, filter, val)
-		  return (alert[filter] and (tonumber(alert[filter]) == tonumber(val)))
-	       end,
-	    },
 	    srv_addr = {
 	       lint = http_lint.validateNetwork,
 	       match = function(context, val)
-		  local server_ip = flow.getServerIp()
-		  -- Attempt exact match
-		  if server_ip == val then return true end
-		  -- Attempt IPv4 network match
-		  local network, netmask = ipv4_utils.cidr_2_addr(val)
-		  if network and netmask then return ipv4_utils.includes(network, netmask, server_ip) end
-		  -- No match
-		  return false
+		  -- NO match, match is done in C++
 	       end,
 	       sqlite = function(val)
 		  -- Keep in sync with SQLite database schema declared in AlertsManager.cpp
@@ -238,124 +213,6 @@ local available_subdirs = {
 		  return (alert[filter] and (alert[filter] == val))
 	       end,
 	    },
-	    srv_port = {
-	       lint = http_lint.validatePort,
-	       match = function(context, val) return flow.getServerPort() == tonumber(val) end,
-	       sqlite = function(val)
-		  -- Keep in sync with SQLite database schema declared in AlertsManager.cpp
-		  return string.format("srv_port = %u", val)
-	       end,
-	       find = function(alert, alert_json, filter, val)
-		  return (alert[filter] and (tonumber(alert[filter]) == tonumber(val)))
-	       end,
-	    },
-	    l7_proto = {
-	       lint = http_lint.validateProtocolIdOrName,
-	       match = function(context, val)
-		  -- If val is the application name, then it is converted to application id
-		  if not tonumber(val) then val = interface.getnDPIProtoId(val) end
-		  -- For integers represented as strings
-		  val = tonumber(val)
-		  -- Check for equality on either the master or application ids
-		  return flow.getnDPIMasterProtoId() == val or flow.getnDPIAppProtoId() == val
-	       end,
-	       sqlite = function(val)
-		  -- If val is the application name, then it is converted to application id
-		  if not tonumber(val) then val = interface.getnDPIProtoId(val) end
-		  -- Keep in sync with SQLite database schema declared in AlertsManager.cpp
-		  -- Match both on the master and app proto
-		  return string.format("(l7_proto = %u OR l7_master_proto = %u)", val, val)
-	       end,
-	       find = function(alert, alert_json, filter, val)
-		  -- Converting value into it's id if value is under string format
-		  local value = tonumber(val)
-		  if not value then value = interface.getnDPIProtoId(val) end
-
-		  return (alert[filter] and (tonumber(alert[filter]) == value))
-	       end,
-	    },
-	    proto = {
-	       lint = http_lint.validateProtocolIdOrName,
-	       match = function(context, val)
-		  -- If val is the protocol name, then it is converted to L4 protocol id
-		  if not tonumber(val) then val = l4_proto_to_id(val) end
-		  -- Check for equality on either the master or application protocol
-		  return flow.getProtocol() == tonumber(val)
-	       end,
-	       sqlite = function(val)
-		  -- Keep in sync with SQLite database schema declared in AlertsManager.cpp
-		  return string.format("proto = %u", val)
-	       end,
-	       find = function(alert, alert_json, filter, val)
-		  -- Converting value into it's id if value is under string format
-		  local value = tonumber(val)
-		  if not value then value = l4_proto_to_id(val) end
-		  
-		  return (alert[filter] and (tonumber(alert[filter]) == value))
-	       end,
-	    },
-	    flow_risk_bitmap = {
-	       lint = http_lint.validateNumber,
-	       match = function(context, val)
-		  -- Convert the string-bitmap to a number
-		  val = tonumber(val)
-		  -- Check if there's at least one risk in common between val
-		  -- and the actual flow bitmap of risks
-		  return (val & flow.getRiskBitmap()) ~= 0
-	       end,
-	       sqlite = function(val)
-		  -- Keep in sync with SQLite database schema declared in AlertsManager.cpp
-		  return string.format("flow_risk_bitmap = %u", val)
-	       end,
-	       find = function(alert, alert_json, filter, val)
-		  return (alert[filter] and (tonumber(alert[filter]) == tonumber(val)))
-	       end,
-	    },
-	    info = {
-	       lint = http_lint.validateSingleWord,
-	       match = function(context, val)
-		  -- Search for substring val inside the flow info field
-		  return not not flow.getFlowInfoField():find(val)
-	       end,
-	       sqlite = function(val)
-		  -- Keep in sync with SQLite database schema declared in AlertsManager.cpp
-		  -- As the info is stored inside the JSON alert, it is necessary to
-		  -- use sqlite json_extract to access it
-		  return string.format("json_extract(alert_json, '$.info') like '%%%s%%'", val)
-	       end,
-	       find = function(alert, alert_json, filter, val)
-		  -- Search for substring val inside the flow info field
-		  if alert_json and val then
-		     return (alert_json[filter] and alert_json[filter]:find(val))
-		  end
-		  return false
-	       end,
-	    },
-	    l7_cat = {
-	       lint = http_lint.validateCategory,
-	       match = function(context, val)
-		  -- If val is the application name, then it is converted to application id
-		  if not tonumber(val) then val = interface.getnDPICategoryId(val) end
-		  -- For integers represented as strings
-		  val = tonumber(val)
-		  -- Check for equality on either the master or application ids
-		  return flow.getnDPICategoryId() == val
-	       end,
-	       sqlite = function(val)
-		  -- If val is the application name, then it is converted to application id
-		  if not tonumber(val) then val = interface.getnDPICategoryId(val) end
-		  -- Keep in sync with SQLite database schema declared in AlertsManager.cpp
-		  -- Match both on the master and app proto
-		  return string.format("l7_cat = %u", val)
-	       end,
-	       find = function(alert, alert_json, filter, val)
-		  -- If val is the application name, then it is converted to application id
-		  local value = tonumber(val)
-		  if not value then value = interface.getnDPICategoryId(val) end
-		  
-		  return (alert[filter] and (tonumber(alert[filter]) == value))
-	       end,
-	    },
 	 },
       },
       -- No pools for flows
@@ -789,7 +646,7 @@ local function init_user_script(user_script, mod_fname, full_path, plugin, scrip
    end
 
    -- Expand hooks
-   if(user_script.hooks["all"] ~= nil) then
+   if(user_script.hooks and user_script.hooks["all"] ~= nil) then
       local callback = user_script.hooks["all"]
       user_script.hooks["all"] = nil
 
@@ -834,7 +691,7 @@ local function loadAndCheckScript(mod_fname, full_path, plugin, script_type, sub
       return(nil)
    end
 
-   if(table.empty(user_script.hooks)) then
+   if(subdir ~= "flow" and table.empty(user_script.hooks)) then
       traceError(TRACE_WARNING, TRACE_CONSOLE, string.format("No 'hooks' defined in user script '%s', skipping", mod_fname))
       return(nil)
    end
@@ -951,7 +808,7 @@ function user_scripts.load(ifid, script_type, subdir, options)
 	    -- Checks passed, now load the script information
 
             -- Populate hooks fast lookup table
-            for hook, hook_fn in pairs(user_script.hooks) do
+            for hook, hook_fn in pairs(user_script.hooks or {}) do
 	       -- load previously computed benchmarks (if any)
 	       -- benchmarks are loaded even if their computation is disabled with a do_benchmark ~= true
                if(rv.hooks[hook] == nil) then
@@ -1125,6 +982,9 @@ local function saveConfigset(configset)
    -- Reload the periodic scripts as the configuration has changed
    ntop.reloadPeriodicScripts()
 
+   -- Reload flow callbacks executed in C++
+   ntop.reloadFlowCallbacks()
+
    return true
 end
 
@@ -1137,10 +997,6 @@ local cached_config_set = nil
 function user_scripts.getConfigset()
    if not cached_config_set then
       cached_config_set = json.decode(ntop.getCache(CONFIGSET_KEY))
-
-      if not cached_config_set then
-	 traceError(TRACE_ERROR, TRACE_CONSOLE, string.format("Unable to load a valid configset"))
-      end
    end
 
    return cached_config_set
@@ -1441,23 +1297,16 @@ end
 
 -- @brief Initializes a default configuration for user scripts
 -- @param overwrite If true, a possibly existing configuration is overwritten with default values
-function user_scripts.initDefaultConfig(overwrite)
-   if not overwrite and json.decode(ntop.getCache(CONFIGSET_KEY)) then
-      -- Nothing to do, already initialized
-      return
-   end
-
+function user_scripts.initDefaultConfig()
    local ifid = getSystemInterfaceId()
-   -- Default per user-script configuration
-   local default_conf = {}
-   -- Default per user-script filters
-   local default_filters = {}
 
-   if default_conf then
-      -- Drop possible nested values due to a previous bug
-      default_conf.config = nil
-   end
-   
+   -- Current (possibly not-existing, not yet created configset)
+   local configset = user_scripts.getConfigset() or {}
+   -- Default per user-script configuration
+   local default_conf = configset.config or {}
+   -- Default per user-script filters
+   local default_filters = configset.filters or {}
+
    for type_id, script_type in pairs(user_scripts.script_types) do
       for _, subdir in pairs(script_type.subdirs) do
 	 local scripts = user_scripts.load(ifid, script_type, subdir, {return_all = true})
@@ -1485,25 +1334,31 @@ function user_scripts.initDefaultConfig(overwrite)
 
 	    if usermod.filter and usermod.filter.default_filters then
 	       default_filters[subdir] = default_filters[subdir] or {}
-	       default_filters[subdir][key] = usermod.filter.default_filters
+
+	       if not default_filters[subdir][key] then
+		  -- Do not override filter of an existing configuration
+		  default_filters[subdir][key] = usermod.filter.default_filters
+	       end
 	    end
 	 end
       end
    end
    
+   -- This is the new configset with all defaults
    local configset = {
       config = default_conf,
       filters = default_filters,
    }
 
-   saveConfigset(configset)
+   saveConfigset(configset)  
 end
 
 -- ##############################################
 
 function user_scripts.resetConfigset()
    cached_config_set = nil
-   user_scripts.initDefaultConfig(true --[[ Overwrite a possibly existing configuration --]])
+   ntop.delCache(CONFIGSET_KEY)
+   user_scripts.initDefaultConfig()
 
    return(true)
 end
@@ -1567,7 +1422,7 @@ function user_scripts.getScriptConfig(configset, script, subdir)
    local script_type = user_scripts.getScriptType(subdir)
    local hooks = ternary(script_type.has_per_hook_config, script.hooks, {[ALL_HOOKS_CONFIG_KEY]=1})
 
-   for hook in pairs(script.hooks) do
+   for hook in pairs(script.hooks or {}) do
       rv[hook] = default_config
    end
 
@@ -1659,18 +1514,9 @@ function user_scripts.getFilterPreset(alert, alert_info)
       return ''
    end
 
-   local script_key       = alert_generation["script_key"]
    local subdir           = alert_generation["subdir"]
-   local filter_string    = ''
-   local script_type      = user_scripts.getScriptType(subdir)
-   local script           = user_scripts.loadModule(interface.getId(), script_type, subdir, script_key)
-   local filter_to_use   = {}
    local subdir_id        = getSubdirId(subdir)
 
-   if not script then
-      return ''
-   end
-
    if subdir_id == -1 then
       return ''
    end
@@ -1679,11 +1525,9 @@ function user_scripts.getFilterPreset(alert, alert_info)
       return ''
    end
 
-   -- Checking if the script has default filter fields or not
-   -- if not, getting the default for the subdir
-   if script["filter"] and script["filter"]["default_fields"] then
-      filter_to_use = script["filter"]["default_fields"]
-   elseif available_subdirs[subdir_id]["filter"]["default_fields"] then
+   local filter_to_use = {}
+
+   if available_subdirs[subdir_id]["filter"]["default_fields"] then
       filter_to_use = available_subdirs[subdir_id]["filter"]["default_fields"]
    end