Merge branch 'dev' into add-scan

This commit is contained in:
Nicolo Maio 2023-07-20 15:24:08 +00:00
commit 217df4b012
51 changed files with 12175 additions and 10945 deletions

View file

@ -56,9 +56,11 @@ function alert_longlived.format(ifid, alert, alert_type_params)
end
end
local alert_json = json.decode(alert["json"])
if alert_json and not isEmptyString(alert_json["info"]) then
res = string.format("%s [%s]", res, alert_json["info"])
if not isEmptyString(alert["json"]) then
local alert_json = json.decode(alert["json"]) or {}
if not isEmptyString(alert_json["info"]) then
res = string.format("%s [%s]", res, alert_json["info"])
end
end
return res

View file

@ -828,6 +828,15 @@ end
-- ##############################################
function alert_store:set_order_by(sort_column, sort_order)
self._order_by = {
sort_column = sort_column,
sort_order = sort_order
}
end
-- ##############################################
-- @brief Specify the sort criteria of the query
-- @param sort_column The column to be used for sorting
-- @param sort_order Order, either `asc` or `desc`
@ -848,10 +857,7 @@ function alert_store:add_order_by(sort_column, sort_order)
-- Creating the order by if not defined and valid
if not self._order_by and sort_column and self:_valid_fields(sort_column) and
(sort_order == "asc" or sort_order == "desc") then
self._order_by = {
sort_column = sort_column,
sort_order = sort_order
}
self:set_order_by(sort_column, sort_order)
return true
end
@ -1743,6 +1749,54 @@ function alert_store:select_request(filter, select_fields, download --[[ Availab
-- Add limits and sort criteria only after the count has been done
self:add_request_ranges()
-- Handle Custom Queries (query_preset)
local p = _GET["query_preset"] -- Example: &query_preset=contacts
if not isEmptyString(p) and ntop.isEnterpriseL() then
package.path = dirs.installdir .. "/scripts/lua/pro/modules/?.lua;" .. package.path
local db_query_presets = require "db_query_presets"
local query_presets = db_query_presets.get_presets(
os_utils.fixPath(dirs.installdir .. "/scripts/historical/alerts/" .. self._alert_entity.alert_store_name)
)
if query_presets[p] then
local preset = query_presets[p]
-- Select fields
if not isEmptyString(preset.select.sql) then
select_fields = preset.select.sql
end
-- Filters
if preset.filters and not isEmptyString(preset.filters.sql) then
filter = preset.filters.sql -- append to where
end
-- Group by fields
if not isEmptyString(preset.groupby.sql) then
self:group_by(preset.groupby.sql)
end
-- Sort by field
if #preset.sortby.items > 0 then
if not self._order_by or
not self._order_by.sort_column or
(not table.contains(preset.groupby.items,
self._order_by.sort_column,
(function(n) return n.name == self._order_by.sort_column end))
and not table.contains(preset.select.items,
self._order_by.sort_column,
(function(n) return n.func and n.name == self._order_by.sort_column end))) then
-- No order by column or invalid column, using default from preset
self:set_order_by(
preset.sortby.items[1].name,
preset.sortby.items[1].order
)
end
end
end
end
local res, info =
self:select_historical(filter, select_fields, download --[[ Available only with ClickHouse ]] )
@ -2014,11 +2068,21 @@ function alert_store:format_json_record_common(value, entity_id)
record[BASE_RNAME.USER_LABEL.name] = value["user_label"]
record[BASE_RNAME.DURATION.name] = tonumber(value["duration"]) or
(tonumber(value["tstamp_end"]) - tonumber(value["tstamp"]))
if tonumber(value["duration"]) then
record[BASE_RNAME.DURATION.name] = tonumber(value["duration"])
elseif tonumber(value["tstamp_end"]) and tonumber(value["tstamp"]) then
record[BASE_RNAME.DURATION.name] = (tonumber(value["tstamp_end"]) - tonumber(value["tstamp"]))
else
record[BASE_RNAME.DURATION.name] = 0 -- unable to compute
end
record[BASE_RNAME.COUNT.name] = tonumber(value["count"]) or 1
local alert_json = json.decode(value["json"]) or {}
local alert_json = {}
if not isEmptyString(value["json"]) then
alert_json = json.decode(value["json"]) or {}
end
record[BASE_RNAME.SCRIPT_KEY.name] = alert_json["alert_generation"] and alert_json["alert_generation"]["script_key"]
return record

View file

@ -638,8 +638,13 @@ local RNAME = {
FLOW_RELATED_INFO = { name = "flow_related_info", export = true },
MSG = { name = "msg", export = true, elements = {"name", "value", "description"}},
FLOW = { name = "flow", export = true, elements = {"srv_ip.label", "srv_ip.value", "srv_port", "cli_ip.label", "cli_ip.value", "cli_port"}},
VLAN_ID = { name = "vlan_id", export = true},
CLI_IP = { name = "cli_ip", export = false},
SRV_IP = { name = "srv_ip", export = false},
CLI_PORT = { name = "cli_port", export = false},
SRV_PORT = { name = "srv_port", export = false},
PROTO = { name = "proto", export = true},
L7_PROTO = { name = "l7_proto", export = true},
LINK_TO_PAST_FLOWS = { name = "link_to_past_flows", export = false},
@ -680,20 +685,26 @@ function flow_alert_store:format_record(value, no_html)
local record = self:format_json_record_common(value, alert_entities.flow.entity_id, no_html)
local alert_info = alert_utils.getAlertInfo(value)
local alert_name = alert_consts.alertTypeLabel(tonumber(value["alert_id"]), true --[[ no_html --]], alert_entities.flow.entity_id)
local alert_risk = ntop.getFlowAlertRisk(tonumber(value.alert_id))
local l4_protocol = l4_proto_to_string(value["proto"])
local l7_protocol = interface.getnDPIFullProtoName(tonumber(value["l7_master_proto"]), tonumber(value["l7_proto"]))
local show_cli_port = (value["cli_port"] ~= '' and value["cli_port"] ~= '0')
local show_srv_port = (value["srv_port"] ~= '' and value["srv_port"] ~= '0')
local msg = alert_utils.formatFlowAlertMessage(interface.getId(), value, alert_info)
local active_url = ""
local attacker = ""
local victim = ""
local alert_json = {}
if not isEmptyString(value.json) then
alert_json = json.decode(value.json) or {}
end
-- Add link to active flow
local alert_json = json.decode(value.json)
local flow_related_info = addExtraFlowInfo(alert_json, value)
local alert_risk
if tonumber(value.alert_id) then
alert_risk = ntop.getFlowAlertRisk(tonumber(value.alert_id))
end
-- TLS IssuerDN
local flow_tls_issuerdn = nil
if alert_risk and alert_risk > 0 and
@ -719,7 +730,9 @@ function flow_alert_store:format_record(value, no_html)
local other_alerts_by_score = {} -- Table used to keep messages ordered by score
local additional_alerts = {}
other_alerts_by_score, additional_alerts = alert_utils.format_other_alerts(value.alerts_map, value['alert_id'])
if value.alerts_map then
other_alerts_by_score, additional_alerts = alert_utils.format_other_alerts(value.alerts_map, value['alert_id'])
end
-- Print additional issues, sorted by score
record[RNAME.ADDITIONAL_ALERTS.name] = ''
@ -738,10 +751,6 @@ function flow_alert_store:format_record(value, no_html)
end
end
-- Handle VLAN as a separate field
local cli_ip = value["cli_ip"]
local srv_ip = value["srv_ip"]
local shorten_msg
record[RNAME.ADDITIONAL_ALERTS.name] = {
@ -757,7 +766,8 @@ function flow_alert_store:format_record(value, no_html)
shorten_descr = shorten_msg,
}
end
local proto = string.lower(interface.getnDPIProtoName(tonumber(value["l7_master_proto"])))
-- local proto = string.lower(interface.getnDPIProtoName(tonumber(value["l7_master_proto"])))
local flow_server_name = getExtraFlowInfoServerName(alert_json)
local flow_domain
@ -826,9 +836,11 @@ function flow_alert_store:format_record(value, no_html)
-- Format Client
local cli_ip = value["cli_ip"]
local reference_html = ""
if not no_html then
reference_html = hostinfo2detailshref({ip = value["cli_ip"], value["vlan_id"]}, nil, href_icon, "", true, nil, false)
if not isEmptyString(cli_ip) and not no_html then
reference_html = hostinfo2detailshref({ip = cli_ip, value["vlan_id"]}, nil, href_icon, "", true, nil, false)
if reference_html == href_icon then
reference_html = ""
end
@ -837,7 +849,7 @@ function flow_alert_store:format_record(value, no_html)
-- In case no country is found, let's check if the host is in memory and retrieve country info
local country = value["cli_country"]
if isEmptyString(country) or country == "nil" then
if (isEmptyString(country) or country == "nil") and not isEmptyString(cli_ip) then
local host_info = interface.getHostMinInfo(cli_ip)
if host_info then
country = host_info["country"] or ""
@ -850,8 +862,9 @@ function flow_alert_store:format_record(value, no_html)
}
local flow_cli_ip = {
value = cli_ip,
label = cli_ip,
value = cli_ip or "",
ip = cli_ip or "",
label = cli_ip or "",
reference = reference_html,
country = country,
blacklisted = value["cli_blacklisted"]
@ -861,25 +874,34 @@ function flow_alert_store:format_record(value, no_html)
flow_cli_ip["name"] = value["cli_name"]
end
local label = hostinfo2label(self:_alert2hostinfo(value, true --[[ As client --]]), false --[[ Show VLAN --]], false --[[ Shorten --]], true --[[ Skip Resolution ]])
-- Shortened label if necessary for UI purposes
flow_cli_ip["label"] = label
flow_cli_ip["label_long"] = label
if not isEmptyString(cli_ip) then
local label = hostinfo2label(self:_alert2hostinfo(value,
true --[[ As client --]]),
false --[[ Show VLAN --]],
false --[[ Shorten --]],
true --[[ Skip Resolution ]])
-- Shortened label if necessary for UI purposes
flow_cli_ip["label"] = label
flow_cli_ip["label_long"] = label
end
-- Format Server
local srv_ip = value["srv_ip"]
reference_html = ""
if not no_html then
reference_html = hostinfo2detailshref({ip = value["srv_ip"], vlan = value["vlan_id"]}, nil, href_icon, "", true)
if not no_html and not isEmptyString(srv_ip) then
reference_html = hostinfo2detailshref({ip = srv_ip, vlan = value["vlan_id"]}, nil, href_icon, "", true)
if reference_html == href_icon then
reference_html = ""
reference_html = ""
end
end
-- In case no country is found, let's check if the host is in memory and retrieve country info
country = value["srv_country"]
if isEmptyString(country) or country == "nil" then
if (isEmptyString(country) or country == "nil") and not isEmptyString(srv_ip) then
local host_info = interface.getHostMinInfo(srv_ip)
if host_info then
country = host_info["country"] or ""
@ -887,8 +909,9 @@ function flow_alert_store:format_record(value, no_html)
end
local flow_srv_ip = {
value = srv_ip,
label = srv_ip,
value = srv_ip or "",
ip = srv_ip or "",
label = srv_ip or "",
reference = reference_html,
country = country,
blacklisted = value["srv_blacklisted"]
@ -898,15 +921,24 @@ function flow_alert_store:format_record(value, no_html)
flow_srv_ip["name"] = value["srv_name"]
end
label = hostinfo2label(self:_alert2hostinfo(value, false --[[ As server --]]), false --[[ Show VLAN --]], false --[[ Shorten --]], true --[[ Skip Resolution ]])
-- Shortened label if necessary for UI purposes
flow_srv_ip["label"] = label
flow_srv_ip["label_long"] = label
if not isEmptyString(srv_ip) then
local label = hostinfo2label(self:_alert2hostinfo(value,
false --[[ As server --]]),
false --[[ Show VLAN --]],
false --[[ Shorten --]],
true --[[ Skip Resolution ]])
local flow_cli_port = value["cli_port"]
local flow_srv_port = value["srv_port"]
-- Shortened label if necessary for UI purposes
flow_srv_ip["label"] = label
flow_srv_ip["label_long"] = label
end
local vlan = {
label = "",
title = "",
value = 0
}
local vlan
if value["vlan_id"] and tonumber(value["vlan_id"]) ~= 0 then
vlan = {
label = getFullVlanName(value["vlan_id"], true --[[ Compact --]]),
@ -915,19 +947,38 @@ function flow_alert_store:format_record(value, no_html)
}
end
local flow_cli_port = {
value = value["cli_port"],
}
local flow_srv_port = {
value = value["srv_port"],
}
-- Used to render custom queries (compatible with historical flows columns definition)
record[RNAME.CLI_IP.name] = flow_cli_ip
record[RNAME.SRV_IP.name] = flow_srv_ip
record[RNAME.CLI_PORT.name] = flow_cli_port
record[RNAME.SRV_PORT.name] = flow_srv_port
record[RNAME.VLAN_ID.name] = vlan
-- Used to render the flow column in raw alerts
record[RNAME.FLOW.name] = {
vlan = vlan,
cli_ip = flow_cli_ip,
srv_ip = flow_srv_ip,
cli_port = flow_cli_port,
srv_port = flow_srv_port,
cli_port = value["cli_port"],
srv_port = value["srv_port"],
active_url = active_url,
}
record[RNAME.VLAN_ID.name] = value["vlan_id"]
local l4_protocol
if not isEmptyString(value["proto"]) then
l4_protocol = l4_proto_to_string(value["proto"])
end
record[RNAME.PROTO.name] = {
value = value["proto"],
label = l4_protocol
value = value["proto"] or "",
label = l4_protocol or ""
}
if value["is_cli_victim"] == "1" then record["cli_role"] = { value = 'victim', label = i18n("victim"), tag_label = i18n("victim") } end
@ -935,23 +986,29 @@ function flow_alert_store:format_record(value, no_html)
if value["is_srv_victim"] == "1" then record["srv_role"] = { value = 'victim', label = i18n("victim"), tag_label = i18n("victim") } end
if value["is_srv_attacker"] == "1" then record["srv_role"] = { value = 'attacker', label = i18n("attacker"), tag_label = i18n("attacker") } end
local l7_protocol
if tonumber(value["l7_master_proto"]) and tonumber(value["l7_proto"]) then
l7_protocol = interface.getnDPIFullProtoName(tonumber(value["l7_master_proto"]), tonumber(value["l7_proto"]))
end
-- Check the two labels, otherwise an ICMP:ICMP label could be possible
local proto_label = l7_protocol
if l4_protocol ~= l7_protocol then
proto_label = l4_protocol..":"..l7_protocol
if l4_protocol and l7_protocol and l4_protocol ~= l7_protocol then
proto_label = l4_protocol..":"..l7_protocol
end
record[RNAME.L7_PROTO.name] = {
value = ternary(tonumber(value["l7_proto"]) ~= 0, value["l7_proto"], value["l7_master_proto"]),
l4_label = l4_protocol,
l7_label = l7_protocol,
label = proto_label,
l4_label = l4_protocol or "",
l7_label = l7_protocol or "",
label = proto_label or "",
confidence = format_confidence_from_json(value)
}
-- Add link to historical flow
if ntop.isEnterpriseM() and hasClickHouseSupport() and not no_html then
if ntop.isEnterpriseM() and hasClickHouseSupport() and not no_html
and tonumber(value["tstamp"]) and tonumber(value["tstamp_end"]) then
local op_suffix = tag_utils.SEPARATOR .. 'eq'
local href = string.format('%s/lua/pro/db_search.lua?epoch_begin=%u&epoch_end=%u&cli_ip=%s%s&srv_ip=%s%s&cli_port=%s%s&srv_port=%s%s&l4proto=%s%s',
ntop.getHttpPrefix(),
@ -961,7 +1018,7 @@ function flow_alert_store:format_record(value, no_html)
value["srv_ip"], op_suffix,
ternary(show_cli_port, tostring(value["cli_port"]), ''), op_suffix,
ternary(show_srv_port, tostring(value["srv_port"]), ''), op_suffix,
l4_protocol, op_suffix)
l4_protocol or "", op_suffix)
if vlan then
href = href .. string.format('&vlan_id=%s%s', vlan.value, op_suffix)
@ -970,20 +1027,47 @@ function flow_alert_store:format_record(value, no_html)
record[RNAME.LINK_TO_PAST_FLOWS.name] = href
end
-- Add BPF filter
-- Add BPF filter (for PCAP extractions)
-- and Tag filters (e.g. to jump from custom queries to raw alerts)
record['filter'] = {}
local rules = {}
rules[#rules+1] = 'host ' .. value["cli_ip"]
rules[#rules+1] = 'host ' .. value["srv_ip"]
if value["cli_port"] and tonumber(value["cli_port"]) > 0 then
local filters = {}
local op_suffix = 'eq'
if not isEmptyString(value["alert_id"]) and tonumber(value["alert_id"]) > 0 then
filters[#filters+1] = { id = "alert_id", value = value["alert_id"], op = op_suffix }
end
if not isEmptyString(value["vlan_id"]) and tonumber(value["vlan_id"]) > 0 then
filters[#filters+1] = { id = "vlan_id", value = value["vlan_id"], op = op_suffix }
end
if not isEmptyString(value["cli_ip"]) then
rules[#rules+1] = 'host ' .. value["cli_ip"]
filters[#filters+1] = { id = "cli_ip", value = value["cli_ip"], op = op_suffix }
end
if not isEmptyString(value["srv_ip"]) then
rules[#rules+1] = 'host ' .. value["srv_ip"]
filters[#filters+1] = { id = "srv_ip", value = value["srv_ip"], op = op_suffix }
end
if not isEmptyString(value["cli_port"]) and tonumber(value["cli_port"]) > 0 then
rules[#rules+1] = 'port ' .. tostring(value["cli_port"])
filters[#filters+1] = { id = "cli_port", value = value["cli_port"], op = op_suffix }
end
if not isEmptyString(value["srv_port"]) and tonumber(value["srv_port"]) > 0 then
rules[#rules+1] = 'port ' .. tostring(value["srv_port"])
filters[#filters+1] = { id = "srv_port", value = value["srv_port"], op = op_suffix }
end
if not isEmptyString(value["info"]) then
filters[#filters+1] = { id = "info", value = value["info"], op = op_suffix }
end
record['filter'] = {
epoch_begin = tonumber(value["tstamp"]),
epoch_end = tonumber(value["tstamp_end"]) + 1,
bpf = table.concat(rules, " and "),
}
if #rules > 0 and tonumber(value["tstamp"]) and tonumber(value["tstamp_end"]) then
record['filter'].epoch_begin = tonumber(value["tstamp"])
record['filter'].epoch_end = tonumber(value["tstamp_end"]) + 1
record['filter'].bpf = table.concat(rules, " and ")
end
record['filter'].tag_filters = filters
local probe_ip = ''
local probe_label = ''

View file

@ -367,9 +367,13 @@ end
function alert_utils.formatFlowAlertMessage(ifid, alert, alert_json, add_score)
local msg
local alert_risk = ntop.getFlowAlertRisk(tonumber(alert.alert_id))
local alert_risk
if (alert_json == nil) then
if tonumber(alert.alert_id) then
alert_risk = ntop.getFlowAlertRisk(tonumber(alert.alert_id))
end
if not alert_json then
alert_json = alert_utils.getAlertInfo(alert)
end
@ -391,14 +395,15 @@ function alert_utils.formatFlowAlertMessage(ifid, alert, alert_json, add_score)
msg = string.format('%s <small><span class="text-muted">%s</span></small>', msg, alert["user_label"])
end
local alert_score = ntop.getFlowAlertScore(tonumber(alert.alert_id))
if add_score then
msg = alert_utils.format_score(msg, alert_score)
end
if tonumber(alert.alert_id) then
local alert_score = ntop.getFlowAlertScore(tonumber(alert.alert_id))
msg = alert_utils.format_score(msg, alert_score)
end
end
-- Add the link to the documentation
if alert_risk > 0 then
if alert_risk and alert_risk > 0 then
msg = string.format("%s %s", msg, flow_risk_utils.get_documentation_link(alert_risk))
local info_msg = alert_utils.get_flow_risk_info(alert_risk, alert_json)
@ -422,117 +427,61 @@ function alert_utils.getLinkToPastFlows(ifid, alert, alert_json)
-- Fetch the alert id
local alert_id = alert_consts.getAlertType(alert.alert_id, alert.entity_id)
if alert_id then
-- If there's a function that generates the filter available for this particular alert,
-- then the function is called to fetch the filter for the historical flows
if alert_consts.alert_types[alert_id].filter_to_past_flows then
if not alert_json then
alert_json = alert_utils.getAlertInfo(alert)
local final_filter = {}
local filters = {}
local epoch_begin = alert["tstamp"]
local epoch_end = alert["tstamp_end"]
-- Look a bit around the epochs
epoch_begin = epoch_begin - (5 * 60)
epoch_end = epoch_end + (5 * 60)
-- IP
if not isEmptyString(alert["ip"]) then
filters[#filters + 1] = {
name = "ip",
op = "eq",
val = alert["ip"]
}
-- Add the hostname here cause it's needed to check if the ip is equal to the name
-- Hostname
if not isEmptyString(alert["name"]) and (alert["ip"] ~= alert["name"]) then
filters[#filters + 1] = {
name = "name",
op = "eq",
val = alert["name"]
}
end
-- Fetch the filter
local past_flows_filter = alert_consts.alert_types[alert_id].filter_to_past_flows(ifid, alert, alert_json)
local epoch_begin, epoch_end
-- Add a default start time, if no start time has been added by the filter-generation function
if not past_flows_filter["epoch_begin"] then
past_flows_filter["epoch_begin"] = tonumber(alert["tstamp"])
end
epoch_begin = tonumber(past_flows_filter["epoch_begin"])
past_flows_filter["epoch_begin"] = nil
-- Add a default end time, if not end time has been added by the filter-generation function
if not past_flows_filter["epoch_end"] then
local duration = tonumber(alert["duration"]) or
(tonumber(alert["tstamp_end"]) - tonumber(alert["tstamp"]))
past_flows_filter["epoch_end"] = epoch_begin + duration
end
epoch_end = tonumber(past_flows_filter["epoch_end"])
past_flows_filter["epoch_end"] = nil
local tags = {}
for name, val in pairs(past_flows_filter) do
local filter_op, filter_val
if name == "epoch_end" or name == "epoch_begin" then
-- They are not tags, they will be inserted as-is
goto continue
elseif val == true then
-- Assumes > 0
tags[#tags + 1] = {
name = name,
op = "gt",
val = "0"
}
elseif string.contains(name, "ip") then
-- Unpack the hostkey into two separate fields, one for the VLAN (if present and positive) and one for the IP
local host_info = hostkey2hostinfo(val)
tags[#tags + 1] = {
name = name,
op = "eq",
val = host_info["host"]
}
if host_info["vlan"] > 0 then
tags[#tags + 1] = {
name = "vlan_id",
op = "eq",
val = tostring(host_info["vlan"])
}
end
elseif string.contains(name, "tcp_flags") then
-- Assumes IN query
if val >= 0 then
-- Assumes IN
tags[#tags + 1] = {
name = name,
op = "in",
val = tostring(val)
}
else
-- A negative value assumes NOT IN
tags[#tags + 1] = {
name = name,
op = "nin",
val = tostring(-val)
}
end
else
-- Fallback, assume equality
tags[#tags + 1] = {
name = name,
op = "eq",
val = tostring(val)
}
end
::continue::
end
-- Look a bit around the epochs...
epoch_begin = epoch_begin - (5 * 60)
epoch_end = epoch_end + (5 * 60)
-- ... but not too much
if epoch_end - epoch_begin > 600 then
epoch_end = epoch_begin + 600
end
-- Join the TAG filters using the predefined operator
local final_filter = {}
for _, tag in pairs(tags) do
final_filter[tag.name] = string.format("%s%s%s", tag.val, alert_consts.SEPARATOR, tag.op)
end
-- tprint({formatEpoch(epoch_begin), formatEpoch(epoch_end), formatEpoch(tonumber(alert.tstamp)), formatEpoch(tonumber(alert.tstamp_end))})
-- Return the link augmented with the filter
local res = string.format("%s/lua/pro/db_search.lua?epoch_begin=%u&epoch_end=%u&%s", ntop.getHttpPrefix(),
epoch_begin, epoch_end, table.tconcat(final_filter, "=", "&"))
return res
end
-- VLAN ID
if not isEmptyString(alert["vlan_id"]) and tonumber(alert["vlan_id"]) > 0 then
filters[#filters + 1] = {
name = "vlan_id",
op = "eq",
val = alert["vlan_id"]
}
end
-- Host alerts could have a custom function to format the url, in case call it
-- and then merge the filters
if alert_consts.alert_types[alert_id].filter_to_past_flows then
local past_flows_filter = alert_consts.alert_types[alert_id].filter_to_past_flows(ifid, alert, alert_json)
table.merge(filters, past_flows_filter)
end
for _, tag in pairs(filters) do
final_filter[tag.name] = string.format("%s%s%s", tag.val, alert_consts.SEPARATOR, tag.op)
end
-- Return the link augmented with the filter
local res = string.format("%s/lua/pro/db_search.lua?epoch_begin=%u&epoch_end=%u&%s", ntop.getHttpPrefix(),
epoch_begin, epoch_end, table.tconcat(final_filter, "=", "&"))
return res
end
return nil
end
-- #################################

View file

@ -56,4 +56,368 @@ function datatable_utils.has_saved_column_preferences(table_name)
return not isEmptyString(columns)
end
------------------------------------------------------------------------
-- DataTable columns definitions (JSON)
-- #####################################
local function build_datatable_column_def_default(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
}
end
-- #####################################
local function build_datatable_column_def_number(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap" },
render_type = "full_number",
}
end
-- #####################################
local function build_datatable_column_def_ip(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_port(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_flow(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "text-nowrap" },
render_type = "formatFlowTuple",
}
end
-- #####################################
local function build_datatable_column_def_nw_latency(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap" },
}
end
-- #####################################
local function build_datatable_column_def_asn(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
}
end
-- #####################################
local function build_datatable_column_def_snmp_interface(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_network(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_pool_id(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_country(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_community_id(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_packets(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap", "text-center" },
render_type = "full_number",
}
end
-- #####################################
local function build_datatable_column_def_bytes(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap" },
render_type = "bytes",
}
end
-- #####################################
local function build_datatable_column_def_tcp_flags(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_dscp(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_float(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap", "text-center" },
}
end
-- #####################################
local function build_datatable_column_def_msec(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap" },
render_type = "ms",
}
end
-- #####################################
local all_datatable_columns_def_by_tag = {
['first_seen'] = {
title_i18n = "db_search.first_seen",
data_field = "first_seen",
sortable = true,
class = { "no-wrap" },
},
['last_seen'] = {
title_i18n = "db_search.last_seen",
data_field = "last_seen",
sortable = true,
class = { "no-wrap" },
},
['l4proto'] = {
title_i18n = "db_search.l4proto",
data_field = "l4proto",
sortable = true,
class = { "no-wrap" },
render_generic = "l4proto",
},
['l7proto'] = {
title_i18n = "db_search.l7proto",
data_field = "l7proto",
sortable = true,
class = { "no-wrap" },
},
['score'] = {
title_i18n = "score",
data_field = "score",
sortable = true,
class = { "no-wrap" },
render_type = "formatValueLabel",
},
["flow"] = build_datatable_column_def_flow("flow", "flow"),
['vlan_id'] = {
title_i18n = "db_search.vlan_id",
data_field = "vlan_id",
sortable = true,
class = { "no-wrap" },
render_generic = "vlan_id",
},
['cli_ip'] = build_datatable_column_def_ip('cli_ip', "db_search.client"),
['srv_ip'] = build_datatable_column_def_ip('srv_ip', "db_search.server"),
['cli_port'] = build_datatable_column_def_port('cli_port', "db_search.cli_port"),
['srv_port'] = build_datatable_column_def_port('srv_port', "db_search.srv_port"),
['packets'] = build_datatable_column_def_packets('packets', "db_search.packets"),
['bytes'] = build_datatable_column_def_bytes('bytes', "db_search.bytes"),
['throughput'] = {
title_i18n = "db_search.throughput",
data_field = "throughput",
sortable = true,
class = { "no-wrap" },
},
['cli_asn'] = build_datatable_column_def_asn('cli_asn', "db_search.cli_asn"),
['srv_asn'] = build_datatable_column_def_asn('srv_asn', "db_search.srv_asn"),
['l7cat'] = {
title_i18n = "db_search.l7cat",
data_field = "l7cat",
sortable = true,
class = { "no-wrap" },
render_generic = "l7cat",
},
['alert_id'] = {
title_i18n = "db_search.alert_id",
data_field = "alert_id",
sortable = true,
class = { "no-wrap" },
render_generic = "alert_id",
},
['flow_risk'] = {
title_i18n = "db_search.flow_risk",
data_field = "flow_risk",
sortable = true,
class = { "no-wrap" },
},
['src2dst_tcp_flags'] = build_datatable_column_def_tcp_flags('src2dst_tcp_flags', "db_search.src2dst_tcp_flags"),
['dst2src_tcp_flags'] = build_datatable_column_def_tcp_flags('dst2src_tcp_flags', "db_search.dst2src_tcp_flags"),
['src2dst_dscp'] = build_datatable_column_def_dscp('src2dst_dscp', "db_search.src2dst_dscp"),
['dst2src_dscp'] = build_datatable_column_def_dscp('dst2src_dscp', "db_search.dst2src_dscp"),
['cli_nw_latency'] = build_datatable_column_def_nw_latency('cli_nw_latency', "db_search.cli_nw_latency"),
['srv_nw_latency'] = build_datatable_column_def_nw_latency('srv_nw_latency', "db_search.srv_nw_latency"),
['info'] = {
title_i18n = "db_search.info",
data_field = "info",
sortable = true,
class = { "no-wrap" },
},
['observation_point_id'] = {
title_i18n = "db_search.observation_point_id",
data_field = "observation_point_id",
sortable = true,
class = { "no-wrap" },
render_generic = "observation_point_id",
},
['probe_ip'] = {
title_i18n = "db_search.probe_ip",
data_field = "probe_ip",
sortable = true,
class = { "no-wrap" },
render_type = "formatProbeIP",
},
['cli_network'] = build_datatable_column_def_network('cli_network', "db_search.tags.cli_network"),
['srv_network'] = build_datatable_column_def_network('srv_network', "db_search.tags.srv_network"),
['cli_host_pool_id'] = build_datatable_column_def_pool_id('cli_host_pool_id', "db_search.tags.cli_host_pool_id"),
['srv_host_pool_id'] = build_datatable_column_def_pool_id('srv_host_pool_id', "db_search.tags.srv_host_pool_id"),
["input_snmp"] = build_datatable_column_def_snmp_interface("input_snmp", "db_search.tags.input_snmp"),
["output_snmp"] = build_datatable_column_def_snmp_interface("output_snmp", "db_search.tags.output_snmp"),
['cli_country'] = build_datatable_column_def_country('cli_country', "db_search.tags.cli_country"),
['srv_country'] = build_datatable_column_def_country('srv_country', "db_search.tags.srv_country"),
['community_id'] = build_datatable_column_def_community_id('community_id', "db_search.tags.community_id"),
}
-- #####################################
datatable_utils.datatable_column_def_builder_by_type = {
['default'] = build_datatable_column_def_default,
['number'] = build_datatable_column_def_number,
['ip'] = build_datatable_column_def_ip,
['port'] = build_datatable_column_def_port,
['asn'] = build_datatable_column_def_asn,
['tcp_flags'] = build_datatable_column_def_tcp_flags,
['dscp'] = build_datatable_column_def_dscp,
['packets'] = build_datatable_column_def_packets,
['bytes'] = build_datatable_column_def_bytes,
['float'] = build_datatable_column_def_float,
['msec'] = build_datatable_column_def_msec,
['network'] = build_datatable_column_def_network,
['pool_id'] = build_datatable_column_def_pool_id,
['country'] = build_datatable_column_def_country,
['snmp_interface'] = build_datatable_column_def_snmp_interface,
}
-- #####################################
function datatable_utils.get_datatable_column_def_by_tag(tag)
if all_datatable_columns_def_by_tag[tag] then
return all_datatable_columns_def_by_tag[tag]
else
return build_datatable_column_def_default(tag, i18n("db_search.tags."..tag) or tag)
end
end
return datatable_utils

View file

@ -1460,371 +1460,6 @@ function historical_flow_utils.format_clickhouse_record(record, csv_format, form
return processed_record
end
------------------------------------------------------------------------
-- DataTable columns definitions (JSON)
-- #####################################
local function build_datatable_column_def_default(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
}
end
-- #####################################
local function build_datatable_column_def_number(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap" },
render_type = "full_number",
}
end
-- #####################################
local function build_datatable_column_def_ip(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_port(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_flow(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "text-nowrap" },
render_type = "formatFlowTuple",
}
end
-- #####################################
local function build_datatable_column_def_nw_latency(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap" },
}
end
-- #####################################
local function build_datatable_column_def_asn(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
}
end
-- #####################################
local function build_datatable_column_def_snmp_interface(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_network(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_pool_id(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_country(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_community_id(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = false,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_packets(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap", "text-center" },
render_type = "full_number",
}
end
-- #####################################
local function build_datatable_column_def_bytes(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap" },
render_type = "bytes",
}
end
-- #####################################
local function build_datatable_column_def_tcp_flags(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_dscp(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
class = { "no-wrap" },
render_generic = name,
}
end
-- #####################################
local function build_datatable_column_def_float(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap", "text-center" },
}
end
-- #####################################
local function build_datatable_column_def_msec(name, i18n_label)
return {
data_field = name,
title_i18n = i18n_label,
sortable = true,
style = "text-align:right;",
class = { "no-wrap" },
render_type = "ms",
}
end
-- #####################################
historical_flow_utils.datatable_column_def_builder_by_type = {
['default'] = build_datatable_column_def_default,
['number'] = build_datatable_column_def_number,
['ip'] = build_datatable_column_def_ip,
['port'] = build_datatable_column_def_port,
['asn'] = build_datatable_column_def_asn,
['tcp_flags'] = build_datatable_column_def_tcp_flags,
['dscp'] = build_datatable_column_def_dscp,
['packets'] = build_datatable_column_def_packets,
['bytes'] = build_datatable_column_def_bytes,
['float'] = build_datatable_column_def_float,
['msec'] = build_datatable_column_def_msec,
['network'] = build_datatable_column_def_network,
['pool_id'] = build_datatable_column_def_pool_id,
['country'] = build_datatable_column_def_country,
['snmp_interface'] = build_datatable_column_def_snmp_interface,
}
-- #####################################
local all_datatable_columns_def_by_tag = {
['first_seen'] = {
title_i18n = "db_search.first_seen",
data_field = "first_seen",
sortable = true,
class = { "no-wrap" },
},
['last_seen'] = {
title_i18n = "db_search.last_seen",
data_field = "last_seen",
sortable = true,
class = { "no-wrap" },
},
['l4proto'] = {
title_i18n = "db_search.l4proto",
data_field = "l4proto",
sortable = true,
class = { "no-wrap" },
render_generic = "l4proto",
},
['l7proto'] = {
title_i18n = "db_search.l7proto",
data_field = "l7proto",
sortable = true,
class = { "no-wrap" },
},
['score'] = {
title_i18n = "score",
data_field = "score",
sortable = true,
class = { "no-wrap" },
render_type = "formatValueLabel",
},
["flow"] = build_datatable_column_def_flow("flow", "flow"),
['vlan_id'] = {
title_i18n = "db_search.vlan_id",
data_field = "vlan_id",
sortable = true,
class = { "no-wrap" },
render_generic = "vlan_id",
},
['cli_ip'] = build_datatable_column_def_ip('cli_ip', "db_search.client"),
['srv_ip'] = build_datatable_column_def_ip('srv_ip', "db_search.server"),
['cli_port'] = build_datatable_column_def_port('cli_port', "db_search.cli_port"),
['srv_port'] = build_datatable_column_def_port('srv_port', "db_search.srv_port"),
['packets'] = build_datatable_column_def_packets('packets', "db_search.packets"),
['bytes'] = build_datatable_column_def_bytes('bytes', "db_search.bytes"),
['throughput'] = {
title_i18n = "db_search.throughput",
data_field = "throughput",
sortable = true,
class = { "no-wrap" },
},
['cli_asn'] = build_datatable_column_def_asn('cli_asn', "db_search.cli_asn"),
['srv_asn'] = build_datatable_column_def_asn('srv_asn', "db_search.srv_asn"),
['l7cat'] = {
title_i18n = "db_search.l7cat",
data_field = "l7cat",
sortable = true,
class = { "no-wrap" },
render_generic = "l7cat",
},
['alert_id'] = {
title_i18n = "db_search.alert_id",
data_field = "alert_id",
sortable = true,
class = { "no-wrap" },
render_generic = "alert_id",
},
['flow_risk'] = {
title_i18n = "db_search.flow_risk",
data_field = "flow_risk",
sortable = true,
class = { "no-wrap" },
},
['src2dst_tcp_flags'] = build_datatable_column_def_tcp_flags('src2dst_tcp_flags', "db_search.src2dst_tcp_flags"),
['dst2src_tcp_flags'] = build_datatable_column_def_tcp_flags('dst2src_tcp_flags', "db_search.dst2src_tcp_flags"),
['src2dst_dscp'] = build_datatable_column_def_dscp('src2dst_dscp', "db_search.src2dst_dscp"),
['dst2src_dscp'] = build_datatable_column_def_dscp('dst2src_dscp', "db_search.dst2src_dscp"),
['cli_nw_latency'] = build_datatable_column_def_nw_latency('cli_nw_latency', "db_search.cli_nw_latency"),
['srv_nw_latency'] = build_datatable_column_def_nw_latency('srv_nw_latency', "db_search.srv_nw_latency"),
['info'] = {
title_i18n = "db_search.info",
data_field = "info",
sortable = true,
class = { "no-wrap" },
},
['observation_point_id'] = {
title_i18n = "db_search.observation_point_id",
data_field = "observation_point_id",
sortable = true,
class = { "no-wrap" },
render_generic = "observation_point_id",
},
['probe_ip'] = {
title_i18n = "db_search.probe_ip",
data_field = "probe_ip",
sortable = true,
class = { "no-wrap" },
render_type = "formatProbeIP",
},
['cli_network'] = build_datatable_column_def_network('cli_network', "db_search.tags.cli_network"),
['srv_network'] = build_datatable_column_def_network('srv_network', "db_search.tags.srv_network"),
['cli_host_pool_id'] = build_datatable_column_def_pool_id('cli_host_pool_id', "db_search.tags.cli_host_pool_id"),
['srv_host_pool_id'] = build_datatable_column_def_pool_id('srv_host_pool_id', "db_search.tags.srv_host_pool_id"),
["input_snmp"] = build_datatable_column_def_snmp_interface("input_snmp", "db_search.tags.input_snmp"),
["output_snmp"] = build_datatable_column_def_snmp_interface("output_snmp", "db_search.tags.output_snmp"),
['cli_country'] = build_datatable_column_def_country('cli_country', "db_search.tags.cli_country"),
['srv_country'] = build_datatable_column_def_country('srv_country', "db_search.tags.srv_country"),
['community_id'] = build_datatable_column_def_community_id('community_id', "db_search.tags.community_id"),
}
-- #####################################
function historical_flow_utils.get_datatable_column_def_by_tag(tag)
if all_datatable_columns_def_by_tag[tag] then
return all_datatable_columns_def_by_tag[tag]
else
return build_datatable_column_def_default(tag, i18n("db_search.tags."..tag) or tag)
end
end
-- #####################################
function historical_flow_utils.get_historical_url(label, tag, value, add_hyperlink, title, add_copy_button)

View file

@ -1744,6 +1744,7 @@ local known_parameters = {
["all"] = validateBool, -- To remove limit on results
-- NAVIGATION
["gui"] = validateBool, -- Return data in html format (backward compatibility)
["page"] = validateSingleWord, -- Currently active subpage tab
["tab"] = validateSingleWord, -- Currently active tab, handled by javascript
["system_interface"] = validateBool,
@ -1789,7 +1790,7 @@ local known_parameters = {
["syslog_alert_format"] = http_lint.validateEmptyOr(http_lint.validateSyslogFormat),
["syslog_protocol"] = http_lint.validateEmptyOr(http_lint.validateChoiceInline({"tcp", "udp", ""})),
["syslog_host"] = http_lint.validateEmptyOr(http_lint.validateHost),
["syslog_host"] = http_lint.validateEmptyOr(validateServer),
["syslog_port"] = http_lint.validateEmptyOr(http_lint.validatePort),
["telegram_channel"] = http_lint.validateEmptyOr(http_lint.validateSingleWord),

View file

@ -1548,7 +1548,7 @@ function driver:timeseries_top(options, top_tags)
local ifindex = query_tag.if_index or query_tag.port
local ext_label = nil
if cached_device then
ext_label = shortenString(snmp_utils.get_snmp_interface_label(cached_device["interfaces"][ifindex]), 32)
ext_label = snmp_utils.get_snmp_interface_label(cached_device["interfaces"][ifindex])
if isEmptyString(ext_label) then
ext_label = ifindex
end

View file

@ -918,7 +918,7 @@ function driver:timeseries_top(options, top_tags)
local ifindex = available_tags[top_item][1].if_index or available_tags[top_item][1].port
local ext_label = nil
if cached_device then
ext_label = shortenString(snmp_utils.get_snmp_interface_label(cached_device["interfaces"][ifindex]), 32)
ext_label = snmp_utils.get_snmp_interface_label(cached_device["interfaces"][ifindex])
if isEmptyString(ext_label) then
ext_label = ifindex
end

View file

@ -555,11 +555,11 @@ local community_timeseries = {{
measure_unit = "number",
scale = i18n('graphs.metric_labels.score'),
timeseries = {
cli_score = {
score_as_cli = {
label = i18n('graphs.cli_score'),
color = timeseries_info.get_timeseries_color('cli_score')
},
srv_score = {
score_as_srv = {
label = i18n('graphs.srv_score'),
color = timeseries_info.get_timeseries_color('srv_score')
}