mirror of
https://github.com/ntop/ntopng.git
synced 2026-04-29 07:29:32 +00:00
Nicolo Maio
parent
c7e7ee70f2
commit
34f7a42308
33 changed files with 1885 additions and 466 deletions
|
|
@ -45,6 +45,7 @@ local prefs_host_values_key = "ntopng.prefs.vs.hosts_conf"
|
|||
local periodic_scan_key = "ntopng.vs.periodic_scan"
|
||||
local periodic_scan_type_key = "ntopng.vs.periodic_scan_type"
|
||||
local ondemand_scan_key = "ntopng.vs.scan_all"
|
||||
local single_scan_key = "ntopng.vs.single_scan"
|
||||
local scanned_hosts_count_key = "ntopng.prefs.host_to_scan.count_scanned"
|
||||
local scanned_hosts_changes_queue_key = "ntopng.alerts.scanned_hosts_changes"
|
||||
local host_in_scanning_hash_key = "ntopng.vs.hosts.in_scanning"
|
||||
|
|
@ -59,10 +60,15 @@ local periodic_scan_host_info_key = "ntopng.vs.periodic_scan.info"
|
|||
-- redis keys for scan all info
|
||||
local ondemand_scan_host_info_key = "ntopng.vs.scan_all.info"
|
||||
|
||||
-- redis key for single scan info
|
||||
local single_scan_info_key = "ntopng.vs.single_scan.info"
|
||||
|
||||
local json = require("dkjson")
|
||||
local format_utils = require("format_utils")
|
||||
local recipients = require("recipients")
|
||||
local cve_utils = require("cve_utils")
|
||||
local vs_db_utils = require("vs_db_utils")
|
||||
local vs_rest_utils = require("vs_rest_utils")
|
||||
|
||||
-- Enable debug with:
|
||||
-- redis-cli set "ntopng.prefs.vs.debug_enabled" "1"
|
||||
|
|
@ -90,10 +96,10 @@ vs_utils.scan_status = {
|
|||
scanning = 4
|
||||
}
|
||||
|
||||
vs_utils.ports_diff_case = {
|
||||
no_diff = 2, -- case 1 or 2 (combined)
|
||||
ntopng_more_t_vs = 3,
|
||||
vs_more_t_ntopng = 4
|
||||
vs_utils.scan_in_exec_type = {
|
||||
single_scan = 1,
|
||||
scan_all = 2,
|
||||
periodic_scan = 3
|
||||
}
|
||||
|
||||
-- **********************************************************
|
||||
|
|
@ -435,6 +441,7 @@ local function check_differences(host, host_name, scan_type, old_data, new_data)
|
|||
local date_string = format_utils.formatPastEpochShort(new_data.last_scan_time)
|
||||
date_string = string.gsub(date_string," ","_")
|
||||
rsp["date"] = date_string
|
||||
rsp["epoch"] = new_data.last_scan_time
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -656,11 +663,13 @@ end
|
|||
-- **********************************************************
|
||||
|
||||
-- Function to select correctly redis keys on periodic or scan all
|
||||
local function get_counter_periodic_all_scan_keys(is_periodic)
|
||||
if (is_periodic) then
|
||||
local function get_counter_periodic_all_scan_keys(exec_type)
|
||||
if (exec_type == vs_utils.scan_in_exec_type.periodic_scan) then
|
||||
return periodic_scan_host_info_key --host_periodic_scan_cve_num_key,host_periodic_scan_udp_ports_key,host_periodic_scan_tcp_ports_key
|
||||
else
|
||||
elseif (exec_type == vs_utils.scan_in_exec_type.scan_all) then
|
||||
return ondemand_scan_host_info_key --host_scan_all_cve_num_key,host_scan_all_udp_ports_key,host_scan_all_tcp_ports_key
|
||||
elseif(exec_type == vs_utils.scan_in_exec_type.single_scan) then
|
||||
return single_scan_info_key
|
||||
end
|
||||
end
|
||||
|
||||
|
|
@ -668,9 +677,9 @@ end
|
|||
|
||||
-- Function to update counters of periodically scan or scan all
|
||||
-- @param is_periodic (true -> is a periodic scan, false -> is a scan all)
|
||||
local function update_periodicity_or_all_scan_info(is_periodic, new_item)
|
||||
local function update_scan_info_for_report(type_of_scan_execution, new_item, host_hash_key)
|
||||
-- select correctly redis keys
|
||||
local redis_info_key = get_counter_periodic_all_scan_keys(is_periodic)
|
||||
local redis_info_key = get_counter_periodic_all_scan_keys(type_of_scan_execution)
|
||||
|
||||
local info_string = ntop.getCache(redis_info_key)
|
||||
local info_json = nil
|
||||
|
|
@ -742,7 +751,7 @@ function vs_utils.restore_host_to_scan()
|
|||
|
||||
-- set status to scheduled
|
||||
vs_utils.set_status_scan(host_info_to_restore.scan_type, host_info_to_restore.host, host_info_to_restore.ports,
|
||||
host_info_to_restore.id, host_info_to_restore.is_periodicity, host_info_to_restore.is_all,
|
||||
host_info_to_restore.id, host_info_to_restore.is_periodicity, host_info_to_restore.is_all, host_info_to_restore.is_single_scan,
|
||||
vs_utils.scan_status.scheduled)
|
||||
end
|
||||
end
|
||||
|
|
@ -955,7 +964,7 @@ function vs_utils.save_host_to_scan(scan_type, host, scan_result, last_scan_time
|
|||
new_item.scan_frequency = old_data.scan_frequency
|
||||
end
|
||||
|
||||
-- the is_periodicity param and is_all param are set outside the save_host_to_scan into the set_status method
|
||||
-- the is_periodicity, is_all and is_single_scan params are set outside the save_host_to_scan into the set_status method
|
||||
if (old_data and old_data.is_periodicity ~= nil) then
|
||||
new_item.is_periodicity = old_data.is_periodicity
|
||||
end
|
||||
|
|
@ -964,7 +973,11 @@ function vs_utils.save_host_to_scan(scan_type, host, scan_result, last_scan_time
|
|||
new_item.is_all = old_data.is_all
|
||||
end
|
||||
|
||||
if(scan_result ~= nil) then
|
||||
if (old_data and old_data.is_single_scan ~= nil) then
|
||||
new_item.is_single_scan = old_data.is_single_scan
|
||||
end
|
||||
|
||||
if(scan_result ~= nil and not ntop.isClickHouseEnabled()) then
|
||||
local handle = io.open(get_report_path(scan_type, host), "w")
|
||||
local result = handle:write(scan_result)
|
||||
handle:close()
|
||||
|
|
@ -976,21 +989,52 @@ function vs_utils.save_host_to_scan(scan_type, host, scan_result, last_scan_time
|
|||
-- traceError(TRACE_NORMAL, TRACE_CONSOLE, "Updating host " .. host)
|
||||
end
|
||||
|
||||
-- add ports statistics comparing the ntopng passive monitoring info
|
||||
if (scan_type == "tcp_portscan" or scan_type == "tcp_openports" or scan_type == "udp_portscan") then
|
||||
local compare_info = vs_rest_utils.compare_scan_info_ntopng_info(host, scan_type, new_item.tcp_ports_list, new_item.udp_ports_list)
|
||||
|
||||
new_item.host_in_mem = compare_info.host_in_mem
|
||||
if (scan_type == "tcp_portscan" or scan_type == "tcp_openports") then
|
||||
new_item.tcp_ports_unused = compare_info.tcp_ports_unused
|
||||
new_item.tcp_ports_filtered = compare_info.tcp_ports_filtered
|
||||
new_item.tcp_ports_case = compare_info.tcp_ports_case
|
||||
elseif (scan_type == "udp_portscan") then
|
||||
rsp.udp_ports_unused = compare_info.udp_ports_unused
|
||||
rsp.udp_ports_filtered = compare_info.udp_ports_filtered
|
||||
rsp.udp_ports_case = compare_info.udp_ports_case
|
||||
end
|
||||
end
|
||||
-- edit case
|
||||
ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(new_item))
|
||||
|
||||
local counts = vs_utils.update_ts_counters()
|
||||
|
||||
if (new_item.is_periodicity) then
|
||||
update_periodicity_or_all_scan_info(true, new_item)
|
||||
update_scan_info_for_report(vs_utils.scan_in_exec_type.periodic_scan, new_item, host_hash_key)
|
||||
end
|
||||
|
||||
if (new_item.is_all) then
|
||||
update_periodicity_or_all_scan_info(false, new_item)
|
||||
update_scan_info_for_report(vs_utils.scan_in_exec_type.scan_all, new_item, host_hash_key)
|
||||
end
|
||||
|
||||
remove_scanning_host({host=host, scan_type=scan_type, ports=ports})
|
||||
|
||||
-- save on db here
|
||||
|
||||
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
|
||||
local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,host_hash_key)
|
||||
-- hash value found
|
||||
local hash_pref_value = json.decode(hash_prefs_string)
|
||||
if (hash_pref_value ~= nil) then
|
||||
-- is necessary this check to avoid to save entries not presents in the config and to save all data of the entry.
|
||||
for key,value in pairs(hash_pref_value) do
|
||||
if (key ~= 'is_ok_last_scan') then
|
||||
new_item[key] = value
|
||||
end
|
||||
end
|
||||
|
||||
vs_db_utils.save_vs_result(scan_type, host, new_item.last_scan.epoch, json.encode(new_item), scan_result)
|
||||
end
|
||||
return result, new_item.id
|
||||
end
|
||||
|
||||
|
|
@ -1075,13 +1119,8 @@ end
|
|||
|
||||
-- **********************************************************
|
||||
|
||||
-- Function to send notification after a periodic scan
|
||||
-- @param is_periodic (true -> is a periodic scan message, false -> is an all scan message)
|
||||
-- @param periodicity (can be nil in case of scan all)
|
||||
function vs_utils.notify_scan_results(is_periodic, periodicity)
|
||||
local notification_message = ""
|
||||
|
||||
local info_redis_key = get_counter_periodic_all_scan_keys(is_periodic)
|
||||
local function retrieve_email_info(exec_type)
|
||||
local info_redis_key = get_counter_periodic_all_scan_keys(exec_type)
|
||||
|
||||
local info_string = ntop.getCache(info_redis_key)
|
||||
|
||||
|
|
@ -1095,45 +1134,117 @@ function vs_utils.notify_scan_results(is_periodic, periodicity)
|
|||
tcp_ports = 0
|
||||
}
|
||||
end
|
||||
local cve_num = tonumber(info_json.cves) or 0
|
||||
local udp_ports = tonumber(info_json.udp_ports) or 0
|
||||
local tcp_ports = tonumber(info_json.tcp_ports) or 0
|
||||
local scanned_hosts = tonumber(info_json.scanned_hosts) or 0
|
||||
local email_info = {
|
||||
cve_num = tonumber(info_json.cves) or 0,
|
||||
udp_ports = tonumber(info_json.udp_ports) or 0,
|
||||
tcp_ports = tonumber(info_json.tcp_ports) or 0,
|
||||
scanned_hosts = tonumber(info_json.scanned_hosts) or 0,
|
||||
begin_epoch_t = tonumber(info_json.begin_epoch),
|
||||
end_epoch_t = os.time(),
|
||||
report_type = exec_type
|
||||
}
|
||||
|
||||
email_info.duration = email_info.end_epoch_t - email_info.begin_epoch_t
|
||||
|
||||
ntop.setCache(info_redis_key,json.encode({
|
||||
cves = 0,
|
||||
udp_ports = 0,
|
||||
tcp_ports = 0,
|
||||
begin_epoch = 0,
|
||||
scanned_hosts = 0
|
||||
}))
|
||||
return email_info
|
||||
end
|
||||
|
||||
local function retrieve_report_info(date)
|
||||
local host_scanned_info = vs_utils.retrieve_hosts_to_scan()
|
||||
|
||||
local info = {
|
||||
cves = 0,
|
||||
tcp_ports = 0,
|
||||
udp_ports = 0,
|
||||
scanned_hosts = 0,
|
||||
}
|
||||
for _, item in ipairs(host_scanned_info) do
|
||||
if (not isEmptyString(item.num_vulnerabilities_found)) then
|
||||
info.cves = info.cves + tonumber(item.num_vulnerabilities_found)
|
||||
end
|
||||
|
||||
info.tcp_ports = info.tcp_ports + tonumber(item.tcp_ports)
|
||||
info.udp_ports = info.udp_ports + tonumber(item.udp_ports)
|
||||
info.scanned_hosts = info.scanned_hosts + 1
|
||||
end
|
||||
|
||||
if (date) then
|
||||
-- case of periodic scan and scan all
|
||||
-- in order to save same date on email and report
|
||||
info.date = tonumber(date)
|
||||
else
|
||||
info.date = tonumber(os.time())
|
||||
end
|
||||
|
||||
local info_date_formatted = tostring(formatEpoch(info.date))
|
||||
info.name = "Report of "..info_date_formatted
|
||||
|
||||
info.all_data_details = host_scanned_info
|
||||
|
||||
return info
|
||||
end
|
||||
-- **********************************************************
|
||||
|
||||
-- params date used only in case of periodic or scan all exec
|
||||
function vs_utils.generate_report(date)
|
||||
|
||||
local report_info = retrieve_report_info(date)
|
||||
|
||||
vs_db_utils.save_report_info(report_info)
|
||||
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
||||
-- Function to send notification after a periodic scan
|
||||
-- @param exec_type (2 -> is an all scan message, 3 -> is a periodic scan message)
|
||||
-- @param periodicity (can be nil in case of scan all)
|
||||
function vs_utils.notify_scan_results(exec_type, periodicity)
|
||||
local notification_message = ""
|
||||
|
||||
local email_info = retrieve_email_info(exec_type)
|
||||
|
||||
local title = i18n("hosts_stats.page_scan_hosts.email.vulnerability_scan_report_title",{host = getHttpHost()})
|
||||
local duration = 0
|
||||
local duration_label = ''
|
||||
|
||||
local begin_epoch_t = tonumber(info_json.begin_epoch)
|
||||
local end_epoch_t = os.time()
|
||||
|
||||
if (info_json.begin_epoch ~= nil) then
|
||||
duration = end_epoch_t - begin_epoch_t
|
||||
duration_label = secondsToTime(duration)
|
||||
|
||||
if (email_info.duration ~= nil) then
|
||||
duration_label = secondsToTime(email_info.duration)
|
||||
end
|
||||
|
||||
local start_date_formatted = formatEpoch(begin_epoch_t)
|
||||
local end_date_formatted = formatEpoch(end_epoch_t)
|
||||
local start_date_formatted = formatEpoch(email_info.begin_epoch_t)
|
||||
local end_date_formatted = formatEpoch(email_info.end_epoch_t)
|
||||
|
||||
local report_date = tonumber(email_info.end_epoch_t)
|
||||
|
||||
ntop.setCache(hosts_scan_last_report_dates, json.encode({start_date = start_date_formatted, end_date = end_date_formatted}))
|
||||
|
||||
if (periodicity and periodicity == "1day") then
|
||||
notification_message = i18n("hosts_stats.page_scan_hosts.email.periodicity_scan_1_day_ended", {
|
||||
cves = format_num_for_email(cve_num,0),
|
||||
udp_ports = format_num_for_email(udp_ports,1),
|
||||
tcp_ports = format_num_for_email(tcp_ports,2),
|
||||
scanned_hosts = format_num_for_email(scanned_hosts, 3),
|
||||
url = getHttpHost() .. ntop.getHttpPrefix() .. "/lua/pro/reportng.lua?report_template=vs_result",
|
||||
cves = format_num_for_email(email_info.cve_num,0),
|
||||
udp_ports = format_num_for_email(email_info.udp_ports,1),
|
||||
tcp_ports = format_num_for_email(email_info.tcp_ports,2),
|
||||
scanned_hosts = format_num_for_email(email_info.scanned_hosts, 3),
|
||||
url = string.format(getHttpHost() .. ntop.getHttpPrefix() .. "/lua/enterprise/vulnerability_scan_report.lua?epoch_end=%u&epoch_begin=%u",report_date,report_date),
|
||||
duration = duration_label,
|
||||
start_date = start_date_formatted,
|
||||
end_date = end_date_formatted
|
||||
})
|
||||
elseif (periodicity and periodicity == "1week") then
|
||||
notification_message = i18n("hosts_stats.page_scan_hosts.email.periodicity_scan_1_week_ended", {
|
||||
cves = format_num_for_email(cve_num,0),
|
||||
udp_ports = format_num_for_email(udp_ports,1),
|
||||
tcp_ports = format_num_for_email(tcp_ports,2),
|
||||
scanned_hosts = format_num_for_email(scanned_hosts, 3),
|
||||
url = getHttpHost() .. ntop.getHttpPrefix() .. "/lua/pro/reportng.lua?report_template=vs_result",
|
||||
cves = format_num_for_email(email_info.cve_num,0),
|
||||
udp_ports = format_num_for_email(email_info.udp_ports,1),
|
||||
tcp_ports = format_num_for_email(email_info.tcp_ports,2),
|
||||
scanned_hosts = format_num_for_email(email_info.scanned_hosts, 3),
|
||||
url = string.format(getHttpHost() .. ntop.getHttpPrefix() .. "/lua/enterprise/vulnerability_scan_report.lua?epoch_end=%u&epoch_begin=%u",report_date,report_date),
|
||||
duration = duration_label,
|
||||
start_date = start_date_formatted,
|
||||
end_date = end_date_formatted
|
||||
|
|
@ -1141,11 +1252,11 @@ function vs_utils.notify_scan_results(is_periodic, periodicity)
|
|||
else
|
||||
-- on demand scan
|
||||
notification_message = i18n("hosts_stats.page_scan_hosts.email.scan_all_ended", {
|
||||
cves = format_num_for_email(cve_num,0),
|
||||
udp_ports = format_num_for_email(udp_ports,1),
|
||||
tcp_ports = format_num_for_email(tcp_ports,2),
|
||||
scanned_hosts = format_num_for_email(scanned_hosts, 3),
|
||||
url = getHttpHost() .. ntop.getHttpPrefix() .. "/lua/pro/reportng.lua?report_template=vs_result",
|
||||
cves = format_num_for_email(email_info.cve_num,0),
|
||||
udp_ports = format_num_for_email(email_info.udp_ports,1),
|
||||
tcp_ports = format_num_for_email(email_info.tcp_ports,2),
|
||||
scanned_hosts = format_num_for_email(email_info.scanned_hosts, 3),
|
||||
url = string.format(getHttpHost() .. ntop.getHttpPrefix() .. "/lua/enterprise/vulnerability_scan_report.lua?epoch_end=%u&epoch_begin=%u",report_date,report_date),
|
||||
duration = duration_label,
|
||||
start_date = start_date_formatted,
|
||||
end_date = end_date_formatted,
|
||||
|
|
@ -1156,16 +1267,10 @@ function vs_utils.notify_scan_results(is_periodic, periodicity)
|
|||
if verbose then
|
||||
traceError(TRACE_NORMAL,TRACE_CONSOLE, "Vulnerability Scan completed. Sending " .. title .."\n")
|
||||
end
|
||||
vs_utils.generate_report(email_info.end_epoch_t)
|
||||
|
||||
recipients.sendMessageByNotificationType({periodicity = periodicity, success = true, message = notification_message, title = title}, "vulnerability_scans")
|
||||
|
||||
ntop.setCache(info_redis_key,json.encode({
|
||||
cves = 0,
|
||||
udp_ports = 0,
|
||||
tcp_ports = 0,
|
||||
begin_epoch = 0,
|
||||
scanned_hosts = 0
|
||||
}))
|
||||
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
|
@ -1251,6 +1356,41 @@ function vs_utils.is_ondemand_scan_completed()
|
|||
return false
|
||||
end
|
||||
|
||||
function vs_utils.is_single_scan_completed()
|
||||
local single_scan_in_progress = ntop.getCache(single_scan_key) == "1"
|
||||
|
||||
if (single_scan_in_progress) then
|
||||
local hosts_details = vs_utils.retrieve_hosts_to_scan()
|
||||
for _,item in ipairs(hosts_details) do
|
||||
|
||||
-- verify status of in progress periodic scanning
|
||||
if(item.is_single_scan and (item.is_ok_last_scan == vs_utils.scan_status.scheduled or item.is_ok_last_scan == vs_utils.scan_status.scanning)) then
|
||||
return false
|
||||
end
|
||||
end
|
||||
|
||||
ntop.setCache(single_scan_key, "0")
|
||||
|
||||
for _,item in ipairs(hosts_details) do
|
||||
local host_hash_key = vs_utils.get_host_hash_key(item.host, item.scan_type)
|
||||
local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
|
||||
if(not isEmptyString(host_hash_value_string)) then
|
||||
|
||||
local host_hash_value = json.decode(host_hash_value_string)
|
||||
|
||||
host_hash_value.is_single_scan = false
|
||||
|
||||
ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
|
||||
end
|
||||
end
|
||||
|
||||
return true
|
||||
end
|
||||
|
||||
return false
|
||||
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
||||
-- Function to enable periodic scan end check on callbacks
|
||||
|
|
@ -1260,26 +1400,30 @@ end
|
|||
|
||||
-- **********************************************************
|
||||
|
||||
local function retrieve_host_by_hash_key(host_hash_key)
|
||||
local hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
|
||||
local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,host_hash_key)
|
||||
local hash_value = json.decode(hash_prefs_string)
|
||||
if (not isEmptyString(hash_value_string)) then
|
||||
-- hash value found
|
||||
hash_value = json.decode(hash_value_string)
|
||||
local hash_pref_value = json.decode(hash_prefs_string) or {}
|
||||
for k,value in pairs(hash_pref_value) do
|
||||
if (k ~= 'is_ok_last_scan') then
|
||||
hash_value[k] = value
|
||||
end
|
||||
end
|
||||
end
|
||||
return hash_value
|
||||
end
|
||||
-- **********************************************************
|
||||
|
||||
-- Function to retrieve a specific host scan info
|
||||
function vs_utils.retrieve_host(host)
|
||||
local hosts_scanned = ntop.getHashKeysCache(host_to_scan_key) or {}
|
||||
for key, _ in pairs(hosts_scanned) do
|
||||
if key:find(host) and key:find("cve") then
|
||||
local hash_value_string = ntop.getHashCache(host_to_scan_key, key)
|
||||
local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,key)
|
||||
local hash_value = json.decode(hash_prefs_string)
|
||||
if (not isEmptyString(hash_value_string)) then
|
||||
-- hash value found
|
||||
hash_value = json.decode(hash_value_string)
|
||||
local hash_pref_value = json.decode(hash_prefs_string) or {}
|
||||
for k,value in pairs(hash_pref_value) do
|
||||
if (k ~= 'is_ok_last_scan') then
|
||||
hash_value[k] = value
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
return hash_value
|
||||
return retrieve_host_by_hash_key(key)
|
||||
end
|
||||
end
|
||||
return nil
|
||||
|
|
@ -1288,37 +1432,43 @@ end
|
|||
-- **********************************************************
|
||||
|
||||
-- Function to retrieve hosts list to scan
|
||||
function vs_utils.retrieve_hosts_to_scan()
|
||||
local hash_keys = ntop.getHashKeysCache(prefs_host_values_key)
|
||||
local rsp = {}
|
||||
if hash_keys then
|
||||
for k in pairs(hash_keys) do
|
||||
local hash_value_string = ntop.getHashCache(host_to_scan_key, k)
|
||||
local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,k)
|
||||
local hash_value = json.decode(hash_prefs_string)
|
||||
function vs_utils.retrieve_hosts_to_scan(epoch)
|
||||
if (isEmptyString(epoch) or (not ntop.isClickHouseEnabled())) then
|
||||
-- not a request for historical data (only for report)
|
||||
local hash_keys = ntop.getHashKeysCache(prefs_host_values_key)
|
||||
local rsp = {}
|
||||
if hash_keys then
|
||||
for k in pairs(hash_keys) do
|
||||
local hash_value_string = ntop.getHashCache(host_to_scan_key, k)
|
||||
local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,k)
|
||||
local hash_value = json.decode(hash_prefs_string)
|
||||
|
||||
if (not isEmptyString(hash_value_string)) then
|
||||
-- hash value found
|
||||
if (not isEmptyString(hash_value_string)) then
|
||||
-- hash value found
|
||||
|
||||
hash_value = json.decode(hash_value_string)
|
||||
local hash_pref_value = json.decode(hash_prefs_string) or {}
|
||||
hash_value = json.decode(hash_value_string)
|
||||
local hash_pref_value = json.decode(hash_prefs_string) or {}
|
||||
|
||||
for key,value in pairs(hash_pref_value) do
|
||||
if (key ~= 'is_ok_last_scan') then
|
||||
hash_value[key] = value
|
||||
for key,value in pairs(hash_pref_value) do
|
||||
if (key ~= 'is_ok_last_scan') then
|
||||
hash_value[key] = value
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
-- hash value not found
|
||||
ntop.setHashCache(host_to_scan_key, k, hash_prefs_string)
|
||||
else
|
||||
-- hash value not found
|
||||
ntop.setHashCache(host_to_scan_key, k, hash_prefs_string)
|
||||
|
||||
end
|
||||
|
||||
rsp[#rsp+1] = hash_value
|
||||
end
|
||||
|
||||
rsp[#rsp+1] = hash_value
|
||||
end
|
||||
return rsp
|
||||
else
|
||||
-- request for report
|
||||
local all_details, data = vs_db_utils.retrieve_report(epoch)
|
||||
return(data)
|
||||
end
|
||||
|
||||
return rsp
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
|
@ -1348,9 +1498,10 @@ function vs_utils.check_in_progress_status()
|
|||
end
|
||||
|
||||
-- **********************************************************
|
||||
-- Function restrieve scan result from FS
|
||||
|
||||
-- Function to retrieve last host scan result
|
||||
function vs_utils.retrieve_hosts_scan_result(scan_type, host)
|
||||
local function retrieve_scan_result_from_file(scan_type, host)
|
||||
-- for retrocompatibility
|
||||
local path = get_report_path(scan_type, host)
|
||||
|
||||
if(ntop.exists(path)) then
|
||||
|
|
@ -1362,6 +1513,49 @@ function vs_utils.retrieve_hosts_scan_result(scan_type, host)
|
|||
else
|
||||
return("Unable to read file "..path)
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
-- Function to retrieve last host scan result
|
||||
function vs_utils.retrieve_hosts_scan_result(scan_type, host, epoch)
|
||||
|
||||
-- retrieve from DB
|
||||
local db_result
|
||||
local fs_result
|
||||
|
||||
local is_clickhouse_enabled = ntop.isClickHouseEnabled()
|
||||
|
||||
-- retrieve data here
|
||||
if (is_clickhouse_enabled) then
|
||||
-- retrieve from DB
|
||||
db_result = vs_db_utils.retrieve_scan_result(scan_type, host, tonumber(epoch))
|
||||
else
|
||||
-- retrieve from FS
|
||||
fs_result = retrieve_scan_result_from_file(scan_type, host)
|
||||
|
||||
return fs_result
|
||||
end
|
||||
|
||||
-- return data
|
||||
if (is_clickhouse_enabled and db_result == "") then
|
||||
|
||||
-- ****************************************
|
||||
-- retrocompatibility case:
|
||||
|
||||
-- when there were results saved on FS,
|
||||
-- but the user updates ntopng to use clickhouse,
|
||||
-- and there are currently no results in clickhouse.
|
||||
-- ****************************************
|
||||
|
||||
-- clickhouse is enabled but maybe the user had old data on FS and no data on DB
|
||||
fs_result = retrieve_scan_result_from_file(scan_type, host)
|
||||
return fs_result
|
||||
else
|
||||
-- the result from DB is ok.
|
||||
return db_result
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
|
@ -1384,6 +1578,7 @@ function vs_utils.delete_host_to_scan(host, scan_type, all)
|
|||
os.remove(path_to_s_result)
|
||||
ntop.delHashCache(host_to_scan_key, host_hash_key)
|
||||
ntop.delHashCache(prefs_host_values_key, host_hash_key)
|
||||
ntop.delHashCache(host_in_scanning_hash_key, host_hash_key)
|
||||
|
||||
-- Remove this host from active schedules
|
||||
local elems = {}
|
||||
|
|
@ -1519,7 +1714,7 @@ function vs_utils.scan_host(scan_type, host, ports, scan_id, use_coroutines)
|
|||
|
||||
if(ntop.isShuttingDown()) then return(false) end
|
||||
|
||||
vs_utils.set_status_scan(scan_type, host, ports_scan_param, id, nil,nil, vs_utils.scan_status.scanning)
|
||||
vs_utils.set_status_scan(scan_type, host, ports_scan_param, id, nil,nil,nil, vs_utils.scan_status.scanning)
|
||||
|
||||
-- Save on redis the scanning host to avoid inconsistent state on ntopng restarts
|
||||
local scanning_host = {scan_type = scan_type, host = host, ports = ports_scan_param, id = scan_id}
|
||||
|
|
@ -1561,8 +1756,23 @@ end
|
|||
|
||||
-- **********************************************************
|
||||
|
||||
local function set_single_scan_in_progress()
|
||||
|
||||
if (ntop.getCache(single_scan_key) ~= '1') then
|
||||
ntop.setCache(single_scan_key, "1")
|
||||
ntop.setCache(single_scan_info_key, json.encode({
|
||||
cves = 0,
|
||||
udp_ports = 0,
|
||||
tcp_ports = 0,
|
||||
begin_epoch = os.time(),
|
||||
scanned_host = 1
|
||||
}))
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
-- Function to update single host status
|
||||
function vs_utils.set_status_scan(scan_type, host, ports, id, is_periodicity, is_all, status)
|
||||
function vs_utils.set_status_scan(scan_type, host, ports, id, is_periodicity, is_all,is_single_scan, status)
|
||||
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
|
||||
local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
|
||||
local host_hash_value
|
||||
|
|
@ -1574,14 +1784,19 @@ function vs_utils.set_status_scan(scan_type, host, ports, id, is_periodicity, is
|
|||
end
|
||||
|
||||
host_hash_value.is_ok_last_scan = status
|
||||
if (is_periodicity ~= nil) then
|
||||
if (is_periodicity ~= nil and is_periodicity) then
|
||||
host_hash_value.is_periodicity = is_periodicity
|
||||
end
|
||||
|
||||
if (is_all ~= nil) then
|
||||
if (is_all ~= nil and is_all) then
|
||||
host_hash_value.is_all = is_all
|
||||
end
|
||||
|
||||
if (is_single_scan ~= nil and is_single_scan) then
|
||||
host_hash_value.is_single_scan = is_single_scan
|
||||
set_single_scan_in_progress()
|
||||
end
|
||||
|
||||
ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
|
||||
|
||||
return true
|
||||
|
|
@ -1589,9 +1804,11 @@ end
|
|||
|
||||
-- **********************************************************
|
||||
|
||||
function vs_utils.schedule_ondemand_single_host_scan(scan_type, host, ports, scan_id, is_periodicity, is_all)
|
||||
function vs_utils.schedule_ondemand_single_host_scan(scan_type, host, ports, scan_id, is_periodicity, is_all, is_single_scan)
|
||||
local scan = { scan_type = scan_type, host = host, ports = ports, id= scan_id}
|
||||
vs_utils.set_status_scan(scan_type, host, ports, scan_id, is_periodicity, is_all, vs_utils.scan_status.scheduled)
|
||||
|
||||
|
||||
vs_utils.set_status_scan(scan_type, host, ports, scan_id, is_periodicity, is_all, is_single_scan, vs_utils.scan_status.scheduled)
|
||||
|
||||
ntop.rpushCache(host_scan_queue_key, json.encode(scan))
|
||||
|
||||
|
|
@ -1606,7 +1823,7 @@ function vs_utils.schedule_ondemand_all_hosts_scan()
|
|||
local is_scanning_almost_one = false
|
||||
if #host_to_scan_list > 0 then
|
||||
for _,scan_info in ipairs(host_to_scan_list) do
|
||||
vs_utils.schedule_ondemand_single_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, false, true)
|
||||
vs_utils.schedule_ondemand_single_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, false, true, false)
|
||||
is_scanning_almost_one = true
|
||||
end
|
||||
end
|
||||
|
|
@ -1642,7 +1859,7 @@ function vs_utils.schedule_periodic_scan(periodicity)
|
|||
for _,scan_info in ipairs(host_to_scan_list) do
|
||||
local frequency = scan_info.scan_frequency
|
||||
if(frequency == periodicity) then
|
||||
vs_utils.schedule_ondemand_single_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, true, false)
|
||||
vs_utils.schedule_ondemand_single_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, true, false, false)
|
||||
is_scanning_almost_one = true
|
||||
end
|
||||
end
|
||||
|
|
@ -1840,102 +2057,15 @@ function vs_utils.is_available()
|
|||
return (#scan_modules > 0)
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
||||
function vs_utils.format_port_label(port, service_name, protocol)
|
||||
if (isEmptyString(service_name)) then
|
||||
return string.format("%s/%s",port,protocol)
|
||||
else
|
||||
return string.format("%s/%s (%s)",port,protocol,service_name)
|
||||
end
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
-- Retrieves detected ports by ntopng
|
||||
|
||||
function vs_utils.retrieve_detected_ports(host)
|
||||
local host_info = interface.getHostInfo(host)
|
||||
|
||||
local tcp_ports_detected = {}
|
||||
local udp_ports_detected = {}
|
||||
local host_in_mem = false
|
||||
if (host_info and host_info.used_ports and host_info.used_ports.local_server_ports) then
|
||||
for port, l7_proto in pairs(host_info.used_ports.local_server_ports) do
|
||||
local port_details = split(port, ":")
|
||||
local id_port = port_details[2]
|
||||
local l4_proto = port_details[1]
|
||||
|
||||
if (l4_proto == 'tcp') then
|
||||
tcp_ports_detected[#tcp_ports_detected+1] = id_port
|
||||
end
|
||||
|
||||
if (l4_proto == 'udp') then
|
||||
udp_ports_detected[#udp_ports_detected+1] = id_port
|
||||
end
|
||||
end
|
||||
host_in_mem = true
|
||||
end
|
||||
|
||||
return tcp_ports_detected, host_in_mem, udp_ports_detected
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
||||
-- Search port in ports list
|
||||
local function find_port(port_to_find, port_list)
|
||||
for _, port in ipairs(port_list) do
|
||||
if(port_to_find == tonumber(port)) then
|
||||
return true
|
||||
end
|
||||
end
|
||||
return false
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
||||
-- Compare vs ports and ntopng detected ports
|
||||
function vs_utils.compare_ports(vs_scan_port_string_list, ntopng_ports)
|
||||
local vs_scan_ports = split(vs_scan_port_string_list, ",")
|
||||
|
||||
local ports_unused = {}
|
||||
local filtered_ports = {}
|
||||
-- check vs_scan_ports with ntopng_ports
|
||||
local not_found_a_port = false
|
||||
for _,vs_port in ipairs(vs_scan_ports) do
|
||||
local find_actual_port = find_port(tonumber(vs_port), ntopng_ports)
|
||||
|
||||
if (not find_actual_port) then
|
||||
not_found_a_port = true
|
||||
ports_unused[#ports_unused+1] = vs_port
|
||||
end
|
||||
|
||||
end
|
||||
|
||||
local diff_case
|
||||
|
||||
if (not_found_a_port) then
|
||||
diff_case = vs_utils.ports_diff_case.vs_more_t_ntopng
|
||||
end
|
||||
|
||||
if (#vs_scan_ports == #ntopng_ports) then
|
||||
diff_case = vs_utils.ports_diff_case.no_diff
|
||||
else
|
||||
local filtered = false
|
||||
|
||||
for _,ntop_port in ipairs(ntopng_ports) do
|
||||
if (not find_port(tonumber(ntop_port), vs_scan_ports)) then
|
||||
filtered = true
|
||||
filtered_ports[#filtered_ports+1] = ntop_port
|
||||
end
|
||||
|
||||
end
|
||||
if (filtered) then
|
||||
diff_case = vs_utils.ports_diff_case.ntopng_more_t_vs
|
||||
end
|
||||
end
|
||||
|
||||
return ports_unused, filtered_ports, diff_case
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
||||
|
|
@ -2059,4 +2189,28 @@ end
|
|||
|
||||
-- **********************************************************
|
||||
|
||||
function vs_utils.retrieve_report_list(epoch)
|
||||
local sort_on = "DATE"
|
||||
return (vs_db_utils.retrieve_reports(sort_on,epoch))
|
||||
end
|
||||
|
||||
|
||||
-- **********************************************************
|
||||
|
||||
function vs_utils.retrieve_report(report_name)
|
||||
return (vs_db_utils.retrieve_report(report_name))
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
||||
function vs_utils.delete_report(epoch)
|
||||
return(vs_db_utils.delete_report(epoch))
|
||||
end
|
||||
|
||||
-- **********************************************************
|
||||
|
||||
function vs_utils.edit_report(epoch, report_name)
|
||||
return(vs_db_utils.edit_report(epoch,report_name))
|
||||
end
|
||||
|
||||
return vs_utils
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue