Add comments on vs_utils module.

This commit is contained in:
Nicolo Maio 2024-05-13 10:26:57 +02:00
parent a02dd13bc8
commit 0d30c14c94

View file

@ -39,13 +39,21 @@ package.path = dirs.installdir .. "/scripts/lua/modules/recipients/?.lua;" .. pa
require "lua_utils" -- used by tprint (debug)
-- redis key with info found with scans
local host_to_scan_key = "ntopng.vs.hosts.scanned_values"
-- redis key with the current configuration
local prefs_host_values_key = "ntopng.prefs.vs.hosts_conf"
-- redis key for periodic scan in progress
local periodic_scan_key = "ntopng.vs.periodic_scan"
local periodic_scan_type_key = "ntopng.vs.periodic_scan_type"
-- redis key for scann all in progress
local ondemand_scan_key = "ntopng.vs.scan_all"
-- redis key for single scan in progress
local single_scan_key = "ntopng.vs.single_scan"
-- redis keys to handle scan in progress or scheduled
local scanned_hosts_count_key = "ntopng.prefs.host_to_scan.count_scanned"
local scanned_hosts_changes_queue_key = "ntopng.alerts.scanned_hosts_changes"
local host_in_scanning_hash_key = "ntopng.vs.hosts.in_scanning"
@ -63,6 +71,7 @@ local ondemand_scan_host_info_key = "ntopng.vs.scan_all.info"
-- redis key for single scan info
local single_scan_info_key = "ntopng.vs.single_scan.info"
-- imports
local json = require("dkjson")
local format_utils = require("format_utils")
local recipients = require("recipients")
@ -81,6 +90,7 @@ local vs_utils = {}
local use_slow_scan;
-- **********************************************************
-- Function to retrieve host hash key used on redis
function vs_utils.get_host_hash_key(host, scan_type)
return string.format("%s-%s", host, scan_type)
@ -88,6 +98,7 @@ end
-- **********************************************************
-- Scan status
vs_utils.scan_status = {
error = 0,
ok = 1,
@ -97,6 +108,7 @@ vs_utils.scan_status = {
failed = 5,
}
-- Scan execution type
vs_utils.scan_in_exec_type = {
single_scan = 1,
scan_all = 2,
@ -104,7 +116,7 @@ vs_utils.scan_in_exec_type = {
}
-- **********************************************************
-- Function to retrieve nmap path on OS
function vs_utils.get_nmap_path()
local path = {
"/usr/bin/nmap",
@ -122,7 +134,7 @@ function vs_utils.get_nmap_path()
end
-- **********************************************************
-- Function to check if nmap is installed
function vs_utils.is_nmap_installed()
local module_path = {
"/usr/share/nmap/scripts/",
@ -143,7 +155,8 @@ function vs_utils.is_nmap_installed()
end
-- **********************************************************
-- Function to get old report path (now the reports are
-- saved on the DB)
local function get_report_path(scan_type, ip, all)
local base_dir
local ret = ""
@ -161,7 +174,7 @@ local function get_report_path(scan_type, ip, all)
end
-- ##############################################
-- Function to split string on \n
local function lines(str)
local result = {}
@ -173,7 +186,8 @@ end
-- ##############################################
-- Function to concatenate string item list into
-- comma separated list
local function format_port_list_to_string(ports)
local scan_ports = ""
@ -191,7 +205,7 @@ local function format_port_list_to_string(ports)
end
-- ##############################################
-- Function to search for an item in a list
local function find_port(port, port_list)
local found = false
@ -205,22 +219,30 @@ local function find_port(port, port_list)
end
-- ##############################################
-- Function to check ports differences between two
-- different scans on same host
local function check_ports_diffences(num_old_ports, old_ports, num_new_ports, new_ports)
-- num_old_ports: number of old open ports (on the host before the last scan)
-- old_ports: list of old ports
-- num_new_ports: number of new open ports found (with the last host scan)
-- new_ports: list of new open ports
local rsp = {
trigger = true
}
if (num_old_ports == 0 and num_new_ports ~= 0) then
-- new ports found no old ports closed
rsp.open_ports = new_ports
rsp.open_ports_num = num_new_ports
rsp.closed_ports_num = 0
rsp.case = 'new_ports'
elseif(num_old_ports ~= 0 and num_new_ports == 0) then
-- no new ports found but old ports closed found
rsp.open_ports_num = 0
rsp.closed_ports_num = num_old_ports
rsp.closed_ports = old_ports
rsp.case = 'ports_closed'
elseif(num_old_ports ~= 0 and num_new_ports ~= 0) then
-- case new ports and old ports closed found
local closed_ports = {}
local open_ports = {}
@ -243,6 +265,8 @@ local function check_ports_diffences(num_old_ports, old_ports, num_new_ports, ne
if((not diff) and (num_old_ports == num_new_ports)) then
-- case no differences and the old ports num is equals to
-- new ports open
rsp.trigger = false
else
rsp.open_ports = open_ports
@ -265,7 +289,8 @@ local function check_ports_diffences(num_old_ports, old_ports, num_new_ports, ne
end
-- ##############################################
-- Function to split ports string comma separated
-- list into array
local function split_port_list(data, is_tcp)
if (is_tcp) then
@ -282,7 +307,9 @@ local function split_port_list(data, is_tcp)
end
-- ##############################################
-- Function to save last scan result
-- in the old case it saves on the FS now with CH
-- it saves on DB
local function save_last_result(scan_result, scan_type, host, epoch, last_port_scanned_label)
if(scan_result ~= nil and not ntop.isClickHouseEnabled()) then
@ -298,7 +325,8 @@ local function save_last_result(scan_result, scan_type, host, epoch, last_port_s
end
-- ##############################################
-- Function to verify if a port is open or is a
-- false positive
local function verify_status_ports(possible_changed_ports, host, scan_type, open_ports_case, epoch)
local real_ports = {}
local scan_module = vs_utils.load_module(scan_type)
@ -330,7 +358,8 @@ local function verify_status_ports(possible_changed_ports, host, scan_type, open
end
-- ##############################################
-- Function to analyze ports differences and build
-- an object with the info for the alert
local function analyze_ports_diff(ports_difference, host, scan_type, epoch)
local rsp = {}
local need_to_trigger_alert_after_changes_verification = ports_difference.trigger
@ -381,6 +410,8 @@ local function analyze_ports_diff(ports_difference, host, scan_type, epoch)
return rsp
end
-- ##############################################
-- Function to get ports changes
local function get_ports_changes(host, scan_type, old_data, new_data)
local is_tcp = false
@ -436,14 +467,6 @@ local function check_differences(host, host_name, scan_type, old_data, new_data)
scan_type = 'tcp_portscan'
end
--[[
if tonumber(old_data.ports or 0) ~= tonumber(new_data.ports or 0) then
rsp["num_ports"] = {
old_num_ports = old_data.ports or 0,
new_num_ports = new_data.ports or 0
}
end
--]]
local num_cve_solved = 0
local num_new_cve_issues = 0
local cve_solved = {}
@ -525,7 +548,8 @@ local function check_differences(host, host_name, scan_type, old_data, new_data)
end
-- ##############################################
-- Function to obtain the port from the scan result
-- line
function vs_utils.cleanup_port(is_tcp, line)
local splitted_line = {}
@ -542,6 +566,9 @@ function vs_utils.cleanup_port(is_tcp, line)
end
-- ##############################################
-- Function to cleanup and get info from the scan
-- result.
-- Remove the first/last few lines that contain nmap information that change at each scan
function vs_utils.cleanup_nmap_result(scan_result, scan_type)
if(scan_result ~= nil) then
@ -610,6 +637,7 @@ function vs_utils.cleanup_nmap_result(scan_result, scan_type)
end
end
-- ##############################################
-- Remove the first/last few lines that contain nmap information that change at each scan
function vs_utils.cleanup_nmap_check_host_result(scan_result)
if(scan_result ~= nil) then
@ -703,7 +731,7 @@ local function isAlreadyPresent(item)
end
-- **********************************************************
-- Compare function to sort CVE on the score
local function compare(a,b)
local a_array = split(a,"|")
@ -786,7 +814,7 @@ local function get_counter_periodic_all_scan_keys(exec_type)
end
-- **********************************************************
-- Function to get host id
local function get_host_id(host_details)
local host_id = ternary(isEmptyString(host_details.host_name),host_details.host, string.format("%s (%s)",host_details.host_name,host_details.host))
return host_id
@ -1035,8 +1063,7 @@ function vs_utils.retrieve_hosts_backup()
end
-- **********************************************************
--Function to save
-- Function to save host on configuration
function vs_utils.add_host_pref(scan_type, host, ports, scan_frequency, discovered_host_scan_type, cidr)
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
@ -1064,6 +1091,8 @@ function vs_utils.add_host_pref(scan_type, host, ports, scan_frequency, discover
return result
end
-- *********************************************************
-- Function to edit host on configuration
function vs_utils.edit_host_pref(scan_type, host, ports, scan_frequency, discovered_host_scan_type)
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
local old_item_string = ntop.getHashCache(prefs_host_values_key,host_hash_key)
@ -1080,7 +1109,7 @@ function vs_utils.edit_host_pref(scan_type, host, ports, scan_frequency, discove
end
-- **********************************************************
-- Function to trigger alert when an host is now down
local function trigger_alert_host_down(host,host_name, epoch)
local host_info_to_cache = {
host = host,
@ -1254,15 +1283,9 @@ function vs_utils.save_host_to_scan(scan_type, host, scan_result, last_scan_time
new_item[prefix_key.."ports_case"] = compare_info[prefix_key.."ports_case"]
end
-- edit case
ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(new_item))
local counts = vs_utils.update_ts_counters()
-- save on db here
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,host_hash_key)
-- hash value found
@ -1343,7 +1366,7 @@ function vs_utils.save_host_to_scan(scan_type, host, scan_result, last_scan_time
end
-- **********************************************************
-- Function to update timeseries data
function vs_utils.update_ts_counters()
local hosts_details = vs_utils.retrieve_hosts_to_scan()
local count_cve = 0
@ -1382,7 +1405,7 @@ end
-- **********************************************************
-- Function to format num for emails
-- @param case: 0 - cve, 1 - udp, 2 - tcp
-- @param case: 0 - cve, 1 - udp, 2 - tcp, 3 - scanned_hosts, 4 - hosts unreachable, 5 - no hosts down now
local function format_num_for_email(num, case)
local formatted_num = format_high_num_value_for_tables({num = num}, "num")
@ -1435,7 +1458,7 @@ local function format_num_for_email(num, case)
end
-- **********************************************************
-- Function to retrieve info for email notification
local function retrieve_email_info(exec_type)
local info_redis_key = get_counter_periodic_all_scan_keys(exec_type)
@ -1536,8 +1559,8 @@ local function retrieve_report_info(date)
return info
end
-- **********************************************************
-- **********************************************************
-- params date used only in case of periodic or scan all exec
function vs_utils.generate_report(date)
@ -1549,6 +1572,8 @@ function vs_utils.generate_report(date)
end
-- **********************************************************
-- Function to format open port for email notification
local function add_ports_open_for_email_report(l4_key_prefix, host_details)
if (tonumber(host_details[l4_key_prefix.."_ports"]) > 0) then
return (i18n("hosts_stats.page_scan_hosts.email.host_details_open_ports", {
@ -1560,6 +1585,8 @@ local function add_ports_open_for_email_report(l4_key_prefix, host_details)
return nil
end
-- **********************************************************
-- Function to format hosts details for email notification
local function format_all_hosts_details_info_for_email(all_hosts_details)
local formatted_hosts_details_string = ""
for _,host_details in ipairs(all_hosts_details) do
@ -1792,14 +1819,12 @@ function vs_utils.notify_scan_results(exec_type, periodicity)
end
-- **********************************************************
-- Function to retrieve dates of scan all or periodic scan
function vs_utils.get_scan_all_dates()
return ntop.getCache(hosts_scan_last_report_dates)
end
-- **********************************************************
-- Function to verify if periodic scan is ended
function vs_utils.is_periodic_scan_completed()
local periodicity_scan_in_progress = ntop.getCache(periodic_scan_key) == "1"
@ -1838,7 +1863,6 @@ function vs_utils.is_periodic_scan_completed()
end
-- **********************************************************
-- Function to verify if scan all is ended
function vs_utils.is_ondemand_scan_completed()
local scan_all_in_progress = ntop.getCache(ondemand_scan_key) == "1"
@ -1874,6 +1898,8 @@ function vs_utils.is_ondemand_scan_completed()
return false
end
-- **********************************************************
-- Function to verify if single scan is ended
function vs_utils.is_single_scan_completed()
local single_scan_in_progress = ntop.getCache(single_scan_key) == "1"
@ -1910,14 +1936,14 @@ function vs_utils.is_single_scan_completed()
end
-- **********************************************************
-- Function to enable periodic scan end check on callbacks
function vs_utils.is_periodic_scan_running()
return ntop.getCache(periodic_scan_key) == "1"
end
-- **********************************************************
-- Function to retrieve single host scan info from
-- host hash key
local function retrieve_host_by_hash_key(host_hash_key)
local hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,host_hash_key)
@ -1934,8 +1960,8 @@ local function retrieve_host_by_hash_key(host_hash_key)
end
return hash_value
end
-- **********************************************************
-- **********************************************************
-- Function to retrieve a specific host scan info with scan type CVE
-- (for host_details page )
function vs_utils.retrieve_host(host)
@ -1949,7 +1975,6 @@ function vs_utils.retrieve_host(host)
end
-- **********************************************************
-- Function to retrieve hosts list to scan
function vs_utils.retrieve_hosts_to_scan(epoch)
if (isEmptyString(epoch) or (not ntop.isClickHouseEnabled())) then
@ -1991,7 +2016,6 @@ function vs_utils.retrieve_hosts_to_scan(epoch)
end
-- **********************************************************
-- Function to retrieve hosts list to scan just for status_info
function vs_utils.check_in_progress_status()
local hash_keys = ntop.getHashKeysCache(host_to_scan_key)
@ -2018,7 +2042,6 @@ end
-- **********************************************************
-- Function restrieve scan result from FS
local function retrieve_scan_result_from_file(scan_type, host)
-- for retrocompatibility
local path = get_report_path(scan_type, host)
@ -2078,7 +2101,6 @@ function vs_utils.retrieve_hosts_scan_result(scan_type, host, epoch)
end
-- **********************************************************
-- Function to delete host to scan
function vs_utils.delete_host_to_scan(host, scan_type, all)
if all then
@ -2122,8 +2144,7 @@ function vs_utils.delete_host_to_scan(host, scan_type, all)
return true
end
-- **********************************************************
-- *********************************************************
-- Function to delete host to scan by id
function vs_utils.delete_host_to_scan_by_id(id)
local hosts_details = vs_utils.retrieve_hosts_to_scan()
@ -2149,7 +2170,6 @@ function vs_utils.delete_host_to_scan_by_id(id)
end
-- **********************************************************
-- Function to retrieve scan types list
function vs_utils.retrieve_scan_types()
local scan_types = vs_utils.list_scan_modules()
@ -2163,7 +2183,7 @@ function vs_utils.retrieve_scan_types()
end
-- **********************************************************
-- Function to list scan modules
function vs_utils.list_scan_modules()
local dirs = ntop.getDirs()
local basedir = dirs.scriptdir .. "/lua/modules/vulnerability_scan/modules"
@ -2184,7 +2204,7 @@ function vs_utils.list_scan_modules()
end
-- **********************************************************
-- Function to load VS module by name
function vs_utils.load_module(name)
package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/modules/?.lua;".. package.path
@ -2192,20 +2212,17 @@ function vs_utils.load_module(name)
end
-- **********************************************************
-- Function to discover open ports
function vs_utils.discover_open_ports(host)
local result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports, scan_ports, network_alert_store,now
local scan_module = vs_utils.load_module("tcp_portscan")
now,result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports = scan_module:scan_host(host, ports)
-- FIX ME -> only tcp for now
return format_port_list_to_string(tcp_ports)
end
-- **********************************************************
-- Function to exec single host scan
function vs_utils.scan_host(scan_type, host, ports, scan_id, use_coroutines, cidr)
if(ntop.isShuttingDown()) then return(false) end
@ -2243,7 +2260,6 @@ function vs_utils.scan_host(scan_type, host, ports, scan_id, use_coroutines, cid
return false
end
-- Scan host
local scan_module = vs_utils.load_module(scan_type)
local now,result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports, discovered_hosts = scan_module:scan_host(host, ports, use_coroutines, cidr)
@ -2282,7 +2298,7 @@ function vs_utils.scan_host(scan_type, host, ports, scan_id, use_coroutines, cid
end
-- **********************************************************
-- Function to set singles can in progress redis key
local function set_single_scan_in_progress()
if (ntop.getCache(single_scan_key) ~= '1') then
@ -2298,6 +2314,7 @@ local function set_single_scan_in_progress()
end
-- **********************************************************
-- Function to update single host status
function vs_utils.set_status_scan(scan_type, host, ports, id, is_periodicity, is_all,is_single_scan, status)
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
@ -2330,11 +2347,10 @@ function vs_utils.set_status_scan(scan_type, host, ports, id, is_periodicity, is
end
-- **********************************************************
-- Function to schedule ondemand single host scan
function vs_utils.schedule_ondemand_single_host_scan(scan_type, host, ports, scan_id, is_periodicity, is_all, is_single_scan, cidr)
local scan = { scan_type = scan_type, host = host, ports = ports, id= scan_id, cidr = cidr}
vs_utils.set_status_scan(scan_type, host, ports, scan_id, is_periodicity, is_all, is_single_scan, vs_utils.scan_status.scheduled)
ntop.rpushCache(host_scan_queue_key, json.encode(scan))
@ -2343,7 +2359,7 @@ function vs_utils.schedule_ondemand_single_host_scan(scan_type, host, ports, sca
end
-- **********************************************************
-- Function to schedule ondemand scan all hosts
function vs_utils.schedule_ondemand_all_hosts_scan()
local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()
@ -2371,7 +2387,6 @@ function vs_utils.schedule_ondemand_all_hosts_scan()
end
-- **********************************************************
-- periodicity can be set to "1day" "1week" "disabled"
function vs_utils.schedule_periodic_scan(periodicity)
local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()
@ -2413,7 +2428,6 @@ function vs_utils.schedule_periodic_scan(periodicity)
end
-- **********************************************************
-- Process a single host scan request that has been queued
function vs_utils.process_oldest_scheduled_scan(use_coroutines)
if(ntop.isShuttingDown()) then return(false) end
@ -2446,8 +2460,7 @@ function vs_utils.process_oldest_scheduled_scan(use_coroutines)
end
-- **********************************************************
-- Process a single host scan request that has been queued
-- Process all host scans request that has been queued
function vs_utils.process_all_scheduled_scans(max_num_scans, use_coroutines)
local num = 0
local co = {}
@ -2515,17 +2528,14 @@ function vs_utils.process_all_scheduled_scans(max_num_scans, use_coroutines)
end
-- **********************************************************
-- Example vs_utils.get_active_hosts("192.168.2.0", "24")
function vs_utils.get_active_hosts(host, cidr)
local result = vs_utils.exec_netscan(host, cidr)
return result
end
-- **********************************************************
-- Function to cleanup netscan result
function vs_utils.netscan_cleanup(scan_result)
scan_result = lines(scan_result)
@ -2551,8 +2561,7 @@ function vs_utils.netscan_cleanup(scan_result)
end
-- **********************************************************
-- Example vs_utils.get_active_hosts("192.168.2.0", "24")
-- Example vs_utils.exec_netscan("192.168.2.0", "24")
function vs_utils.exec_netscan(host, cidr)
local result = {}
local out
@ -2601,7 +2610,6 @@ function vs_utils.exec_netscan(host, cidr)
end
-- **********************************************************
-- Update all scan frequencies
function vs_utils.update_all_periodicity(scan_frequency)
local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()
@ -2633,7 +2641,7 @@ function vs_utils.update_all_periodicity(scan_frequency)
end
-- **********************************************************
-- Function to verify if VS is enabled
function vs_utils.is_available()
if (ntop.isnEdge()) then
return false
@ -2643,7 +2651,7 @@ function vs_utils.is_available()
end
-- **********************************************************
-- Function to run command
function vs_utils.runCommand(scan_command, use_coroutines)
local result
local debug_me = false
@ -2680,7 +2688,7 @@ function vs_utils.runCommand(scan_command, use_coroutines)
end
-- **********************************************************
-- Function to scan host
function vs_utils.nmap_scan_host(command, host_ip, ports, use_coroutines, module_name)
local scan_command
@ -2712,7 +2720,7 @@ function vs_utils.nmap_scan_host(command, host_ip, ports, use_coroutines, module
end
-- **********************************************************
-- Function to check if a host is available
function vs_utils.nmap_check_host(host_ip, use_coroutines)
local nmap = vs_utils.get_nmap_path()
local scan_command = nmap.." -sn"
@ -2734,8 +2742,8 @@ function vs_utils.nmap_check_host(host_ip, use_coroutines)
return is_up, scan_duration, start_scan, end_scan
end
-- **********************************************************
-- **********************************************************
-- Migrate old configurations
function vs_utils.migrate_keys()
local old_hash_key = "ntopng.prefs.host_to_scan"
@ -2781,7 +2789,6 @@ function vs_utils.migrate_keys()
end
-- **********************************************************
-- init once
if(ntop.getCache("ntopng.prefs.vs.vs_slow_scan") == "1") then
use_slow_scan = " -T polite --max-parallelism 1"
@ -2823,7 +2830,6 @@ end
-- **********************************************************
-- Function to find if an ip is configured with a specific scan_type
function vs_utils.isVSConfiguredHostScanType(ip,scan_type)
local host_configured_key = vs_utils.get_host_hash_key(ip, scan_type)
local pref_value = ntop.getHashCache(prefs_host_values_key,host_configured_key)
@ -2833,7 +2839,6 @@ end
-- **********************************************************
-- Function to find if an ip is configured
function vs_utils.isVSConfiguredHost(ip)
local hosts_scanned = ntop.getHashKeysCache(prefs_host_values_key) or {}
for key, _ in pairs(hosts_scanned) do
@ -2847,7 +2852,6 @@ end
-- **********************************************************
-- Function to trigger an alert if the host
-- is not configured
function vs_utils.triggerHostNotConfiguredAlert(host, scan_type)
trigger_alert_host_not_configured(host, scan_type)
end
@ -2858,25 +2862,26 @@ end
-- VS DB Functions
-- **********************************************************
-- Function to retrieve reports list from the DB
function vs_utils.retrieve_report_list(epoch)
local sort_on = "DATE"
return (vs_db_utils.retrieve_reports(sort_on,epoch))
end
-- **********************************************************
-- Function to retrieve single report from the DB
function vs_utils.retrieve_report(report_name)
return (vs_db_utils.retrieve_report(report_name))
end
-- **********************************************************
-- Function to delete single report on the DB
function vs_utils.delete_report(epoch)
return(vs_db_utils.delete_report(epoch))
end
-- **********************************************************
-- Function to edit single report on the DB
function vs_utils.edit_report(epoch, report_name)
return(vs_db_utils.edit_report(epoch,report_name))
end