mirror of
https://github.com/ntop/ntopng.git
synced 2026-04-30 07:59:35 +00:00
615 lines
18 KiB
Lua
615 lines
18 KiB
Lua
--
|
|
-- (C) 2013-23 - ntop.org
|
|
--
|
|
|
|
--
|
|
-- This file implements some utility functions used by the REST API
|
|
-- in the vulnerability pages
|
|
--
|
|
--
|
|
-- https://geekflare.com/nmap-vulnerability-scan/
|
|
-- cd /usr/share/nmap/scripts/
|
|
-- git clone https://github.com/scipag/vulscan.git
|
|
-- ln -s `pwd`/scipag_vulscan /usr/share/nmap/scripts/vulscan
|
|
-- cd vulscan/utilities/updater/
|
|
-- chmod +x updateFiles.sh
|
|
-- ./updateFiles.sh
|
|
--
|
|
-- Example:
|
|
-- nmap -sV --script vulscan --script-args vulscandb=openvas.csv <target> -p 80,233
|
|
--
|
|
--
|
|
-- exploitdb.csv
|
|
-- osvdb.csv
|
|
-- securitytracker.csv
|
|
-- openvas.csv
|
|
-- scipvuldb.csv
|
|
-- xforce.csv
|
|
-- securityfocus.csv
|
|
-- cve.csv
|
|
--
|
|
|
|
-- **********************************************************
|
|
|
|
local dirs = ntop.getDirs()
|
|
package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path
|
|
package.path = dirs.installdir .. "/scripts/lua/pro/modules/?.lua;" .. package.path
|
|
package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/?.lua;" .. package.path
|
|
|
|
require "lua_utils" -- used by tprint (debug)
|
|
|
|
local host_to_scan_key = "ntopng.prefs.host_to_scan"
|
|
local host_scan_queue_key = "ntopng.vs_scan_queue"
|
|
local scanned_hosts_changes_key = "ntopng.alerts.scanned_hosts_changes"
|
|
|
|
local json = require("dkjson")
|
|
local format_utils = require("format_utils")
|
|
|
|
local vs_utils = {}
|
|
|
|
-- **********************************************************
|
|
|
|
function vs_utils.get_host_hash_key(host, scan_type)
|
|
return string.format("%s-%s",host,scan_type)
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
function vs_utils.is_nmap_installed()
|
|
local path = {
|
|
"/usr/bin/nmap",
|
|
"/usr/local/bin/nmap",
|
|
"/opt/homebrew/bin/nmap"
|
|
}
|
|
local module_path = {
|
|
"/usr/share/nmap/scripts/",
|
|
"opt/homebrew/share/nmap/scripts/vulscan/",
|
|
"/usr/local/share/nmap/scripts/vulscan",
|
|
}
|
|
|
|
for _,p in pairs(path) do
|
|
if(ntop.exists(p)) then
|
|
-- nmap is present. Now check if vulscan is present
|
|
for _,m in pairs(module_path) do
|
|
if(ntop.exists(m)) then
|
|
return true
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
return false
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
local function get_report_path(scan_type, ip, all)
|
|
local base_dir = dirs.workingdir .. "/-1/vulnerability_scan"
|
|
ntop.mkdir(base_dir)
|
|
local ret = ""
|
|
if (not all or all == nil) then
|
|
ret = base_dir .. "/"..ip.."_"..scan_type..".txt"
|
|
else
|
|
ret = base_dir .. "/*.txt"
|
|
end
|
|
|
|
return(ret)
|
|
end
|
|
|
|
-- ##############################################
|
|
|
|
local function lines(str)
|
|
local result = {}
|
|
|
|
for line in str:gmatch '[^\n]+' do
|
|
table.insert(result, line)
|
|
end
|
|
return result
|
|
end
|
|
|
|
-- ##############################################
|
|
|
|
-- This function checks the differences between an old and a new host scan
|
|
-- and return a table containing those differences
|
|
local function check_differences(host, scan_type, old_data, new_data)
|
|
local rsp = {}
|
|
-- security checks
|
|
if host == nil or scan_type == nil then
|
|
return nil
|
|
end
|
|
|
|
if tonumber(old_data.ports or 0) ~= tonumber(new_data.ports or 0) then
|
|
rsp["num_ports"] = {
|
|
old_num_ports = old_data.ports or 0,
|
|
new_num_ports = new_data.ports or 0
|
|
}
|
|
end
|
|
|
|
local num_cve_solved = 0
|
|
local num_new_cve_issues = 0
|
|
local cve_solved = {}
|
|
local new_cve = {}
|
|
-- Checking the solved vulnerabilities
|
|
for _, cve in ipairs(old_data.cve or {}) do
|
|
-- If the new table does not contains the cve it means that it is solved
|
|
if not (table.contains(new_data.cve or {}, cve)) then
|
|
num_cve_solved = num_cve_solved + 1
|
|
-- Add at most 5 cve
|
|
if num_cve_solved <= 5 then
|
|
cve_solved[#cve_solved + 1] = cve
|
|
end
|
|
end
|
|
end
|
|
|
|
-- Checking the new vulnerabilities
|
|
for _, cve in ipairs(new_data.cve or {}) do
|
|
-- If the new table does not contains the cve it means that it is solved
|
|
if not (table.contains(old_data.cve or {}, cve)) then
|
|
num_new_cve_issues = num_new_cve_issues + 1
|
|
-- Add at most 5 cve
|
|
if num_new_cve_issues <= 5 then
|
|
new_cve[#new_cve + 1] = cve
|
|
end
|
|
end
|
|
end
|
|
|
|
if num_cve_solved > 0 then
|
|
rsp["num_cve_solved"] = num_cve_solved
|
|
rsp["cve_solved"] = cve_solved
|
|
end
|
|
|
|
if num_new_cve_issues > 0 then
|
|
rsp["num_new_cve_issues"] = num_new_cve_issues
|
|
rsp["new_cve"] = new_cve
|
|
end
|
|
|
|
if table.empty(rsp) then
|
|
rsp = nil
|
|
else
|
|
rsp["host"] = host
|
|
rsp["scan_type"] = scan_type
|
|
end
|
|
|
|
return rsp
|
|
end
|
|
|
|
-- ##############################################
|
|
|
|
-- remove the first/last few lines that contain nmap information that change at each scan
|
|
function vs_utils.cleanup_nmap_result(scan_result, scan_type)
|
|
scan_result = scan_result:gsub("|", "")
|
|
scan_result = scan_result:gsub("_", "")
|
|
|
|
scan_result = lines(scan_result)
|
|
|
|
for i=1,4 do
|
|
table.remove(scan_result, 1)
|
|
end
|
|
|
|
for i=1,3 do
|
|
table.remove(scan_result, #scan_result)
|
|
end
|
|
|
|
local num_open_ports = 0
|
|
local num_vulnerabilities = 0
|
|
local cve = {}
|
|
local scan_out = {}
|
|
|
|
for _,l in pairs(scan_result) do
|
|
local t = string.find(l, "/tcp ") or 0
|
|
local u = string.find(l, "/udp ") or 0
|
|
|
|
if((t > 0) or (u > 0)) then
|
|
num_open_ports = num_open_ports + 1
|
|
end
|
|
|
|
if(string.sub(l, 1, 2) == " [") then
|
|
local c = string.split(string.sub(l,3), "]")
|
|
|
|
if(scan_type == "cve") then
|
|
l = '[<A HREF="https://nvd.nist.gov/vuln/detail/'..c[1]..'">'..c[1]..'</A>]'..c[2]
|
|
elseif(scan_type == "openvas") then
|
|
l = '[<A HREF="https://vulners.com/openvas/OPENVAS:'..c[1]..'">'..c[1]..'</A>]'..c[2]
|
|
end
|
|
|
|
table.insert(cve, c[1])
|
|
num_vulnerabilities = num_vulnerabilities + 1
|
|
end
|
|
|
|
table.insert(scan_out, l)
|
|
end
|
|
|
|
scan_result = table.concat(scan_out, "\n")
|
|
|
|
return scan_result, num_open_ports, num_vulnerabilities, cve
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Function to save host configuration
|
|
function vs_utils.save_host_to_scan(scan_type, host, scan_result, last_scan_time, last_duration,
|
|
is_ok_last_scan, ports, scan_frequency, num_open_ports,
|
|
num_vulnerabilities_found, cve)
|
|
--local saved_hosts_string = ntop.getCache(host_to_scan_key)
|
|
local saved_hosts = {}
|
|
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
|
|
|
|
--if not isEmptyString(saved_hosts_string) then
|
|
local checks = require "checks"
|
|
local trigger_alert = checks.isCheckEnabled("system", "vulnerability_scan") or false
|
|
--saved_hosts = json.decode(saved_hosts_string) or {}
|
|
-- local index_to_remove = 0
|
|
--[[
|
|
for index,value in ipairs(saved_hosts) do
|
|
if value.host == host and value.scan_type == scan_type then
|
|
index_to_remove = index
|
|
end
|
|
end
|
|
--]]
|
|
-- if index_to_remove ~= 0 then
|
|
--local old_data = saved_hosts[index_to_remove]
|
|
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
|
|
local old_data_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
|
|
local old_data = json.decode(old_data_string)
|
|
-- In case the alert needs to be triggered, save the differences in order to lessen
|
|
-- the info dropped on redis
|
|
-- if is_ok_last_scan is nil then no prior scan was done, so do not trigger the alert
|
|
if trigger_alert and old_data and old_data.is_ok_last_scan then
|
|
local host_info_to_cache = check_differences(host,
|
|
scan_type,
|
|
{
|
|
vulnerabilities = old_data.num_vulnerabilities_found,
|
|
ports = old_data.num_open_ports,
|
|
cve = old_data.cve,
|
|
},
|
|
{
|
|
vulnerabilities = num_vulnerabilities_found,
|
|
ports = num_open_ports,
|
|
cve = cve,
|
|
})
|
|
if host_info_to_cache then
|
|
ntop.rpushCache(scanned_hosts_changes_key, json.encode(host_info_to_cache))
|
|
end
|
|
end
|
|
|
|
local new_item = {
|
|
host = host,
|
|
scan_type = scan_type,
|
|
ports = ports,
|
|
num_open_ports = num_open_ports,
|
|
num_vulnerabilities_found = num_vulnerabilities_found,
|
|
cve = cve,
|
|
}
|
|
|
|
if last_scan_time or last_duration then
|
|
local time_formatted = format_utils.formatPastEpochShort(last_scan_time)
|
|
|
|
if last_duration <= 0 then
|
|
last_duration = 1
|
|
end
|
|
|
|
last_duration = secondsToTime(last_duration)
|
|
new_item.last_scan = {
|
|
epoch = last_scan_time,
|
|
time = time_formatted,
|
|
duration = last_duration
|
|
}
|
|
|
|
if is_ok_last_scan then
|
|
new_item.is_ok_last_scan = is_ok_last_scan
|
|
end
|
|
end
|
|
|
|
if not isEmptyString(scan_frequency) then
|
|
new_item.scan_frequency = scan_frequency
|
|
elseif old_data and not isEmptyString(old_data.scan_frequency) then
|
|
new_item.scan_frequency = old_data.scan_frequency
|
|
end
|
|
|
|
if(scan_result ~= nil) then
|
|
local handle = io.open(get_report_path(scan_type, host), "w")
|
|
local result = handle:write(scan_result)
|
|
handle:close()
|
|
end
|
|
|
|
--saved_hosts[#saved_hosts+1] = new_item
|
|
ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(new_item))
|
|
|
|
--ntop.setCache(host_to_scan_key, json.encode(saved_hosts))
|
|
return 1
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Function to retrieve hosts list to scan
|
|
function vs_utils.retrieve_hosts_to_scan()
|
|
|
|
local hash_keys = ntop.getHashKeysCache(host_to_scan_key)
|
|
local rsp = {}
|
|
if hash_keys then
|
|
for k in pairs(hash_keys) do
|
|
local hash_value_string = ntop.getHashCache(host_to_scan_key, k)
|
|
if (not isEmptyString(hash_value_string)) then
|
|
local hash_value = json.decode(hash_value_string)
|
|
rsp[#rsp+1] = hash_value
|
|
end
|
|
end
|
|
end
|
|
|
|
return rsp
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Function to retrieve hosts list to scan just for status_info
|
|
function vs_utils.check_in_progress_status()
|
|
|
|
local hash_keys = ntop.getHashKeysCache(host_to_scan_key)
|
|
|
|
if hash_keys then
|
|
for k in pairs(hash_keys) do
|
|
local hash_value_string = ntop.getHashCache(host_to_scan_key, k)
|
|
|
|
if (not isEmptyString(hash_value_string)) then
|
|
local hash_value = json.decode(hash_value_string)
|
|
-- Check IN PROGRESS --> FIX ME with enums
|
|
if hash_value and hash_value.is_ok_last_scan == 4 then
|
|
return true
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
return false
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Function to retrieve last host scan result
|
|
function vs_utils.retrieve_hosts_scan_result(scan_type, host)
|
|
local path = get_report_path(scan_type, host)
|
|
|
|
if(ntop.exists(path)) then
|
|
local handle = io.open(path, "r")
|
|
local result = handle:read("*a")
|
|
handle:close()
|
|
|
|
return result
|
|
else
|
|
return ""
|
|
end
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Function to delete host to scan
|
|
function vs_utils.delete_host_to_scan(host, scan_type, all)
|
|
if all then
|
|
|
|
ntop.delCache(host_to_scan_key)
|
|
local path_to_s_result = get_report_path(scan_type, host, true)
|
|
os.execute("rm "..path_to_s_result)
|
|
else
|
|
|
|
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
|
|
local path_to_s_result = get_report_path(scan_type, host, false)
|
|
os.remove(path_to_s_result)
|
|
ntop.delHashCache(host_to_scan_key, host_hash_key)
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Function to retrieve scan types list
|
|
function vs_utils.retrieve_scan_types()
|
|
local scan_types = vs_utils.list_scan_modules()
|
|
local ret = {}
|
|
|
|
for _,scan_type in ipairs(scan_types) do
|
|
table.insert(ret, { id = scan_type, label = i18n("hosts_stats.page_scan_hosts.scan_type_list."..scan_type) })
|
|
end
|
|
|
|
return ret
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
function vs_utils.list_scan_modules()
|
|
local dirs = ntop.getDirs()
|
|
local basedir = dirs.scriptdir .. "/lua/modules/vulnerability_scan/modules"
|
|
local modules = {}
|
|
|
|
for name in pairs(ntop.readdir(basedir)) do
|
|
if(ends(name, ".lua")) then
|
|
name = string.sub(name, 1, string.len(name)-4) -- remove .lua trailer
|
|
local m = vs_utils.load_module(name)
|
|
|
|
if(m:is_enabled()) then
|
|
table.insert(modules, name)
|
|
end
|
|
end
|
|
end
|
|
|
|
return(modules)
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
function vs_utils.load_module(name)
|
|
package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/modules/?.lua;".. package.path
|
|
return(require(name):new())
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Function to exec single host scan
|
|
function vs_utils.scan_host(scan_type, host, ports)
|
|
local scan_module = vs_utils.load_module(scan_type)
|
|
local result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve = scan_module:scan_host(host, ports)
|
|
|
|
vs_utils.save_host_to_scan(scan_type, host, result, now, duration, scan_result,
|
|
ports, nil, num_open_ports, num_vulnerabilities_found, cve)
|
|
|
|
return true
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Function to update single host status
|
|
function vs_utils.set_status_scan(scan_type, host, ports)
|
|
|
|
local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
|
|
local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
|
|
if(not isEmptyString(host_hash_value_string)) then
|
|
|
|
local host_hash_value = json.decode(host_hash_value_string)
|
|
|
|
host_hash_value.is_ok_last_scan = 4
|
|
|
|
ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
function vs_utils.schedule_host_scan(scan_type, host, ports)
|
|
local scan = { scan_type = scan_type, host = host, ports = ports }
|
|
vs_utils.set_status_scan(scan_type, host, ports)
|
|
|
|
ntop.rpushCache(host_scan_queue_key, json.encode(scan))
|
|
|
|
return true
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
function vs_utils.schedule_all_hosts_scan(scan_type, host, ports)
|
|
local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()
|
|
|
|
if #host_to_scan_list > 0 then
|
|
for _,scan_info in ipairs(host_to_scan_list) do
|
|
vs_utils.schedule_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports)
|
|
end
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- periodicity can be set to "1day" "1week" "disabled"
|
|
function vs_utils.schedule_periodic_scan(periodicity)
|
|
local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()
|
|
|
|
if #host_to_scan_list > 0 then
|
|
for _,scan_info in ipairs(host_to_scan_list) do
|
|
local frequency = scan_info.scan_frequency
|
|
|
|
if(frequency == periodicity) then
|
|
vs_utils.schedule_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports)
|
|
end
|
|
end
|
|
end
|
|
|
|
return true
|
|
end
|
|
|
|
|
|
-- **********************************************************
|
|
|
|
-- Process a single host scan request that has been queued
|
|
function vs_utils.process_oldest_scheduled_scan()
|
|
local elem = ntop.lpopCache(host_scan_queue_key)
|
|
|
|
if((elem ~= nil) and (elem ~= "")) then
|
|
local elem = json.decode(elem)
|
|
|
|
vs_utils.scan_host(elem.scan_type, elem.host, elem.ports)
|
|
|
|
return true
|
|
else
|
|
return false
|
|
end
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Process a single host scan request that has been queued
|
|
function vs_utils.process_all_scheduled_scans(max_num_scans)
|
|
local num = 0
|
|
|
|
if(max_num_scans == nil) then max_num_scans = 9999 end
|
|
|
|
while(max_num_scans > 0) do
|
|
local res = vs_utils.process_oldest_scheduled_scan()
|
|
|
|
if(res == false) then
|
|
break
|
|
else
|
|
max_num_scans = max_num_scans - 1
|
|
num = num + 1
|
|
end
|
|
end
|
|
|
|
return num
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Example vs_utils.get_active_hosts("192.168.2.0", "24")
|
|
function vs_utils.get_active_hosts(host, cidr)
|
|
local result = {}
|
|
|
|
cidr = tonumber(cidr)
|
|
|
|
if((cidr == 32) or (cidr == 128)
|
|
or (host:find('.') == nil) -- not dots in IP, it looks symbolic
|
|
or (string.sub(host, -1) ~= "0") -- last digit is not 0, so let's assume /32
|
|
) then
|
|
result[#result+1] = host -- return it as is
|
|
else
|
|
local s = string.split(host, '%.')
|
|
local net = s[1].."."..s[2].."."..s[3].."."
|
|
local command = 'nmap -sP -n ' .. net .. '1-254 | grep "Nmap scan report for" | cut -d " " -f 5'
|
|
local handle = io.popen(command)
|
|
local out = handle:read("*a")
|
|
local l = lines(out)
|
|
|
|
handle:close()
|
|
|
|
for _,h in pairs(l) do
|
|
result[#result+1] = h
|
|
end
|
|
end
|
|
|
|
return result
|
|
end
|
|
|
|
-- **********************************************************
|
|
|
|
-- Update all scan frequencies
|
|
function vs_utils.update_all_periodicity(scan_frequency)
|
|
local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()
|
|
|
|
for _,value in ipairs(host_to_scan_list) do
|
|
local host_hash_key = vs_utils.get_host_hash_key(value.host, value.scan_type)
|
|
local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
|
|
if(not isEmptyString(host_hash_value_string)) then
|
|
|
|
local host_hash_value = json.decode(host_hash_value_string)
|
|
|
|
host_hash_value.scan_frequency = scan_frequency
|
|
|
|
ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
|
|
end
|
|
end
|
|
|
|
return true
|
|
end
|
|
-- **********************************************************
|
|
|
|
return vs_utils
|