ntopng/scripts/lua/modules/vulnerability_scan/vs_utils.lua

--
-- (C) 2013-23 - ntop.org
--

--
-- This file implements some utility functions used by the REST API
-- in the vulnerability pages
--
--
-- https://geekflare.com/nmap-vulnerability-scan/
-- cd /usr/share/nmap/scripts/
-- git clone https://github.com/scipag/vulscan.git
-- ln -s `pwd`/scipag_vulscan /usr/share/nmap/scripts/vulscan
-- cd vulscan/utilities/updater/
-- chmod +x updateFiles.sh
-- ./updateFiles.sh
--
-- Example:
-- nmap -sV --script vulscan --script-args vulscandb=openvas.csv <target> -p 80,233
--
--
-- exploitdb.csv
-- osvdb.csv
-- securitytracker.csv
-- openvas.csv
-- scipvuldb.csv
-- xforce.csv
-- securityfocus.csv
-- cve.csv
--

-- **********************************************************

local dirs = ntop.getDirs()
package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path
package.path = dirs.installdir .. "/scripts/lua/pro/modules/?.lua;" .. package.path
package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/?.lua;" .. package.path
package.path = dirs.installdir .. "/scripts/lua/modules/recipients/?.lua;" .. package.path

require "lua_utils" -- used by tprint (debug)

local host_to_scan_key              = "ntopng.prefs.host_to_scan"
local host_to_scan_periodicity_key  = "ntopng.prefs.host_to_scan.periodicity_scan"
local host_scannned_count_key       = "ntopng.prefs.host_to_scan.count_scanned"
local host_scan_queue_key           = "ntopng.vs_scan_queue"
local scanned_hosts_changes_key     = "ntopng.alerts.scanned_hosts_changes"

local json = require("dkjson")
local format_utils = require("format_utils")
local recipients = require("recipients")
local cve_utils = require("cve_utils")

local debug_print = false
local vs_utils = {}

-- **********************************************************

function vs_utils.get_host_hash_key(host, scan_type)
   return string.format("%s-%s",host,scan_type)
end

-- **********************************************************

vs_utils.scan_status = {
   error       = 0,
   ok          = 1,
   scheduled   = 2,
   not_scanned = 3,
   scanning    = 4
}

vs_utils.tcp_ports_diff_case = {
   no_diff          = 2, -- case 1 or 2 (combined)
   ntopng_more_t_vs = 3,
   vs_more_t_ntopng = 4
}

-- **********************************************************

function vs_utils.is_nmap_installed()
   local path = {
      "/usr/bin/nmap",
      "/usr/local/bin/nmap",
      "/opt/homebrew/bin/nmap"
   }
   local module_path = {
      "/usr/share/nmap/scripts/",
      "opt/homebrew/share/nmap/scripts/vulscan/",
      "/usr/local/share/nmap/scripts/vulscan",
   }

   for _,p in pairs(path) do
      if(ntop.exists(p)) then
	 -- nmap is present. Now check if vulscan is present
	 for _,m in pairs(module_path) do
	    if(ntop.exists(m)) then
	       return true
	    end
	 end
      end
   end

   return false
end

-- **********************************************************

local function get_report_path(scan_type, ip, all)
   local base_dir = dirs.workingdir .. "/-1/vulnerability_scan"
   ntop.mkdir(base_dir)
   local ret = ""

   if (not all or all == nil) then
      ret = base_dir .. "/"..ip.."_"..scan_type..".txt"
   else
      ret = base_dir .. "/*.txt"
   end

   return(ret)
end

-- ##############################################

local function lines(str)
   local result = {}

   for line in str:gmatch '[^\n]+' do
      table.insert(result, line)
   end
   return result
end


-- ##############################################

local function format_port_list_to_string(ports)
   local scan_ports = ""

   if (ports ~= nil and #ports > 0) then
      for index,port in ipairs(ports) do
         if (index == 1) then
            scan_ports = ""..port
         else
            scan_ports = scan_ports .. ","..port
         end
      end
   end

   return scan_ports

end

-- ##############################################
local function find_port(port, port_list)
   local found = false
   for _,item in ipairs(port_list) do
      if (item == port) then
         found = true
         break
      end
   end
   return found
end

local function check_ports_diffences(num_old_ports, old_ports, num_new_ports, new_ports)
   local rsp = {
      trigger = true
   }
   if (num_old_ports == 0 and num_new_ports ~= 0) then
      rsp.open_ports = new_ports
      rsp.open_ports_num = num_new_ports
      rsp.closed_ports_num = 0
      rsp.case = 'new_ports'
   elseif(num_old_ports ~= 0 and num_new_ports == 0) then
      rsp.open_ports_num = 0
      rsp.closed_ports_num = num_old_ports
      rsp.closed_ports = old_ports
      rsp.case = 'ports_closed'
   elseif(num_old_ports ~= 0 and num_new_ports ~= 0) then
      local closed_ports = {}
      local open_ports = {}

      local diff = false
      for _, item in ipairs(old_ports) do
         local is_open = find_port(item, new_ports)
         if (not is_open) then
            closed_ports[#closed_ports+1] = item
            diff = true
         end
      end

      for _, item in ipairs(new_ports) do
         local is_open = find_port(item, old_ports)
         if (not is_open) then
            open_ports[#open_ports+1] = item
            diff = true
         end
      end


      if((not diff) and (num_old_ports == num_new_ports)) then
         rsp.trigger = false
      else
         rsp.open_ports = open_ports
         rsp.open_ports_num = #open_ports
         rsp.closed_ports = closed_ports
         rsp.closed_ports_num = #closed_ports

         if (#open_ports ~= 0 and #closed_ports == 0) then
            rsp.case = 'new_ports'
         elseif (#open_ports == 0 and #closed_ports ~= 0) then
            rsp.case = 'ports_closed'
         else
            rsp.case = 'ports_open_and_closed'
         end
      end
   else
      rsp.trigger = false
   end

   return rsp
end

-- ##############################################

-- This function checks the differences between an old and a new host scan
-- and return a table containing those differences
local function check_differences(host, host_name, scan_type, old_data, new_data)
   local rsp = {}
   -- security checks
   if host == nil or scan_type == nil then
      return nil
   end

   if tonumber(old_data.ports or 0) ~= tonumber(new_data.ports or 0) then
      rsp["num_ports"] = {
         old_num_ports = old_data.ports or 0,
         new_num_ports = new_data.ports or 0
      }
   end

   local num_cve_solved = 0
   local num_new_cve_issues = 0
   local cve_solved = {}
   local new_cve = {}
   -- Checking the solved vulnerabilities
   for _, cve in ipairs(old_data.cve or {}) do
      -- If the new table does not contains the cve it means that it is solved
      if not (table.contains(new_data.cve or {}, cve)) then
         num_cve_solved = num_cve_solved + 1
         -- Add at most 5 cve
         if num_cve_solved <= 5 then
            cve_solved[#cve_solved + 1] = cve
         end
      end
   end

   -- Checking the new vulnerabilities
   for _, cve in ipairs(new_data.cve or {}) do
      -- If the new table does not contains the cve it means that it is solved
      if not (table.contains(old_data.cve or {}, cve)) then
         num_new_cve_issues = num_new_cve_issues + 1
         -- Add at most 5 cve
         if num_new_cve_issues <= 5 then
            new_cve[#new_cve + 1] = cve
         end
      end
   end

   -- Checking old_open_tcp_ports and new_open_tcp_ports
   local tcp_old_ports = {}
   local tcp_new_ports = {}


   if (debug_print) then
      tprint("OLD_DATA_TCP_PORTS")
      tprint(old_data.tcp_ports)

      tprint("NEW_DATA_TCP_PORTS")
      tprint(new_data.tcp_ports)
   end

   if (old_data.tcp_ports and old_data.tcp_ports.num_ports ~= 0) then
      tcp_old_ports = split(old_data.tcp_ports.ports,",")
      if (debug_print) then
         tprint(tcp_old_ports)
      end
   end
   if (new_data.tcp_ports.num_ports ~= 0) then

      tcp_new_ports = split(new_data.tcp_ports.ports, ",")
      if (debug_print) then
         tprint(tcp_new_ports)
      end
   end



   local ports_differences = check_ports_diffences(#tcp_old_ports, tcp_old_ports,
                                                   #tcp_new_ports, tcp_new_ports)

   if num_cve_solved > 0 then
      rsp["num_cve_solved"] = num_cve_solved
      rsp["cve_solved"] = cve_solved
   end

   if num_new_cve_issues > 0 then
      rsp["num_new_cve_issues"] = num_new_cve_issues
      rsp["new_cve"] = new_cve
   end

   if (ports_differences.trigger) then

      if (debug_print) then
         tprint("found ports differences")
         tprint(ports_differences)
      end
      rsp["open_ports"] = {
         num = ports_differences.open_ports_num,
         ports = format_port_list_to_string(ports_differences.open_ports)
      }
      rsp["closed_ports"] =  {
         num = ports_differences.closed_ports_num,
         ports = format_port_list_to_string(ports_differences.closed_ports)
      }
      rsp["tcp_ports_case"] = ports_differences.case

      if (debug_print) then
         tprint(ports_differences.case)
      end
   elseif (debug_print) then
      tprint("IS IT TRIGGERED: ")
      tprint(ports_differences.trigger)
   end


   if table.empty(rsp) then
      rsp = nil
   else
      rsp["host"] = host
      rsp["host_name"] = host_name
      rsp["scan_type"] = scan_type
   end

   return rsp
end

-- ##############################################

function vs_utils.cleanup_port(is_tcp, line)
   local splitted_line = {}

   local regex =  "([^/udp]+)"

   if (is_tcp) then
      regex =  "([^/tcp]+)"
   end
   for str in string.gmatch(line, regex) do
      table.insert(splitted_line, str)
   end

   return splitted_line[1]

end

-- remove the first/last few lines that contain nmap information that change at each scan
function vs_utils.cleanup_nmap_result(scan_result, scan_type)
   scan_result = scan_result:gsub("|", "")
   scan_result = scan_result:gsub("_", "")

   scan_result = lines(scan_result)

   for i=1,4 do
      table.remove(scan_result, 1)
   end

   table.remove(scan_result, #scan_result)
  
   local num_open_ports = 0
   local num_vulnerabilities = 0
   local cve = {}
   local scan_out = {}
   local tcp_ports = {}
   local udp_ports = {}

   for _,l in pairs(scan_result) do
      if(string.find(l, "open") ~= nil) then
	 local t = string.find(l, "/tcp ") or 0
	 local u = string.find(l, "/udp ") or 0

	 if (t > 0) then
	    num_open_ports = num_open_ports + 1
	    tcp_ports[#tcp_ports+1] = vs_utils.cleanup_port(true, l)
	 end

	 if(u > 0) then
	    num_open_ports = num_open_ports + 1
	    udp_ports[#udp_ports+1] = vs_utils.cleanup_port(false, l)
	 end
      end

      if(string.sub(l, 1, 2) == " [") then
	 local c = string.split(string.sub(l,3), "]")

	 if(scan_type == "cve") then
	    l = '[<A HREF="https://nvd.nist.gov/vuln/detail/'..c[1]..'">'..c[1]..'</A>]'..c[2]
	 elseif(scan_type == "openvas") then
	    l = '[<A HREF="https://vulners.com/openvas/OPENVAS:'..c[1]..'">'..c[1]..'</A>]'..c[2]
	 end

	 table.insert(cve, c[1])
	 num_vulnerabilities = num_vulnerabilities + 1
      end

      table.insert(scan_out, l)
   end

   scan_result = table.concat(scan_out, "\n")

   return scan_result, num_open_ports, num_vulnerabilities, cve, udp_ports, tcp_ports
end

-- **********************************************************
-- remove the first/last few lines that contain nmap information that change at each scan
function vs_utils.cleanup_nmap_vulners_result(scan_result, scan_type)
   scan_result = scan_result:gsub("|_", "")
   scan_result = scan_result:gsub("|", "")

   scan_result = lines(scan_result)

   for i=1,4 do
      table.remove(scan_result, 1)
   end

   table.remove(scan_result, #scan_result)

   local num_open_ports = 0
   local num_vulnerabilities = 0
   local cve = {}
   local scan_out = {}

   for _,l in pairs(scan_result) do
      if(string.find(l, "open") ~= nil) then
	 local t = string.find(l, "/tcp ") or 0
	 local u = string.find(l, "/udp ") or 0

	 if((t > 0) or (u > 0)) then
	    num_open_ports = num_open_ports + 1
	 end
      end

      if(string.find(l, "https://vulners.com/") ~= nil) then
	 local c = string.split(l, "\t")
	 table.insert(cve, c[2])
	 num_vulnerabilities = num_vulnerabilities + 1
      end

      table.insert(scan_out, l)
   end

   scan_result = table.concat(scan_out, "\n")

   return scan_result, num_open_ports, num_vulnerabilities, cve
end

-- **********************************************************

-- Function to save host configuration
local function isAlreadyPresent(item)

   local hosts_details = vs_utils.retrieve_hosts_to_scan()
   for _,value in ipairs(hosts_details) do
      if (item.host == value.host and item.scan_type == value.scan_type ) then
         return true
      end
   end
   return false
end

-- **********************************************************

local function compare(a,b)

   local a_array = split(a,"|")
   local a_has_score = false
   if (#a_array > 1) then
      a_has_score = true
      a = a_array[2]
   end

   local b_array = split(b,"|")
   local b_has_score = false
   if (#b_array > 1) then
      b_has_score = true
      b = b_array[2]
   end

   if (a_has_score and b_has_score) then
      return a > b
   else
      return a_array[1] > b_array[1]
   end
end

-- **********************************************************

-- Function to format cve list with scores
local function get_cve_with_score(cve)
   local cve_with_score_list = {}
   local max_score = 0
   if(cve ~= nil) then
      for _,cve_id in ipairs(cve) do
	 local score = cve_utils.getCVEscore(cve_id)

	 local cve_formatted = cve
	 if(score ~= nil) then

	    if(max_score < score) then
	       max_score = score
	    end
	    cve_formatted = string.format("%s|%s",cve_id,score)
	 end
	 cve_with_score_list[#cve_with_score_list+1] = cve_formatted
      end
   end

   if next(cve_with_score_list) then
      table.sort(cve_with_score_list, compare)
   end

   return cve_with_score_list, max_score
end

-- **********************************************************

-- Function to save host configuration
function vs_utils.save_host_to_scan(scan_type, host, scan_result, last_scan_time, last_duration,
				    is_ok_last_scan, ports, scan_frequency, num_open_ports,
				    num_vulnerabilities_found, cve, id, is_edit, udp_ports, tcp_ports)
   local checks = require "checks"
   local trigger_alert = checks.isCheckEnabled("active_monitoring", "vulnerability_scan")
      or checks.isCheckEnabled("system", "vulnerability_scan")
   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
   local old_data_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
   local old_data = json.decode(old_data_string)
   -- Getting the hostname, the only way is to scan all the interfaces and retrieve it
   local host_name = ntop.resolveName(host)
   if host_name == host then
      host_name = ""
   end
   -- In case the alert needs to be triggered, save the differences in order to lessen
   -- the info dropped on redis
   -- if is_ok_last_scan is nil then no prior scan was done, so do not trigger the alert
   if trigger_alert and old_data and (not is_edit)  then
      local already_scanned = (old_data.last_scan and old_data.last_scan.epoch)

      if already_scanned then

	 if(debug_print) then
	    tprint("ALREADY PRESENT-> CHECKING DIFF")
	 end

	 local host_info_to_cache = check_differences(host, host_name,
						      scan_type,
						      {
							 vulnerabilities = old_data.num_vulnerabilities_found,
							 ports = old_data.num_open_ports,
							 cve = old_data.cve,
							 tcp_ports = {num_ports = old_data.tcp_ports, ports = old_data.tcp_ports_list },
						      },
						      {
							 vulnerabilities = num_vulnerabilities_found,
							 ports = num_open_ports,
							 cve = cve,
							 tcp_ports = tcp_ports
	 })
         if host_info_to_cache then
            ntop.rpushCache(scanned_hosts_changes_key, json.encode(host_info_to_cache))
         end
      end
   end

   local epoch_id = 0
   if isEmptyString(id) then
      local key = "ntopng.prefs.last_host_id"
      local res = ntop.incrCache(key)
      epoch_id = res
   else
      epoch_id = id
   end

   if (isEmptyString(is_ok_last_scan)) then
      is_ok_last_scan = vs_utils.scan_status.not_scanned
   end


   local cve_formatted, max_score_cve = get_cve_with_score(cve)
   local new_item = {
      host = host,
      host_name = host_name,
      scan_type = scan_type,
      ports = ports,
      num_open_ports = num_open_ports,
      num_vulnerabilities_found = num_vulnerabilities_found,
      cve = cve_formatted,
      max_score_cve = max_score_cve,
      id = epoch_id,
      is_ok_last_scan = is_ok_last_scan
   }

   if tcp_ports ~= nil then

      new_item.tcp_ports = tcp_ports.num_ports
      new_item.tcp_ports_list = tcp_ports.ports
   end

   if udp_ports ~= nil then
      new_item.udp_ports = #udp_ports
   end

   if (udp_ports == nil and tcp_ports == nil) then
      new_item.tcp_ports = num_open_ports
   end

   if last_scan_time or last_duration then
      --local time_formatted = format_utils.formatEpoch(last_scan_time)
      if last_duration <= 0 then
         last_duration = 1
      end

      last_duration = secondsToTime(last_duration)
      new_item.last_scan = {
	 epoch = last_scan_time,
	 --time  = time_formatted,
	 duration = last_duration
      }

      if is_ok_last_scan == vs_utils.scan_status.ok then
	 new_item.is_ok_last_scan = vs_utils.scan_status.ok
      end


   end

   if not isEmptyString(scan_frequency) then
      new_item.scan_frequency = scan_frequency
   elseif old_data and not isEmptyString(old_data.scan_frequency) then
      new_item.scan_frequency = old_data.scan_frequency
   end

   if(scan_result ~= nil) then
      local handle = io.open(get_report_path(scan_type, host), "w")
      local result = handle:write(scan_result)
      handle:close()
   end

   if not isEmptyString(id) and is_edit then
      vs_utils.delete_host_to_scan_by_id(id)
   end

   local result = 1 -- success
   if(not isAlreadyPresent(new_item)) then
      --saved_hosts[#saved_hosts+1] = new_item
      ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(new_item))
   elseif not isEmptyString(id) then

      -- edit case
      ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(new_item))
   else
      result = 2 --aleready_present
   end

   local counts = vs_utils.update_ts_counters()

   vs_utils.notify_end_periodicity()



   --ntop.setCache(host_to_scan_key, json.encode(saved_hosts))
   return result, new_item.id
end

function vs_utils.update_ts_counters()
   local hosts_details = vs_utils.retrieve_hosts_to_scan()

   local count_cve         = 0
   local hosts_scanned
   local open_ports_count  = 0
   local hosts_count       = 0

   for _,item in ipairs(hosts_details) do

      hosts_count = hosts_count + 1

      if item.num_open_ports ~= nil then
         open_ports_count = open_ports_count + item.num_open_ports
      end
      if item.num_vulnerabilities_found ~= nil then
         count_cve = count_cve + item.num_vulnerabilities_found
      end

   end

   local count = ntop.getCache(host_scannned_count_key)
   if (not isEmptyString(count)) then
      hosts_scanned = tonumber(count)
   end

   local response = {
      cve_count = count_cve,
      scanned_hosts = hosts_scanned,
      open_ports = open_ports_count,
      hosts_count = hosts_count
   }

   return response
end


function vs_utils.notify_end_periodicity()

   local periodicity_scan_in_progress = ntop.getCache(host_to_scan_periodicity_key) == "1"

   if (periodicity_scan_in_progress) then
      local hosts_details = vs_utils.retrieve_hosts_to_scan()
      for _,item in ipairs(hosts_details) do
         if(item.is_periodicity and item.is_ok_last_scan == vs_utils.scan_status.scheduled) then
            return
         end
      end

      ntop.setCache(host_to_scan_periodicity_key, "0")

      local periodicity = ntop.getCache(host_to_scan_periodicity_key.."type")


      for _,item in ipairs(hosts_details) do
         local host_hash_key = vs_utils.get_host_hash_key(item.host, item.scan_type)
         local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
         if(not isEmptyString(host_hash_value_string)) then

            local host_hash_value = json.decode(host_hash_value_string)

            host_hash_value.is_periodicity = false

            ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
         end
      end

      local notification_message = ""
      if (periodicity == "1day") then
         notification_message = i18n("hosts_stats.page_scan_hosts.periodicity_scan_1_day_ended")
      elseif (periodicity == "1week") then
         notification_message = i18n("hosts_stats.page_scan_hosts.periodicity_scan_1_week_ended")
      end
      recipients.sendMessageByNotificationType({periodicity = periodicity, success=true, message = notification_message}, "vulnerability_scans")
   end

end

-- **********************************************************

-- Function to retrieve a specific host scan info
function vs_utils.retrieve_host(host)
   local hosts_scanned = ntop.getHashKeysCache(host_to_scan_key) or {}
   for key, _ in pairs(hosts_scanned) do
      if key:find(host) then
         return json.decode(ntop.getHashCache(host_to_scan_key, key) or "")
      end
   end
   return nil
end

-- **********************************************************

-- Function to retrieve hosts list to scan
function vs_utils.retrieve_hosts_to_scan()
   local hash_keys = ntop.getHashKeysCache(host_to_scan_key)
   local rsp = {}
   if hash_keys then
      for k in pairs(hash_keys) do
         local hash_value_string = ntop.getHashCache(host_to_scan_key, k)
         if (not isEmptyString(hash_value_string)) then
            local hash_value = json.decode(hash_value_string)
            rsp[#rsp+1] = hash_value
         end
      end
   end

   return rsp
end

-- **********************************************************

-- Function to retrieve hosts list to scan just for status_info
function vs_utils.check_in_progress_status()

   local hash_keys = ntop.getHashKeysCache(host_to_scan_key)
   local total_in_progress = 0
   local total = 0
   if hash_keys then
      for k in pairs(hash_keys) do
         local hash_value_string = ntop.getHashCache(host_to_scan_key, k)

         if (not isEmptyString(hash_value_string)) then
            local hash_value = json.decode(hash_value_string)
            -- Check IN PROGRESS --> FIX ME with enums
            if hash_value and (hash_value.is_ok_last_scan == vs_utils.scan_status.scheduled or hash_value.is_ok_last_scan == vs_utils.scan_status.scanning) then
               total_in_progress = total_in_progress + 1
            end
            total = total + 1
         end
      end
   end

   return total, total_in_progress
end

-- **********************************************************

-- Function to retrieve last host scan result
function vs_utils.retrieve_hosts_scan_result(scan_type, host)
   local path = get_report_path(scan_type, host)

   if(ntop.exists(path)) then
      local handle = io.open(path, "r")
      local result = handle:read("*a")
      handle:close()

      return result
   else
      return ""
   end
end

-- **********************************************************

-- Function to delete host to scan
function vs_utils.delete_host_to_scan(host, scan_type, all)
   if all then
      ntop.delCache(host_to_scan_key)
      ntop.delCache(host_scan_queue_key)
      ntop.delCache(host_to_scan_periodicity_key)
      ntop.delCache(host_to_scan_periodicity_key.."type")

      local path_to_s_result = get_report_path(scan_type, host, true)
      os.execute("rm -f "..path_to_s_result)
   else
      local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
      local path_to_s_result = get_report_path(scan_type, host, false)
      os.remove(path_to_s_result)
      ntop.delHashCache(host_to_scan_key, host_hash_key)

      -- Remove this host from active schedules
      local elems = {}
      while(true) do
	 local e = ntop.lpopCache(host_scan_queue_key)

	 if(e == nil) then
	    break
	 else
	    local r = json.decode(e)
	    if(not((r.scan_type == "cve") and (r.host == "127.0.0.1"))) then
	       table.insert(elems, e)
	    end
	 end
      end

      for _,i in pairs(elems) do
	 ntop.lpushCache(host_scan_queue_key, i)
      end
   end

   return true
end

-- **********************************************************

-- Function to delete host to scan by id
function vs_utils.delete_host_to_scan_by_id(id)

   local hosts_details = vs_utils.retrieve_hosts_to_scan()
   local host_to_delete = {}
   local id_number = tonumber(id)

   for _,value in ipairs(hosts_details) do
      if(tonumber(value.id) == id_number ) then
         host_to_delete.host = value.host
         host_to_delete.scan_type = value.scan_type
         break
      end
   end

   local host_hash_key = vs_utils.get_host_hash_key(host_to_delete.host, host_to_delete.scan_type)
   local path_to_s_result = get_report_path(host_to_delete.scan_type, host_to_delete.host, false)
   os.remove(path_to_s_result)
   ntop.delHashCache(host_to_scan_key, host_hash_key)


   return true
end

-- **********************************************************

-- Function to retrieve scan types list
function vs_utils.retrieve_scan_types()
   local scan_types = vs_utils.list_scan_modules()
   local ret = {}

   for _,scan_type in ipairs(scan_types) do
      table.insert(ret, { id = scan_type, label = i18n("hosts_stats.page_scan_hosts.scan_type_list."..scan_type) })
   end

   return ret
end

-- **********************************************************

function vs_utils.list_scan_modules()
   local dirs = ntop.getDirs()
   local basedir = dirs.scriptdir .. "/lua/modules/vulnerability_scan/modules"
   local modules = {}

   for name in pairs(ntop.readdir(basedir)) do
      if(ends(name, ".lua")) then
	 name = string.sub(name, 1, string.len(name)-4) -- remove .lua trailer
	 local m = vs_utils.load_module(name)

	 if(m:is_enabled()) then
	    table.insert(modules, name)
	 end
      end
   end

   return(modules)
end

-- **********************************************************

function vs_utils.load_module(name)
   package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/modules/?.lua;".. package.path
   return(require(name):new())
end

-- **********************************************************

function vs_utils.discover_open_portsx(host)

   local result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports, scan_ports, network_alert_store,now

   local scan_module = vs_utils.load_module("tcp_portscan")
   now,result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports = scan_module:scan_host(host, ports)

   -- FIX ME -> only tcp for now

   return format_port_list_to_string(tcp_ports)
end

-- **********************************************************

-- Function to exec single host scan
function vs_utils.scan_host(scan_type, host, ports, scan_id)
   if debug_print then
      if (ports ~= nil) then
         traceError(TRACE_NORMAL,TRACE_CONSOLE,"Scanning Host ".. host .. " on Ports: " .. ports .. "\n")
      else
         traceError(TRACE_NORMAL,TRACE_CONSOLE,"Scanning Host ".. host.."\n")
      end
   end

   local ports_scan_param

   if(string.contains(scan_type, '_openports') or string.contains(scan_type, '_portscan')) then
      -- Nothing to do
   else
      if (isEmptyString(ports)) then
	 ports = vs_utils.discover_open_ports(host)
      end
   end
   
   ports_scan_param = ports
   vs_utils.set_status_scan(scan_type, host, ports_scan_param, id, nil, vs_utils.scan_status.scanning)

   local scan_module = vs_utils.load_module(scan_type)
   local now,result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports = scan_module:scan_host(host, ports)

   -- FIX HERE UDP ports
   if(tcp_ports ~= nil) then
      tcp_ports = {ports = format_port_list_to_string(tcp_ports), num_ports = #tcp_ports}
   else
      tcp_ports = {ports = ports, num_ports = num_open_ports}
   end
   if scan_result then
      scan_result = vs_utils.scan_status.ok

      ntop.incrCache(host_scannned_count_key)
   end
   if debug_print then
      traceError(TRACE_NORMAL,TRACE_CONSOLE,"End scan Host ".. host .. ", result: " .. result .. "\n")
   end

   if (isAlreadyPresent({host= host, scan_type= scan_type})) then
      vs_utils.save_host_to_scan(scan_type, host, result, now, duration, scan_result,
				 ports_scan_param, nil, num_open_ports, num_vulnerabilities_found, cve, scan_id, false, udp_ports, tcp_ports)
   end
   return true
end

-- **********************************************************

-- Function to update single host status
function vs_utils.set_status_scan(scan_type, host, ports, id, is_periodicity, status)

   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
   local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
   if(not isEmptyString(host_hash_value_string)) then

      local host_hash_value = json.decode(host_hash_value_string)

      host_hash_value.is_ok_last_scan = status
      if (is_periodicity ~= nil) then
         host_hash_value.is_periodicity = is_periodicity
      end

      ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
   end

   return true
end

-- **********************************************************

function vs_utils.schedule_host_scan(scan_type, host, ports, scan_id, is_periodicity)
   local scan = { scan_type = scan_type, host = host, ports = ports, id= scan_id}
   vs_utils.set_status_scan(scan_type, host, ports, scan_id, is_periodicity, vs_utils.scan_status.scheduled)

   ntop.rpushCache(host_scan_queue_key, json.encode(scan))

   return true
end

-- **********************************************************

function vs_utils.schedule_all_hosts_scan()
   local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()

   if #host_to_scan_list > 0 then
      for _,scan_info in ipairs(host_to_scan_list) do
	 vs_utils.schedule_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, false)
      end
   end

   return true
end

-- **********************************************************

-- periodicity can be set to "1day" "1week" "disabled"
function vs_utils.schedule_periodic_scan(periodicity)

   local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()

   if (#host_to_scan_list > 0 ) then
      local is_already_running = ntop.getCache(host_to_scan_periodicity_key) == "1"
      if not is_already_running then

         local is_scanning_almost_one = false

	 for _,scan_info in ipairs(host_to_scan_list) do
	    local frequency = scan_info.scan_frequency

	    if(frequency == periodicity) then
	       vs_utils.schedule_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, true)
	       is_scanning_almost_one = true
	    end
	 end


         if is_scanning_almost_one then
            ntop.setCache(host_to_scan_periodicity_key , "1")
            ntop.setCache(host_to_scan_periodicity_key.."type", periodicity)

            local notification_message = ""
            if (periodicity == "1day") then
               notification_message = i18n("hosts_stats.page_scan_hosts.periodicity_scan_1_day_started")
            elseif (periodicity == "1week") then
               notification_message = i18n("hosts_stats.page_scan_hosts.periodicity_scan_1_week_started")
            end

            recipients.sendMessageByNotificationType({periodicity = periodicity, success=true, message = notification_message}, "vulnerability_scans")
         end
      end
   end
   return true
end


-- **********************************************************

-- Process a single host scan request that has been queued
function vs_utils.process_oldest_scheduled_scan()
   local elem = ntop.lpopCache(host_scan_queue_key)

   if((elem ~= nil) and (elem ~= "")) then
      if debug_print then
         traceError(TRACE_NORMAL,TRACE_CONSOLE,"Found vulnerability scan: ".. elem .. "\n")
      end

      local elem = json.decode(elem)

      vs_utils.scan_host(elem.scan_type, elem.host, elem.ports, elem.id)

      return true
   else
      return false
   end
end

-- **********************************************************

-- Process a single host scan request that has been queued
function vs_utils.process_all_scheduled_scans(max_num_scans)
   local num = 0

   if(max_num_scans == nil) then max_num_scans = 9999 end

   while(max_num_scans > 0) do
      local res = vs_utils.process_oldest_scheduled_scan()

      if(res == false) then
	 break
      else
	 max_num_scans = max_num_scans - 1
	 num = num + 1
      end
   end

   return num
end

-- **********************************************************

-- Example vs_utils.get_active_hosts("192.168.2.0", "24")
function vs_utils.get_active_hosts(host, cidr)
   local result = {}

   cidr = tonumber(cidr)

   if((cidr == 32) or (cidr == 128)
      or (host:find('.') == nil) -- not dots in IP, it looks symbolic
      or (string.sub(host, -1) ~= "0") -- last digit is not 0, so let's assume /32
   ) then
      result[#result+1] = host -- return it as is
   else
      local s = string.split(host, '%.')
      local net = s[1].."."..s[2].."."..s[3].."."
      local command = 'nmap -sP -n ' .. net .. '1-254 | grep "Nmap scan report for" | cut -d " " -f 5'
      local out = ntop.execCmd(command)
      local l = lines(out)

      for _,h in pairs(l) do
	 result[#result+1] = h
      end
   end

   return result
end

-- **********************************************************

-- Update all scan frequencies
function vs_utils.update_all_periodicity(scan_frequency)
   local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()

   for _,value in ipairs(host_to_scan_list) do
      local host_hash_key = vs_utils.get_host_hash_key(value.host, value.scan_type)
      local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
      if(not isEmptyString(host_hash_value_string)) then

         local host_hash_value = json.decode(host_hash_value_string)

         host_hash_value.scan_frequency = scan_frequency

         ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
      end
   end

   return true
end

-- **********************************************************

function vs_utils.is_available()
   local scan_modules = vs_utils.list_scan_modules()
   return (#scan_modules > 0)
end

-- **********************************************************

function vs_utils.format_port_label(port, service_name, protocol)
   if (isEmptyString(service_name)) then
      return string.format("%s/%s",port,protocol)
   else
      return string.format("%s/%s (%s)",port,protocol,service_name)
   end
end

-- **********************************************************
-- Retrieves detected ports by ntopng

function vs_utils.retrieve_detected_ports(host)
   local host_info = interface.getHostInfo(host)

   local tcp_ports_detected = {}
   local host_in_mem = false
   if (host_info and host_info.used_ports and host_info.used_ports.local_server_ports) then
      for port, l7_proto in pairs(host_info.used_ports.local_server_ports) do
         local port_details = split(port, ":")
         local id_port = port_details[2]
         local l4_proto = port_details[1]

         if (l4_proto == 'tcp') then
            tcp_ports_detected[#tcp_ports_detected+1] = id_port
         end
      end
      host_in_mem = true
   end

   return tcp_ports_detected, host_in_mem
end

-- Search port in ports list
local function find_port(port_to_find, port_list)
   for _, port in ipairs(port_list) do
      if(port_to_find == tonumber(port)) then
         return true
      end
   end
   return false
end

-- Compare vs ports and ntopng detected ports
function vs_utils.compare_ports(vs_scan_port_string_list, ntopng_ports)

   local vs_scan_ports = split(vs_scan_port_string_list, ",")

   local ports_unused = {}
   local filtered_ports = {}
   -- check vs_scan_ports with ntopng_ports
   local not_found_a_port = false
   for _,vs_port in ipairs(vs_scan_ports) do
      local find_actual_port = find_port(tonumber(vs_port), ntopng_ports)

      if (not find_actual_port) then
         not_found_a_port = true
         ports_unused[#ports_unused+1] = vs_port
      end

   end

   local diff_case

   if (not_found_a_port) then
      diff_case = vs_utils.tcp_ports_diff_case.vs_more_t_ntopng
   end

   if (#vs_scan_ports == #ntopng_ports) then
      diff_case = vs_utils.tcp_ports_diff_case.no_diff
   else
      local filtered = false

      for _,ntop_port in ipairs(ntopng_ports) do
         if (not find_port(tonumber(ntop_port), vs_scan_ports)) then
            filtered = true
            filtered_ports[#filtered_ports+1] = ntop_port
         end

      end
      if (filtered) then
         diff_case = vs_utils.tcp_ports_diff_case.ntopng_more_t_vs
      end
   end

   return ports_unused, filtered_ports, diff_case
end

-- **********************************************************

return vs_utils