ntopng/scripts/lua/modules/vulnerability_scan/vs_utils.lua

--
-- (C) 2013-23 - ntop.org
--

--
-- This file implements some utility functions used by the REST API
-- in the vulnerability pages
--
--
-- https://geekflare.com/nmap-vulnerability-scan/
-- cd /usr/share/nmap/scripts/
-- git clone https://github.com/scipag/vulscan.git
-- ln -s `pwd`/scipag_vulscan /usr/share/nmap/scripts/vulscan
-- cd vulscan/utilities/updater/
-- chmod +x updateFiles.sh
-- ./updateFiles.sh
--
-- Example:
-- nmap -sV --script vulscan --script-args vulscandb=openvas.csv <target> -p 80,233
--
--
-- exploitdb.csv
-- osvdb.csv
-- securitytracker.csv
-- openvas.csv
-- scipvuldb.csv
-- xforce.csv
-- securityfocus.csv
-- cve.csv
--

-- **********************************************************

local dirs = ntop.getDirs()
package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path
package.path = dirs.installdir .. "/scripts/lua/pro/modules/?.lua;" .. package.path
package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/?.lua;" .. package.path
package.path = dirs.installdir .. "/scripts/lua/modules/recipients/?.lua;" .. package.path

require "lua_utils" -- used by tprint (debug)

local host_to_scan_key                    = "ntopng.vs.hosts.scanned_values"
local prefs_host_values_key               = "ntopng.prefs.vs.hosts_conf"

local periodic_scan_key                   = "ntopng.vs.periodic_scan"
local periodic_scan_type_key              = "ntopng.vs.periodic_scan_type"
local ondemand_scan_key                   = "ntopng.vs.scan_all"
local single_scan_key                     = "ntopng.vs.single_scan"
local scanned_hosts_count_key             = "ntopng.prefs.host_to_scan.count_scanned"
local scanned_hosts_changes_queue_key     = "ntopng.alerts.scanned_hosts_changes"
local host_in_scanning_hash_key           = "ntopng.vs.hosts.in_scanning"
local host_scan_queue_key                 = "ntopng.vs.scan_queue"

-- redis key for last scan report dates
local hosts_scan_last_report_dates        = "ntopng.vs.report_dates"

-- redis keys for periodic scan info
local periodic_scan_host_info_key         = "ntopng.vs.periodic_scan.info"

-- redis keys for scan all info
local ondemand_scan_host_info_key         = "ntopng.vs.scan_all.info"

-- redis key for single scan info
local single_scan_info_key                = "ntopng.vs.single_scan.info"

local json = require("dkjson")
local format_utils = require("format_utils")
local recipients = require("recipients")
local cve_utils = require("cve_utils")
local vs_db_utils = require("vs_db_utils")
local vs_rest_utils = require("vs_rest_utils")

-- Enable debug with:
-- redis-cli set "ntopng.prefs.vs.debug_enabled" "1"
-- systemctl restart ntopng
local debug_me = ntop.getCache("ntopng.prefs.vs.debug_enabled") == "1"
local test_me = false
local verbose = false

local vs_utils = {}

local use_slow_scan;

-- **********************************************************

function vs_utils.get_host_hash_key(host, scan_type)
   return string.format("%s-%s", host, scan_type)
end

-- **********************************************************

vs_utils.scan_status = {
   error       = 0,
   ok          = 1,
   scheduled   = 2,
   not_scanned = 3,
   scanning    = 4,
   failed      = 5,
}

vs_utils.scan_in_exec_type = {
   single_scan = 1,
   scan_all = 2,
   periodic_scan = 3
}

-- **********************************************************

function vs_utils.get_nmap_path()
   local path = {
      "/usr/bin/nmap",
      "/usr/local/bin/nmap",
      "/opt/homebrew/bin/nmap"
   }

   for _,p in pairs(path) do
      if(ntop.exists(p)) then
         return(p..use_slow_scan)
      end
   end

   return(nil)
end

-- **********************************************************

function vs_utils.is_nmap_installed()
   local module_path = {
      "/usr/share/nmap/scripts/",
      "/opt/homebrew/share/nmap/scripts/vulscan/",
      "/usr/local/share/nmap/scripts/vulscan",
   }
   local path = vs_utils.get_nmap_path()

   if(path ~= nil) then
      for _,m in pairs(module_path) do
         if(ntop.exists(m)) then
            return true
         end
      end
   end

   return false
end

-- **********************************************************

local function get_report_path(scan_type, ip, all)
   local base_dir
   local ret = ""

   base_dir = dirs.workingdir .. "/" .. getSystemInterfaceId() .. "/vulnerability_scan"
   ntop.mkdir(base_dir)

   if (not all or all == nil) then
      ret = base_dir .. "/"..ip.."_"..scan_type..".txt"
   else
      ret = base_dir .. "/*.txt"
   end

   return(ret)
end

-- ##############################################

local function lines(str)
   local result = {}

   for line in str:gmatch '[^\n]+' do
      table.insert(result, line)
   end
   return result
end


-- ##############################################

local function format_port_list_to_string(ports)
   local scan_ports = ""

   if (ports ~= nil and #ports > 0) then
      for index,port in ipairs(ports) do
         if (index == 1) then
            scan_ports = ""..port
         else
            scan_ports = scan_ports .. ","..port
         end
      end
   end

   return scan_ports
end

-- ##############################################

local function find_port(port, port_list)
   local found = false

   for _,item in ipairs(port_list) do
      if (item == port) then
         found = true
         break
      end
   end
   return found
end

-- ##############################################

local function check_ports_diffences(num_old_ports, old_ports, num_new_ports, new_ports)
   local rsp = {
      trigger = true
   }
   if (num_old_ports == 0 and num_new_ports ~= 0) then
      rsp.open_ports = new_ports
      rsp.open_ports_num = num_new_ports
      rsp.closed_ports_num = 0
      rsp.case = 'new_ports'
   elseif(num_old_ports ~= 0 and num_new_ports == 0) then
      rsp.open_ports_num = 0
      rsp.closed_ports_num = num_old_ports
      rsp.closed_ports = old_ports
      rsp.case = 'ports_closed'
   elseif(num_old_ports ~= 0 and num_new_ports ~= 0) then
      local closed_ports = {}
      local open_ports = {}

      local diff = false
      for _, item in ipairs(old_ports) do
         local is_open = find_port(item, new_ports)
         if (not is_open) then
            closed_ports[#closed_ports+1] = item
            diff = true
         end
      end

      for _, item in ipairs(new_ports) do
         local is_open = find_port(item, old_ports)
         if (not is_open) then
            open_ports[#open_ports+1] = item
            diff = true
         end
      end


      if((not diff) and (num_old_ports == num_new_ports)) then
         rsp.trigger = false
      else
         rsp.open_ports = open_ports
         rsp.open_ports_num = #open_ports
         rsp.closed_ports = closed_ports
         rsp.closed_ports_num = #closed_ports

         if (#open_ports ~= 0 and #closed_ports == 0) then
            rsp.case = 'new_ports'
         elseif (#open_ports == 0 and #closed_ports ~= 0) then
            rsp.case = 'ports_closed'
         else
            rsp.case = 'ports_open_and_closed'
         end
      end
   else
      rsp.trigger = false
   end
   return rsp
end

-- ##############################################

local function split_port_list(data, is_tcp)

   if (is_tcp) then
      if (data.tcp_ports and data.tcp_ports.num_ports ~= 0) then
         return split(data.tcp_ports.ports,",")
      end
   else
      if(data.udp_ports and data.udp_ports.num_ports ~= 0) then
         return split(data.udp_ports.ports, ",")
      end
   end

   return {}
end

-- ##############################################

local function save_last_result(scan_result, scan_type, host, epoch, last_port_scanned_label)
   
   if(scan_result ~= nil and not ntop.isClickHouseEnabled()) then
      local handle = io.open(get_report_path(scan_type, host), "a")
      local result = handle:write("\n\n"..i18n("hosts_stats.page_scan_hosts.inconsistency_state", {port = last_port_scanned_label}).."\n"..scan_result)
      handle:close()
   end

   if (scan_result ~= nil and ntop.isClickHouseEnabled()) then
      vs_db_utils.update_last_result(scan_result, scan_type, host, epoch, last_port_scanned_label)
   end

end

-- ##############################################

local function verify_status_ports(possible_changed_ports, host, scan_type, open_ports_case, epoch)
   local real_ports = {}
   local scan_module = vs_utils.load_module(scan_type)
   if (test_me) then
      tprint("TROVATA inconsistent")
      ntop.msleep(10000)
      tprint("RESCANNING")
   end
   for _,possible_changed_port in ipairs(possible_changed_ports) do
      local now,result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports = scan_module:scan_host(host, possible_changed_port)
      
      local scan_type_label = ternary(scan_type == "tcp_portscan", "tcp", "udp")
      local port_label_for_vs_result = string.format("%u/%s",possible_changed_port,scan_type_label)
      save_last_result(result, scan_type, host, epoch, port_label_for_vs_result)
      if debug_me then
         tprint("SCANNED AGAIN HOST: "..host.." ON PORT: "..possible_changed_port)
      end
      if (open_ports_case and num_open_ports > 0) then
         -- case open port and num_open_ports is > 0 so is a real open port
         real_ports[#real_ports+1] = possible_changed_port
         if debug_me then
            tprint("IS A REAL OPEN")
         end
      elseif (open_ports_case == false and num_open_ports == 0) then
         -- case closed port and num_open_ports is 0 so is a real closed port
         real_ports[#real_ports+1] = possible_changed_port
         if debug_me then
            tprint("IS A REAL CLOSED")
         end
      end
   end
   return real_ports
end

-- ##############################################

local function analyze_ports_diff(ports_difference, host, scan_type, epoch)
   local rsp = {}
   local need_to_trigger_alert_after_changes_verification = ports_difference.trigger
   if (ports_difference.trigger) then
      if (debug_me) then
         tprint("found ports differences")
         tprint(ports_difference)
      end
      local open_ports_case =  true

      local real_open_ports = verify_status_ports(ports_difference.open_ports or {}, host, scan_type, open_ports_case, epoch )
      local real_open_port_num = ternary(real_open_ports ~= nil and next(real_open_ports), #real_open_ports, 0)
      rsp["open_ports"] = {
         num = real_open_port_num,
         ports = format_port_list_to_string(real_open_ports)
      }
      open_ports_case = false
      local real_closed_ports = verify_status_ports(ports_difference.closed_ports or {}, host, scan_type, open_ports_case, epoch)
      local real_closed_ports_num = ternary(real_closed_ports ~= nil and next(real_closed_ports), #real_closed_ports, 0)

      rsp["closed_ports"] =  {
         num = real_closed_ports_num,
         ports = format_port_list_to_string(real_closed_ports)
      }
      rsp["ports_case"] = ports_difference.case

      if ((real_closed_ports_num == ports_difference.closed_ports_num) and (real_open_port_num == ports_difference.open_ports_num)) then
         
         -- case ok we can trigger alert
      else
         -- false positive detected
         if (debug_me) then
            tprint("IT'S A FALSE POSITIVE!!!")
         end

         need_to_trigger_alert_after_changes_verification = false
      end
      if (debug_me) then
         tprint(ports_difference.case)
      end
   elseif (debug_me) then
      tprint("IS IT TRIGGERED: ")
      tprint(ports_difference.trigger)
   end

   rsp["triggered"] = need_to_trigger_alert_after_changes_verification

   return rsp
end

local function get_ports_changes(host, scan_type, old_data, new_data)

   local is_tcp = false
   local scan_type_log_label = "UDP"

   local rsp = {}

   if(scan_type == 'tcp_portscan' or scan_type == 'tcp_openports') then
      is_tcp = true
      scan_type_log_label = "TCP"
   end

   local old_ports = split_port_list(old_data, is_tcp)
   local new_ports = split_port_list(new_data, is_tcp)

   if (debug_me) then
      tprint(scan_type_log_label.." OLD PORTS: ")
      tprint(old_ports)
      tprint(scan_type_log_label.." NEW PORTS: ")
      tprint(new_ports)
   end

   local ports_differences = check_ports_diffences(#old_ports, old_ports,
                                                         #new_ports, new_ports)
   local rsp_diff = analyze_ports_diff(ports_differences, host, scan_type, new_data.last_scan_time)
   if (rsp_diff.triggered) then
      rsp["open_ports"] = rsp_diff.open_ports
      rsp["closed_ports"] = rsp_diff.closed_ports
      rsp["ports_case"] = rsp_diff.ports_case

   end
   rsp["triggered"] = rsp_diff.triggered

   rsp["measurement"] = "ports_changes_detected"

   return rsp
end

-- ##############################################

-- This function checks the differences between an old and a new host scan
-- and return a table containing those differences
local function check_differences(host, host_name, scan_type, old_data, new_data)
   local rsp = {}

   -- security checks
   if host == nil or scan_type == nil then
      return nil
   end

   -- retrocompatibility check
   if (scan_type == 'tcp_openports') then
      scan_type = 'tcp_portscan'
   end

   --[[
   if tonumber(old_data.ports or 0) ~= tonumber(new_data.ports or 0) then
      rsp["num_ports"] = {
         old_num_ports = old_data.ports or 0,
         new_num_ports = new_data.ports or 0
      }
   end
   --]]
   local num_cve_solved = 0
   local num_new_cve_issues = 0
   local cve_solved = {}
   local new_cve = {}

   -- Checking the solved vulnerabilities
   for _, cve in ipairs(old_data.cve or {}) do
      -- the old cves have the score
      cve = split(cve,"|")[1]
      -- If the new table does not contains the cve it means that it is solved
      if not (table.contains(new_data.cve or {}, cve)) then
         num_cve_solved = num_cve_solved + 1
         -- Add at most 5 cve
         if num_cve_solved <= 5 then
            cve_solved[#cve_solved + 1] = cve
         end
      end
   end

   -- Checking the new vulnerabilities
   for _, cve in ipairs(new_data.cve or {}) do
      -- If the new table does not contains the cve it means that it is solved
      if not (table.contains(old_data.cve or {}, cve)) then
         num_new_cve_issues = num_new_cve_issues + 1
         -- Add at most 5 cve
         if num_new_cve_issues <= 5 then
            new_cve[#new_cve + 1] = cve
         end
      end
   end

   -- ***************************************************************

   -- Checking differences beetwen old and new open ports and old and new closed ports
   local differences = get_ports_changes(host, scan_type, old_data, new_data)
   rsp["triggered"] = differences.triggered
   rsp["measurement"] = "ports_changes_detected"
   local prefix_key = "udp_"
   if (scan_type == "tcp_portscan") then
      prefix_key = "tcp_"
   end
   if (differences.triggered) then
      rsp[prefix_key.."open_ports"] = differences.open_ports
      rsp[prefix_key.."closed_ports"] = differences.closed_ports
      rsp[prefix_key.."ports_case"] = differences.ports_case
   end

   -- ***************************************************************

   if num_cve_solved > 0 then
      rsp["num_cve_solved"] = num_cve_solved
      rsp["cve_solved"] = cve_solved
      rsp["measurement"] = "cve_changes_detected"
   end

   if num_new_cve_issues > 0 then
      rsp["num_new_cve_issues"] = num_new_cve_issues
      rsp["new_cve"] = new_cve
      rsp["measurement"] = "cve_changes_detected"
   end


   if table.empty(rsp) or rsp.triggered == false  then
      rsp = nil
   else
      rsp["host"] = host
      rsp["host_name"] = host_name
      rsp["scan_type"] = scan_type

      if (new_data.last_scan_time) then
         local date_string = format_utils.formatPastEpochShort(new_data.last_scan_time)
         date_string = string.gsub(date_string," ","_")
         rsp["date"] = date_string
         rsp["epoch"] = new_data.last_scan_time
      end
   end

   return rsp
end

-- ##############################################

function vs_utils.cleanup_port(is_tcp, line)
   local splitted_line = {}

   local regex =  "([^/udp]+)"

   if (is_tcp) then
      regex =  "([^/tcp]+)"
   end
   for str in string.gmatch(line, regex) do
      table.insert(splitted_line, str)
   end

   return splitted_line[1]

end

-- Remove the first/last few lines that contain nmap information that change at each scan
function vs_utils.cleanup_nmap_result(scan_result, scan_type)
   if(scan_result ~= nil) then
      scan_result = scan_result:gsub("|", "")
      scan_result = scan_result:gsub("_", "")

      scan_result = lines(scan_result)

      for i=1,4 do
         table.remove(scan_result, 1)
      end

      table.remove(scan_result, #scan_result)

      local num_open_ports = 0
      local num_vulnerabilities = 0
      local cve = {}
      local scan_out = {}
      local tcp_ports = {}
      local udp_ports = {}

      for _,l in pairs(scan_result) do
         -- Ignore "open|filtered" ports
         if((string.find(l, "open") ~= nil) and (string.find(l, "filtered") == nil)) then
            local t = string.find(l, "/tcp ") or 0
            local u = string.find(l, "/udp ") or 0

            if (t > 0) then
               num_open_ports = num_open_ports + 1
               tcp_ports[#tcp_ports+1] = vs_utils.cleanup_port(true, l)
            end

            if(u > 0) then
               num_open_ports = num_open_ports + 1
               udp_ports[#udp_ports+1] = vs_utils.cleanup_port(false, l)
            end
         end

         -- Escape XML/HTML code that might be present in the output
         l = l:gsub("<", "&lt;")
         l = l:gsub(">", "&gt;")

         if(string.sub(l, 1, 2) == " [") then
            local c = string.split(string.sub(l,3), "]")
            local url = cve_utils.getDocURL(c[1], scan_type)

            if(scan_type == "cve") then
                  l = '[<A HREF="'..url..'">'..c[1]..'</A>]'..c[2]
            elseif(scan_type == "openvas") then
               l = '[<A HREF="'..url..'">'..c[1]..'</A>]'..c[2]
            end

            table.insert(cve, c[1])
            num_vulnerabilities = num_vulnerabilities + 1
         end

         table.insert(scan_out, l)
      end

      scan_result = table.concat(scan_out, "\n")
      return scan_result, num_open_ports, num_vulnerabilities, cve, udp_ports, tcp_ports
   else
      return "", 0, 0, 0, 0, 0
   end
end

-- Remove the first/last few lines that contain nmap information that change at each scan
function vs_utils.cleanup_nmap_check_host_result(scan_result)
   if(scan_result ~= nil) then

      if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Result: "..scan_result.."\n") end

      scan_result = scan_result:gsub("|", "")
      scan_result = scan_result:gsub("_", "")

      scan_result = lines(scan_result)

      -- remove the first line and the last one
      table.remove(scan_result, 1)
      table.remove(scan_result, #scan_result)

      local is_up_and_run = false 
      for _,l in pairs(scan_result) do
         -- searching for "Host is up" nmap string
         if (string.find(l, "Host is up") ~= nil) then
            is_up_and_run = true
            goto continue
         end
      end

      ::continue::
      return is_up_and_run
   else
      return false
   end
end

-- **********************************************************

-- Remove the first/last few lines that contain nmap information that change at each scan
function vs_utils.cleanup_nmap_vulners_result(scan_result, scan_type)
   scan_result = scan_result:gsub("|_", "")
   scan_result = scan_result:gsub("|", "")

   scan_result = lines(scan_result)

   for i=1,4 do
      table.remove(scan_result, 1)
   end

   table.remove(scan_result, #scan_result)

   local num_open_ports = 0
   local num_vulnerabilities = 0
   local cve = {}
   local scan_out = {}

   for _,l in pairs(scan_result) do
      if(string.find(l, "open") ~= nil) then
         local t = string.find(l, "/tcp ") or 0
         local u = string.find(l, "/udp ") or 0

         if((t > 0) or (u > 0)) then
            num_open_ports = num_open_ports + 1
         end
      end

      if(string.find(l, "https://vulners.com/") ~= nil) then
         local c = string.split(l, "\t")
         table.insert(cve, c[2])
         num_vulnerabilities = num_vulnerabilities + 1
      end

      table.insert(scan_out, l)
   end

   scan_result = table.concat(scan_out, "\n")

   return scan_result, num_open_ports, num_vulnerabilities, cve
end

-- **********************************************************

-- Function to save host configuration
local function isAlreadyPresent(item)
   local hosts_details = vs_utils.retrieve_hosts_to_scan()

   for _,value in ipairs(hosts_details) do
      if (item.host == value.host and item.scan_type == value.scan_type ) then
         return true
      end
   end
   return false
end

-- **********************************************************

local function compare(a,b)

   local a_array = split(a,"|")
   local a_has_score = false
   if (#a_array > 1) then
      a_has_score = true
      a = a_array[2]
   end

   local b_array = split(b,"|")
   local b_has_score = false
   if (#b_array > 1) then
      b_has_score = true
      b = b_array[2]
   end

   if (a_has_score and b_has_score) then
      return a > b
   else
      return a_array[1] > b_array[1]
   end
end

-- **********************************************************

-- Function to format cve list with scores
local function get_cve_with_score(cve)
   local cve_with_score_list = {}
   local max_score = 0
   if(cve ~= nil) then
      for _,cve_id in ipairs(cve) do
         local score = cve_utils.getCVEscore(cve_id)

         local cve_formatted = cve
         if(score ~= nil) then

            if(max_score < score) then
               max_score = score
            end
            cve_formatted = string.format("%s|%s",cve_id,score)
         end
         cve_with_score_list[#cve_with_score_list+1] = cve_formatted
      end
   end

   if next(cve_with_score_list) then
      table.sort(cve_with_score_list, compare)
   end

   return cve_with_score_list, max_score
end

-- **********************************************************

-- Function to remove scanning host
local function remove_scanning_host(host_info)
   local host_to_scan_hash_key = vs_utils.get_host_hash_key(host_info.host, host_info.scan_type)
   ntop.delHashCache(host_in_scanning_hash_key,host_to_scan_hash_key)
end

-- **********************************************************

-- Function to set the actual scanning host on a redis key
local function save_scanning_host(scan_info)
   local host_to_scan_hash_key = vs_utils.get_host_hash_key(scan_info.host, scan_info.scan_type)
   ntop.setHashCache(host_in_scanning_hash_key, host_to_scan_hash_key, json.encode(scan_info))
end

-- **********************************************************

-- Function to select correctly redis keys on periodic or scan all
local function get_counter_periodic_all_scan_keys(exec_type)
   if (exec_type == vs_utils.scan_in_exec_type.periodic_scan) then
      return periodic_scan_host_info_key --host_periodic_scan_cve_num_key,host_periodic_scan_udp_ports_key,host_periodic_scan_tcp_ports_key
   elseif (exec_type == vs_utils.scan_in_exec_type.scan_all) then
      return ondemand_scan_host_info_key --host_scan_all_cve_num_key,host_scan_all_udp_ports_key,host_scan_all_tcp_ports_key
   elseif(exec_type == vs_utils.scan_in_exec_type.single_scan) then
      return single_scan_info_key
   end
end

-- **********************************************************
local function get_host_id(host_details)
   local host_id =  ternary(isEmptyString(host_details.host_name),host_details.host, string.format("%s (%s)",host_details.host_name,host_details.host))
   return host_id
end
-- **********************************************************

-- Function to update counters of periodically scan or scan all
-- @param is_periodic (true -> is a periodic scan, false -> is a scan all)
local function update_scan_info_for_report(type_of_scan_execution, new_item, host_hash_key, discrepancies, is_down)
   -- select correctly redis keys
   local redis_info_key = get_counter_periodic_all_scan_keys(type_of_scan_execution)

   local info_string = ntop.getCache(redis_info_key)
   local info_json = nil
   if (info_string ~= nil) then
      info_json = json.decode(info_string)
   end

   if (info_json == nil) then
      info_json = {}
   end

   local host_id = get_host_id(new_item)
   -- handle hosts down list for email
   if (is_down) then
      info_json.down_hosts = true
      local host_down = i18n("hosts_stats.page_scan_hosts.email.host_down_item", {
         host_id = host_id
      }) 

      if (info_json and isEmptyString(info_json.down_hosts_string_list)) then
         info_json.down_hosts_string_list = host_down
      else
         info_json.down_hosts_string_list = info_json.down_hosts_string_list .. host_down
      end

      if (info_json and info_json.not_scanned_hosts ~= nil) then
         -- count just in success case 
         info_json.not_scanned_hosts = tonumber(info_json.not_scanned_hosts) + 1
      else
         info_json.not_scanned_hosts = 1
      end
      goto continue
   end
   -- **********************************************************
   if (new_item.num_vulnerabilities_found ~= nil) then

      if (info_json ~= {} and info_json.cves ~= nil) then
         info_json.cves = tonumber(info_json.cves) + new_item.num_vulnerabilities_found
      else
         info_json.cves = 0
      end
   end

   if (new_item.udp_ports ~= nil) then

      if (info_json ~= {} and info_json.udp_ports ~= nil) then
         info_json.udp_ports = tonumber(info_json.udp_ports) + new_item.udp_ports
      else
         info_json.udp_ports = 0
      end

   end

   if (new_item.tcp_ports ~= nil) then
      if (info_json ~= {} and info_json.tcp_ports ~= nil) then
         info_json.tcp_ports = tonumber(info_json.tcp_ports) + new_item.tcp_ports
      else
         info_json.tcp_ports = 0
      end
   end

   if (new_item.is_ok_last_scan == vs_utils.scan_status.ok) then
      if (info_json ~= {} and info_json.scanned_hosts ~= nil) then
            -- count just in success case 
            info_json.scanned_hosts = tonumber(info_json.scanned_hosts) + 1
      else
         info_json.scanned_hosts = 1
      end      
   end


   if (info_json ~= {} and info_json.begin_epoch == nil) then
      info_json.begin_epoch = os.time()
   end

   if discrepancies then
      -- HAD DISCREPANCY INFO TO EMAIL REPORT
      local cve_case = false
      if (info_json ~= {} and info_json.num_cve_solved ~= nil and new_item.scan_type == 'cve') then
         info_json.num_cve_solved = tonumber(info_json.num_cve_solved or 0) + tonumber(discrepancies.num_cve_solved or 0)
         cve_case = true
      end

      local host_discrepancies_details = ""
      local port_type = "TCP"
      local prefix_key = "tcp_"
      if (new_item.scan_type == 'udp_portscan' and info_json ~= {}) then
         port_type = "UDP"
         prefix_key = "udp_"
      end

      
      if (not cve_case and info_json ~= {}) then
         -- DISCREPANCY PORTS CASES
         info_json.discrepancy_case = 'ports'

         if (tonumber(discrepancies[prefix_key.."open_ports"].num or 0) > 0) then
            -- take only open ports not the closed ports
            info_json.new_open_ports = tonumber(info_json.new_open_ports or 0) + tonumber(discrepancies[prefix_key.."open_ports"].num or 0)
            host_discrepancies_details = i18n("hosts_stats.page_scan_hosts.email.host_port_discrepancy_description", {
               host_id = host_id,
               port_type = port_type,
               ports = discrepancies[prefix_key.."open_ports"].ports
            })
            
            if (isEmptyString(info_json.hosts_discrepancies_details)) then
               info_json.hosts_discrepancies_details = host_discrepancies_details
            else
               info_json.hosts_discrepancies_details = info_json.hosts_discrepancies_details .. host_discrepancies_details
            end
         end
      end

      if (cve_case and info_json ~= {} and info_json.num_cve_solved and info_json.num_cve_solved > 0) then
         -- DISCREPANCY CVE CASE
         info_json.discrepancy_case = 'cves'

         local cve_string = format_port_list_to_string(discrepancies.cve_solved)
         host_discrepancies_details = i18n("hosts_stats.page_scan_hosts.email.host_cve_discrepancy_description", {
            host_id = host_id,
            cves = cve_string,
         })

         if (isEmptyString(info_json.hosts_discrepancies_details)) then
            info_json.hosts_discrepancies_details = host_discrepancies_details
         else
            info_json.hosts_discrepancies_details = info_json.hosts_discrepancies_details .. host_discrepancies_details
         end
      end

      if (debug_me) then
         if (info_json) then
            tprint(info_json.hosts_discrepancies_details)
         end
      end

   end

   ::continue::
   ntop.setCache(redis_info_key, json.encode(info_json))
end

-- **********************************************************

-- Function to restore scanning host
function vs_utils.restore_host_to_scan()
   local hash_keys = ntop.getHashKeysCache(host_in_scanning_hash_key)

   if hash_keys then
      for k in pairs(hash_keys) do
         local hash_value_string = ntop.getHashCache(host_in_scanning_hash_key, k)

         if (not isEmptyString(hash_value_string)) then
            local host_info_to_restore = json.decode(hash_value_string)

            if (host_info_to_restore) then
               -- enqueue to scan
               ntop.lpushCache(host_scan_queue_key, hash_value_string)

               -- set status to scheduled
               vs_utils.set_status_scan(host_info_to_restore.scan_type, host_info_to_restore.host, host_info_to_restore.ports,
                                        host_info_to_restore.id, host_info_to_restore.is_periodicity, host_info_to_restore.is_all, host_info_to_restore.is_single_scan,
                                        vs_utils.scan_status.scheduled)
            end
         end
      end
   end
end

-- **********************************************************

-- Function to restore backup config
function vs_utils.restore_config_backup(vs_backup)
   -- remove old hash entries
   ntop.delCache(host_to_scan_key)
   ntop.delCache(prefs_host_values_key)

   for _,item in ipairs(vs_backup) do
      -- restoring hash entries with status not scanned
      local host_hash_key = vs_utils.get_host_hash_key(item.host, item.scan_type)

      local item_to_restore = item
      ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(item_to_restore))
      ntop.setHashCache(prefs_host_values_key, host_hash_key, json.encode(item_to_restore))

   end
end

-- *********************************************************

-- Function to retrieve hosts keys for backups
function vs_utils.retrieve_hosts_backup()
   local hash_keys = ntop.getHashKeysCache(prefs_host_values_key)
   local rsp = {}
   if hash_keys then
      for k in pairs(hash_keys) do
         local hash_value_string = ntop.getHashCache(prefs_host_values_key, k)
         if (not isEmptyString(hash_value_string)) then
            local hash_value = json.decode(hash_value_string)
            rsp[#rsp+1] = hash_value
         end
      end
   end

   return rsp
end

-- **********************************************************

--Function to save
function vs_utils.add_host_pref(scan_type, host, ports, scan_frequency)

   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)

   local new_item = {
      host = host,
      host_name = host_name,
      scan_type = scan_type,
      ports = ports,
   }
   if not isEmptyString(scan_frequency) then
      new_item.scan_frequency = scan_frequency
   end

   local result = 1 -- success
   if (debug_me) then
      tprint("SAVING HOST: "..new_item.host)
   end

   --saved_hosts[#saved_hosts+1] = new_item
   ntop.setHashCache(prefs_host_values_key, host_hash_key, json.encode(new_item))

   return result
end

function vs_utils.edit_host_pref(scan_type, host, ports, scan_frequency)
   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
   local old_item_string = ntop.getHashCache(prefs_host_values_key,host_hash_key)
   if (not isEmptyString(old_item_string)) then
      local old_item = json.decode(old_item_string)
      old_item.ports = ports
      old_item.scan_frequency = scan_frequency

      ntop.setHashCache(prefs_host_values_key, host_hash_key, json.encode(old_item))
      return 1 --ok
   end
   return 2 -- not found
end

-- **********************************************************

local function trigger_alert_host_down(host,host_name, epoch)
   local host_info_to_cache = {
      host = host,
      host_name =  host_name,
      epoch = epoch,
      is_up_check_case = true,
   }
   ntop.rpushCache(scanned_hosts_changes_queue_key, json.encode(host_info_to_cache))

end
-- **********************************************************

-- Function to update host scan values
function vs_utils.save_host_to_scan(scan_type, host, scan_result, last_scan_time, last_duration,
                                    is_ok_last_scan, ports, scan_frequency, num_open_ports,
                                    num_vulnerabilities_found, cve, id, is_edit, udp_ports, tcp_ports)
   local checks = require "checks"
   local trigger_alert = checks.isCheckEnabled("active_monitoring", "vulnerability_scan")
      or checks.isCheckEnabled("system", "vulnerability_scan")
   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
   local old_data_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
   local old_data = json.decode(old_data_string)
   -- Getting the hostname, the only way is to scan all the interfaces and retrieve it
   local host_name = ntop.resolveName(host)
   if host_name == host then
      host_name = ""
   end
   -- In case the alert needs to be triggered, save the differences in order to lessen
   -- the info dropped on redis
   -- if is_ok_last_scan is nil then no prior scan was done, so do not trigger the alert
   

   local epoch_id = 0
   if isEmptyString(id) then
      local key = "ntopng.prefs.last_host_id"
      local res = ntop.incrCache(key)
      epoch_id = res
   else
      epoch_id = id
   end
   local is_down = false
   if (isEmptyString(is_ok_last_scan)) then
      -- first time saved without scan
      -- check if possible
      is_ok_last_scan = vs_utils.scan_status.not_scanned
   elseif (is_ok_last_scan == vs_utils.scan_status.failed and trigger_alert) then
      -- case host is not up and running, possible just in TCP/UDP portscan

      trigger_alert_host_down(host,host_name,last_scan_time)
      
      is_down = true
   end

   local cve_formatted, max_score_cve = get_cve_with_score(cve)


   local new_item = {
      num_open_ports = num_open_ports,
      num_vulnerabilities_found = num_vulnerabilities_found,
      cve = cve_formatted,
      max_score_cve = max_score_cve,
      is_ok_last_scan = is_ok_last_scan,
      host_name = host_name
   }

   if (is_down) then
      new_item.is_down = true
   end
   if tcp_ports ~= nil then
      new_item.tcp_ports = tcp_ports.num_ports
      new_item.tcp_ports_list = tcp_ports.ports
   end

   if udp_ports ~= nil then
      new_item.udp_ports = udp_ports.num_ports
      new_item.udp_ports_list = udp_ports.ports
   end

   if (udp_ports == nil and tcp_ports == nil) then
      new_item.tcp_ports = num_open_ports
   end

   if last_scan_time or last_duration then
      --local time_formatted = format_utils.formatEpoch(last_scan_time)
      if (last_duration == nil) or (last_duration <= 0) then
         last_duration = 1
      end

      last_duration = secondsToTime(last_duration)
      new_item.last_scan = {
         epoch = last_scan_time,
         --time  = time_formatted,
         duration = last_duration,
         duration_epoch = last_duration
      }

      if is_ok_last_scan == vs_utils.scan_status.ok then
         new_item.is_ok_last_scan = vs_utils.scan_status.ok
      end
   end

   if not isEmptyString(scan_frequency) then
      new_item.scan_frequency = scan_frequency
   elseif old_data and not isEmptyString(old_data.scan_frequency) then
      new_item.scan_frequency = old_data.scan_frequency
   end

   -- the is_periodicity, is_all and is_single_scan params are set outside the save_host_to_scan into the set_status method
   if (old_data and old_data.is_periodicity ~= nil) then
      new_item.is_periodicity = old_data.is_periodicity
   end

   if (old_data and old_data.is_all ~= nil) then
      new_item.is_all = old_data.is_all
   end

   if (old_data and old_data.is_single_scan ~= nil) then
      new_item.is_single_scan = old_data.is_single_scan
   end

   if(scan_result ~= nil and not ntop.isClickHouseEnabled()) then
      local handle = io.open(get_report_path(scan_type, host), "w")
      local result = handle:write(scan_result)
      handle:close()
   end

   local result = 1 -- success

   if (debug_me) then
      -- traceError(TRACE_NORMAL, TRACE_CONSOLE, "Updating host " .. host)
   end

   -- add ports statistics comparing the ntopng passive monitoring info
   if (scan_type == "tcp_portscan" or scan_type == "tcp_openports" or scan_type == "udp_portscan") then
      local compare_info = vs_rest_utils.compare_scan_info_ntopng_info(host, scan_type, new_item.tcp_ports_list, new_item.udp_ports_list)
      
      new_item.host_in_mem = compare_info.host_in_mem
      local prefix_key = "udp_"
      if (scan_type == "tcp_portscan" or scan_type == "tcp_openports") then
         prefix_key = "tcp_"
      end

      new_item[prefix_key.."ports_unused"] = compare_info[prefix_key.."ports_unused"]
      new_item[prefix_key.."ports_filtered"] = compare_info[prefix_key.."ports_filtered"]
      new_item[prefix_key.."ports_case"] = compare_info[prefix_key.."ports_case"]

   end
   -- edit case
   ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(new_item))

   local counts = vs_utils.update_ts_counters()

   

   -- save on db here

   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
   local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,host_hash_key)
   -- hash value found
   local hash_pref_value = json.decode(hash_prefs_string)
   if (hash_pref_value ~= nil) then 
      -- is necessary this check to avoid to save entries not presents in the config and to save all data of the entry.
      for key,value in pairs(hash_pref_value) do
         if (key ~= 'is_ok_last_scan') then
            new_item[key] = value
         end
      end
      
      vs_db_utils.save_vs_result(scan_type, host, new_item.last_scan.epoch, json.encode(new_item), scan_result)
   end

   local host_info_differences
   if trigger_alert and old_data and (not is_edit)  then
      local already_scanned = (old_data.last_scan and old_data.last_scan.epoch)

      if already_scanned then

         if debug_me then
            -- traceError(TRACE_NORMAL,TRACE_CONSOLE, "Vulnerability Scan: checking for changes in host")
         end

         local old_cve_no_score = {}
         for _,cve in ipairs(old_data.cve) do
            old_cve_no_score[#old_cve_no_score+1] = split(cve,"|")[1]
         end

         local host_info_to_cache = check_differences(host, host_name,
                                                      scan_type,
                                                      {
                                                         vulnerabilities = old_data.num_vulnerabilities_found,
                                                         ports = old_data.num_open_ports,
                                                         cve = old_cve_no_score,
                                                         tcp_ports = {num_ports = old_data.tcp_ports, ports = old_data.tcp_ports_list },
                                                         udp_ports = {num_ports = old_data.udp_ports, ports = old_data.udp_ports_list},
                                                         old_epoch = old_data.last_scan.epoch
                                                      },
                                                      {
                                                         vulnerabilities = num_vulnerabilities_found,
                                                         ports = num_open_ports,
                                                         cve = cve,
                                                         tcp_ports = tcp_ports,
                                                         udp_ports = udp_ports,
                                                         last_scan_time = last_scan_time

                                                      })
         if host_info_to_cache then
            host_info_differences = host_info_to_cache

            if debug_me then
               traceError(TRACE_NORMAL,TRACE_CONSOLE, "Vulnerability Scan detected change: enqueueing event to vulnerability_scan check\n")
            end

            ntop.rpushCache(scanned_hosts_changes_queue_key, json.encode(host_info_to_cache))
         end
      end
   end

   if (new_item.is_periodicity) then
      update_scan_info_for_report(vs_utils.scan_in_exec_type.periodic_scan, new_item, host_hash_key, host_info_differences, is_down)
   end

   if (new_item.is_all) then
      update_scan_info_for_report(vs_utils.scan_in_exec_type.scan_all, new_item, host_hash_key, host_info_differences, is_down)
   end

   remove_scanning_host({host=host, scan_type=scan_type, ports=ports})

   return result, new_item.id
end

-- **********************************************************

function vs_utils.update_ts_counters()
   local hosts_details = vs_utils.retrieve_hosts_to_scan()
   local count_cve         = 0
   local hosts_scanned
   local open_ports_count  = 0
   local hosts_count       = 0

   for _,item in ipairs(hosts_details) do

      hosts_count = hosts_count + 1

      if item.num_open_ports ~= nil then
         open_ports_count = open_ports_count + item.num_open_ports
      end
      if item.num_vulnerabilities_found ~= nil then
         count_cve = count_cve + item.num_vulnerabilities_found
      end

   end

   local count = ntop.getCache(scanned_hosts_count_key)
   if (not isEmptyString(count)) then
      hosts_scanned = tonumber(count)
   end

   local response = {
      cve_count = count_cve,
      scanned_hosts = hosts_scanned,
      open_ports = open_ports_count,
      hosts_count = hosts_count
   }

   return response
end

-- **********************************************************

-- Function to format num for emails
-- @param case: 0 - cve, 1 - udp, 2 - tcp
local function format_num_for_email(num, case)
   if (case == 0) then
      -- cve
      if (num == 0) then
         return(i18n("hosts_stats.page_scan_hosts.email.no_cves"))
      else
         local formatted_num = format_high_num_value_for_tables({num = num}, "num")
         return(i18n("hosts_stats.page_scan_hosts.email.num_cves", {num = formatted_num}))
      end
   elseif (case == 1) then
      -- udp
      if (num == 0) then
         return(i18n("hosts_stats.page_scan_hosts.email.no_udp"))
      else
         local formatted_num = format_high_num_value_for_tables({num = num}, "num")
         return(i18n("hosts_stats.page_scan_hosts.email.num_udp", {num = formatted_num}))
      end
   elseif (case == 2) then
      -- tcp
      if (num == 0) then
         return(i18n("hosts_stats.page_scan_hosts.email.no_tcp"))
      else
         local formatted_num = format_high_num_value_for_tables({num = num}, "num")
         return(i18n("hosts_stats.page_scan_hosts.email.num_tcp", {num = formatted_num}))
      end

   elseif (case == 3) then
      -- scanned_hosts

      if (num == 0) then
         return(i18n("hosts_stats.page_scan_hosts.email.no_scanned_hosts"))
      else
         local formatted_num = format_high_num_value_for_tables({num = num}, "num")
         return(i18n("hosts_stats.page_scan_hosts.email.num_scanned_hosts", {num = formatted_num}))
      end
   elseif (case == 4) then
      -- not scanned_hosts  --> hosts unreachable

      if (num == 0) then
         return(i18n("hosts_stats.page_scan_hosts.email.num_failed_scanned_hosts", {num = 0}))
      else
         local formatted_num = format_high_num_value_for_tables({num = num}, "num")
         return(i18n("hosts_stats.page_scan_hosts.email.num_failed_scanned_hosts", {num = formatted_num}))
      end
   end

end

-- **********************************************************

local function retrieve_email_info(exec_type) 
   local info_redis_key = get_counter_periodic_all_scan_keys(exec_type)

   local info_string = ntop.getCache(info_redis_key)

   local info_json = nil
   if (info_string ~= nil) then
      info_json = json.decode(info_string) or {}
   else
      info_json = {
         cves = 0,
         udp_ports = 0,
         tcp_ports = 0
      }
   end

   if(debug_me) then
      tprint(info_json)
   end
   
   local email_info = {
      cve_num = tonumber(info_json.cves) or 0,
      udp_ports = tonumber(info_json.udp_ports) or 0,
      tcp_ports = tonumber(info_json.tcp_ports) or 0,
      scanned_hosts = tonumber(info_json.scanned_hosts) or 0,
      not_scanned_hosts = tonumber(info_json.not_scanned_hosts) or 0,
      begin_epoch_t = tonumber(info_json.begin_epoch),
      end_epoch_t   = os.time(),
      report_type = exec_type,
      -- has_dicrepancy must be true only if there are new open ports or cves fixed
      has_discrepancy = ((info_json.new_open_ports or 0) > 0) or ((info_json.num_cve_solved or 0) > 0),
      down_hosts = info_json.down_hosts
   } 

   if (email_info.has_discrepancy) then
      email_info.new_open_ports = info_json.new_open_ports or 0
      email_info.fixed_cves = info_json.num_cve_solved or 0
      email_info.discrepancies_details = tostring(info_json.hosts_discrepancies_details)
   end
   if (email_info.down_hosts) then
      email_info.down_hosts_string_list = tostring(info_json.down_hosts_string_list)
   end

   email_info.duration = email_info.end_epoch_t - email_info.begin_epoch_t

   ntop.setCache(info_redis_key,json.encode({
      cves = 0,
      udp_ports = 0,
      tcp_ports = 0,
      begin_epoch = 0,
      scanned_hosts = 0,
      not_scanned_hosts = 0
   }))
   return email_info
end

local function retrieve_report_info(date)
   local host_scanned_info = vs_utils.retrieve_hosts_to_scan()

   local info = {
      cves = 0,
      tcp_ports = 0,
      udp_ports = 0,
      scanned_hosts = 0,
      not_scanned_hosts = 0
   }
   for _, item in ipairs(host_scanned_info) do
      if (not isEmptyString(item.num_vulnerabilities_found)) then
         info.cves = info.cves + tonumber(item.num_vulnerabilities_found)
      end

      info.tcp_ports = info.tcp_ports + tonumber(item.tcp_ports)
      info.udp_ports = info.udp_ports + tonumber(item.udp_ports)

      -- plus 1 because start from 0
      info.scanned_hosts = info.scanned_hosts + 1
      info.not_scanned_hosts = info.not_scanned_hosts + 1
   end

   if (date) then
      -- case of periodic scan and scan all
      -- in order to save same date on email and report
      info.date = tonumber(date)
   else
      info.date = tonumber(os.time())
   end

   local info_date_formatted = tostring(formatEpoch(info.date))
   info.name = "Report of "..info_date_formatted

   info.all_data_details = host_scanned_info

   return info
end
-- **********************************************************

-- params date used only in case of periodic or scan all exec
function vs_utils.generate_report(date)

   local report_info = retrieve_report_info(date)
      
   vs_db_utils.save_report_info(report_info)

   return report_info
   
end

local function add_ports_open_for_email_report(l4_key_prefix, host_details)
   if (tonumber(host_details[l4_key_prefix.."_ports"]) > 0) then
      return (i18n("hosts_stats.page_scan_hosts.email.host_details_open_ports", {
         ports_list = host_details[l4_key_prefix.."_ports_list"],
         l4_proto = ternary(l4_key_prefix == "tcp", "TCP", "UDP")
      }))

   end
   return nil
end

local function format_all_hosts_details_info_for_email(all_hosts_details) 
   local formatted_hosts_details_string = ""
   for _,host_details in ipairs(all_hosts_details) do
      -- by default the first element is the scan type always fullfil
      local label_id_scan_type = string.format("hosts_stats.page_scan_hosts.scan_type_list.%s",host_details.scan_type)
      local scan_type_label = i18n(label_id_scan_type)
      local formatted_host_details_string = i18n("hosts_stats.page_scan_hosts.email.host_details_scan_type",{
         scan_type = scan_type_label
      })
      local host_id = get_host_id(host_details)
      
      local tcp_prefix = "tcp"
      local udp_prefix = "udp"

      local tcp_ports_details = add_ports_open_for_email_report(tcp_prefix, host_details)
      if(tcp_ports_details) then
         formatted_host_details_string = formatted_host_details_string .. tcp_ports_details
      end

      local udp_ports_details = add_ports_open_for_email_report(udp_prefix, host_details)
      if(udp_ports_details) then
         formatted_host_details_string = formatted_host_details_string .. udp_ports_details
      end

      if (host_details.num_vulnerabilities_found > 0) then
         local cve_list_string = cve_utils.getFirst5(host_details.cve, host_details.scan_type, false)
         formatted_host_details_string = formatted_host_details_string .. i18n("hosts_stats.page_scan_hosts.email.host_details_cves", {cves_num = host_details.num_vulnerabilities_found, cves_list = cve_list_string})
      end

      local host_details_email_line = i18n("hosts_stats.page_scan_hosts.email.host_details", {
         host_id = host_id,
         details = formatted_host_details_string
      })
      formatted_hosts_details_string = formatted_hosts_details_string .. host_details_email_line

   end


   return formatted_hosts_details_string
end

-- **********************************************************

-- Function to send notification after a periodic scan
-- @param exec_type (2 -> is an all scan message, 3 -> is a periodic scan message)
-- @param periodicity (can be nil in case of scan all)
function vs_utils.notify_scan_results(exec_type, periodicity)
   local notification_message = ""

   local email_info = retrieve_email_info(exec_type) 

   local title = i18n("hosts_stats.page_scan_hosts.email.vulnerability_scan_report_title",{host = getHttpHost()})
   local duration = 0
   local duration_label = ''

   
   if (email_info.duration ~= nil) then
      duration_label = secondsToTime(email_info.duration)
   end

   local start_date_formatted = formatEpoch(email_info.begin_epoch_t)
   local end_date_formatted   = formatEpoch(email_info.end_epoch_t)

   local report_date = tonumber(email_info.end_epoch_t)

   ntop.setCache(hosts_scan_last_report_dates, json.encode({start_date = start_date_formatted, end_date = end_date_formatted}))

   local email_body_i18n_key
   if (periodicity and periodicity == "1day") then
      -- 1 day scan case
      email_body_i18n_key = "hosts_stats.page_scan_hosts.email.periodicity_scan_1_day_ended"
   elseif (periodicity and periodicity == "1week") then
      -- 1 week scan case
      email_body_i18n_key = "hosts_stats.page_scan_hosts.email.periodicity_scan_1_week_ended"
   else
      -- on demand case
      email_body_i18n_key = "hosts_stats.page_scan_hosts.email.scan_all_ended"
   end

   local skipped_hosts_list = ""
   local no_down_hosts_br = ""
   if (email_info.down_hosts) then
      skipped_hosts_list = i18n("hosts_stats.page_scan_hosts.email.host_down_list", {
         host_down_items = email_info.down_hosts_string_list
      })
   else 
      no_down_hosts_br = "</br><br>"
   end

   notification_message = i18n(email_body_i18n_key, {
      cves = format_num_for_email(email_info.cve_num,0),
      udp_ports = format_num_for_email(email_info.udp_ports,1),
      tcp_ports = format_num_for_email(email_info.tcp_ports,2),
      scanned_hosts = format_num_for_email(email_info.scanned_hosts, 3),
      not_scanned_hosts = format_num_for_email(email_info.not_scanned_hosts, 4),
      no_down_hosts_br = no_down_hosts_br,
      skipped_hosts_list = skipped_hosts_list,
      duration = duration_label,
      start_date = start_date_formatted,
      end_date = end_date_formatted,

   })
   local possible_discrepancies_info = ""
   local add_br = ""
   if (email_info.has_discrepancy) then
      possible_discrepancies_info = i18n("hosts_stats.page_scan_hosts.email.discrepancy", {
         new_ports_open = ternary(email_info.new_open_ports ~= 0, format_high_num_value_for_tables({num = email_info.new_open_ports}, "num"),"0"), 
         cves_fixed = ternary(email_info.fixed_cves ~= 0, format_high_num_value_for_tables({num = email_info.fixed_cves }, "num"),"0"), 
         hosts_discrepancy_details = email_info.discrepancies_details
      })
   else 
      add_br = "<br>"
      possible_discrepancies_info = i18n("hosts_stats.page_scan_hosts.email.no_discrepancy")
   end
   
   notification_message = notification_message .. possible_discrepancies_info
   
   local report_link_line = i18n("hosts_stats.page_scan_hosts.email.report_link_line",{url = string.format(getHttpHost() .. ntop.getHttpPrefix() .. "/lua/enterprise/vulnerability_scan_report.lua?epoch_end=%u&epoch_begin=%u",report_date,report_date), add_br = add_br})
   
   notification_message = notification_message .. report_link_line

   vs_utils.generate_report(email_info.end_epoch_t)

   --local hosts_details = format_all_hosts_details_info_for_email(report_info.all_data_details)
   --notification_message = string.format("%s<br><br>%s",notification_message,hosts_details)
   
   if verbose then
      traceError(TRACE_NORMAL,TRACE_CONSOLE, "Vulnerability Scan completed. Sending " .. title .."\n")
   end
   recipients.sendMessageByNotificationType({periodicity = periodicity, success = true, message = notification_message, title = title}, "vulnerability_scans")
   
end

-- **********************************************************

-- Function to retrieve dates of scan all or periodic scan
function vs_utils.get_scan_all_dates()
   return ntop.getCache(hosts_scan_last_report_dates)
end

-- **********************************************************

-- Function to verify if periodic scan is ended
function vs_utils.is_periodic_scan_completed()
   local periodicity_scan_in_progress = ntop.getCache(periodic_scan_key) == "1"

   if (periodicity_scan_in_progress) then
      local hosts_details = vs_utils.retrieve_hosts_to_scan()
      for _,item in ipairs(hosts_details) do

         -- verify status of in progress periodic scanning
         if(item.is_periodicity and (item.is_ok_last_scan == vs_utils.scan_status.scheduled or item.is_ok_last_scan == vs_utils.scan_status.scanning)) then
            return false
         end
      end

      ntop.setCache(periodic_scan_key, "0")

      local periodicity = ntop.getCache(periodic_scan_type_key)

      for _,item in ipairs(hosts_details) do
         local host_hash_key = vs_utils.get_host_hash_key(item.host, item.scan_type)
         local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
         if(not isEmptyString(host_hash_value_string)) then

            local host_hash_value = json.decode(host_hash_value_string)

            host_hash_value.is_periodicity = false

            ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
         end
      end

      return true, periodicity
   end

   return false
end

-- **********************************************************

-- Function to verify if scan all is ended
function vs_utils.is_ondemand_scan_completed()
   local scan_all_in_progress = ntop.getCache(ondemand_scan_key) == "1"

   if (scan_all_in_progress) then
      local hosts_details = vs_utils.retrieve_hosts_to_scan()
      for _,item in ipairs(hosts_details) do

         -- verify status of in progress periodic scanning
         if(item.is_all and (item.is_ok_last_scan == vs_utils.scan_status.scheduled or item.is_ok_last_scan == vs_utils.scan_status.scanning)) then
            return false
         end
      end

      ntop.setCache(ondemand_scan_key, "0")

      for _,item in ipairs(hosts_details) do
         local host_hash_key = vs_utils.get_host_hash_key(item.host, item.scan_type)
         local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
         if(not isEmptyString(host_hash_value_string)) then

            local host_hash_value = json.decode(host_hash_value_string)

            host_hash_value.is_all = false

            ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
         end
      end

      return true
   end

   return false
end

function vs_utils.is_single_scan_completed()
   local single_scan_in_progress = ntop.getCache(single_scan_key) == "1"

   if (single_scan_in_progress) then
      local hosts_details = vs_utils.retrieve_hosts_to_scan()
      for _,item in ipairs(hosts_details) do

         -- verify status of in progress periodic scanning
         if(item.is_single_scan and (item.is_ok_last_scan == vs_utils.scan_status.scheduled or item.is_ok_last_scan == vs_utils.scan_status.scanning)) then
            return false
         end
      end

      ntop.setCache(single_scan_key, "0")

      for _,item in ipairs(hosts_details) do
         local host_hash_key = vs_utils.get_host_hash_key(item.host, item.scan_type)
         local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
         if(not isEmptyString(host_hash_value_string)) then

            local host_hash_value = json.decode(host_hash_value_string)

            host_hash_value.is_single_scan = false

            ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
         end
      end

      return true
   end

   return false

end

-- **********************************************************

-- Function to enable periodic scan end check on callbacks
function vs_utils.is_periodic_scan_running()
   return ntop.getCache(periodic_scan_key) == "1"
end

-- **********************************************************

local function retrieve_host_by_hash_key(host_hash_key) 
   local hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
   local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,host_hash_key)
   local hash_value = json.decode(hash_prefs_string)
   if (not isEmptyString(hash_value_string)) then
      -- hash value found
      hash_value = json.decode(hash_value_string)
      local hash_pref_value = json.decode(hash_prefs_string) or {}
      for k,value in pairs(hash_pref_value) do
         if (k ~= 'is_ok_last_scan') then
            hash_value[k] = value
         end
      end
   end
   return hash_value
end
-- **********************************************************

-- Function to retrieve a specific host scan info
function vs_utils.retrieve_host(host)
   local hosts_scanned = ntop.getHashKeysCache(host_to_scan_key) or {}
   for key, _ in pairs(hosts_scanned) do
      if key:find(host) and key:find("cve") then
         return retrieve_host_by_hash_key(key)
      end
   end
   return nil
end

-- **********************************************************

-- Function to retrieve hosts list to scan
function vs_utils.retrieve_hosts_to_scan(epoch)
   if (isEmptyString(epoch) or (not ntop.isClickHouseEnabled())) then
      -- not a request for historical data (only for report)
      local hash_keys = ntop.getHashKeysCache(prefs_host_values_key)
      local rsp = {}
      if hash_keys then
         for k in pairs(hash_keys) do
            local hash_value_string = ntop.getHashCache(host_to_scan_key, k)
            local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,k)
            local hash_value = json.decode(hash_prefs_string)

            if (not isEmptyString(hash_value_string)) then
               -- hash value found

               hash_value = json.decode(hash_value_string)
               local hash_pref_value = json.decode(hash_prefs_string) or {}

                  for key,value in pairs(hash_pref_value) do
                     if (key ~= 'is_ok_last_scan') then
                        hash_value[key] = value
                     end
                  end
            else
               -- hash value not found
               ntop.setHashCache(host_to_scan_key, k, hash_prefs_string)

            end

            rsp[#rsp+1] = hash_value
         end
      end
      return rsp
   else
      -- request for report
      local all_details, data = vs_db_utils.retrieve_report(epoch)
      return(data)
   end
end

-- **********************************************************

-- Function to retrieve hosts list to scan just for status_info
function vs_utils.check_in_progress_status()
   local hash_keys = ntop.getHashKeysCache(host_to_scan_key)
   local total_in_progress = 0
   local total = 0

   if hash_keys then
      for k in pairs(hash_keys) do
         local hash_value_string = ntop.getHashCache(host_to_scan_key, k)

         if (not isEmptyString(hash_value_string)) then
            local hash_value = json.decode(hash_value_string)
            -- Check IN PROGRESS --> FIX ME with enums
            if hash_value and (hash_value.is_ok_last_scan == vs_utils.scan_status.scheduled or hash_value.is_ok_last_scan == vs_utils.scan_status.scanning) then
               total_in_progress = total_in_progress + 1
            end
            total = total + 1
         end
      end
   end

   return total, total_in_progress
end

-- **********************************************************
-- Function restrieve scan result from FS

local function retrieve_scan_result_from_file(scan_type, host)
   -- for retrocompatibility 
   local path = get_report_path(scan_type, host)

   if(ntop.exists(path)) then
      local handle = io.open(path, "r")
      local result = handle:read("*a")
      handle:close()

      return result
   else
      return("Unable to read file "..path)
   end
   
end

-- **********************************************************
-- Function to retrieve last host scan result
function vs_utils.retrieve_hosts_scan_result(scan_type, host, epoch)
   
   -- retrieve from DB
   local db_result
   local fs_result

   local is_clickhouse_enabled = ntop.isClickHouseEnabled()
   
   -- retrieve data here
   if (is_clickhouse_enabled) then
      -- retrieve from DB
      db_result = vs_db_utils.retrieve_scan_result(scan_type, host, tonumber(epoch))
   else
      -- retrieve from FS
      fs_result = retrieve_scan_result_from_file(scan_type, host)

      return fs_result
   end

   -- return data
   if (is_clickhouse_enabled and db_result == "") then

      -- ****************************************
      -- retrocompatibility case:

      -- when there were results saved on FS, 
      -- but the user updates ntopng to use clickhouse,
      -- and there are currently no results in clickhouse.
      -- ****************************************

      -- clickhouse is enabled but maybe the user had old data on FS and no data on DB
      fs_result = retrieve_scan_result_from_file(scan_type, host)
      return fs_result
   else
      -- the result from DB is ok.
      return db_result
   end

end

-- **********************************************************

-- Function to delete host to scan
function vs_utils.delete_host_to_scan(host, scan_type, all)
   if all then
      ntop.delCache(prefs_host_values_key)
      ntop.delCache(host_to_scan_key)
      ntop.delCache(host_scan_queue_key)
      ntop.delCache(host_in_scanning_hash_key)
      ntop.delCache(periodic_scan_key)
      ntop.delCache(periodic_scan_type_key)

      local path_to_s_result = get_report_path(scan_type, host, true)
      os.execute("rm -f "..path_to_s_result)
   else
      local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
      local path_to_s_result = get_report_path(scan_type, host, false)
      os.remove(path_to_s_result)
      ntop.delHashCache(host_to_scan_key, host_hash_key)
      ntop.delHashCache(prefs_host_values_key, host_hash_key)
      ntop.delHashCache(host_in_scanning_hash_key, host_hash_key)

      -- Remove this host from active schedules
      local elems = {}
      while(true) do
         local e = ntop.lpopCache(host_scan_queue_key)

         if(e == nil) then
            break
         else
            local r = json.decode(e)
            if(not((r.scan_type == "cve") and (r.host == "127.0.0.1"))) then
               table.insert(elems, e)
            end
         end
      end

      for _,i in pairs(elems) do
         ntop.lpushCache(host_scan_queue_key, i)
      end
   end

   return true
end

-- **********************************************************

-- Function to delete host to scan by id
function vs_utils.delete_host_to_scan_by_id(id)
   local hosts_details = vs_utils.retrieve_hosts_to_scan()
   local host_to_delete = {}
   local id_number = tonumber(id)

   for _,value in ipairs(hosts_details) do
      if(tonumber(value.id) == id_number ) then
         host_to_delete.host = value.host
         host_to_delete.scan_type = value.scan_type
         break
      end
   end

   local host_hash_key = vs_utils.get_host_hash_key(host_to_delete.host, host_to_delete.scan_type)
   local path_to_s_result = get_report_path(host_to_delete.scan_type, host_to_delete.host, false)
   os.remove(path_to_s_result)
   ntop.delHashCache(host_to_scan_key, host_hash_key)
   ntop.delHashCache(prefs_host_values_key, host_hash_key)


   return true
end

-- **********************************************************

-- Function to retrieve scan types list
function vs_utils.retrieve_scan_types()
   local scan_types = vs_utils.list_scan_modules()
   local ret = {}

   for _,scan_type in ipairs(scan_types) do
      table.insert(ret, { id = scan_type, label = i18n("hosts_stats.page_scan_hosts.scan_type_list."..scan_type) })
   end

   return ret
end

-- **********************************************************

function vs_utils.list_scan_modules()
   local dirs = ntop.getDirs()
   local basedir = dirs.scriptdir .. "/lua/modules/vulnerability_scan/modules"
   local modules = {}

   for name in pairs(ntop.readdir(basedir)) do
      if(ends(name, ".lua")) then
         name = string.sub(name, 1, string.len(name)-4) -- remove .lua trailer
         local m = vs_utils.load_module(name)

         if(m:is_enabled()) then
            table.insert(modules, name)
         end
      end
   end

   return(modules)
end

-- **********************************************************

function vs_utils.load_module(name)
   package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/modules/?.lua;".. package.path

   return(require(name):new())
end

-- **********************************************************

function vs_utils.discover_open_ports(host)
   local result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports, scan_ports, network_alert_store,now

   local scan_module = vs_utils.load_module("tcp_portscan")
   now,result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports = scan_module:scan_host(host, ports)

   -- FIX ME -> only tcp for now

   return format_port_list_to_string(tcp_ports)
end

-- **********************************************************

-- Function to exec single host scan
function vs_utils.scan_host(scan_type, host, ports, scan_id, use_coroutines)
   if(ntop.isShuttingDown()) then return(false) end

   if(use_coroutines == nil) then use_coroutines = false end

   if debug_me then
      if (ports ~= nil) then
         traceError(TRACE_NORMAL,TRACE_CONSOLE, "Scanning Host ".. host .. " on Ports: " .. ports .. "\n")
      else
         traceError(TRACE_NORMAL,TRACE_CONSOLE, "Scanning Host ".. host.."\n")
      end
   end

   -- to save on redis the user input
   local ports_scan_param = ports

   if(string.contains(scan_type, '_portscan')) then
      -- Nothing to do
   else
      if (isEmptyString(ports)) then
         ports = vs_utils.discover_open_ports(host)
      end
   end

   if(ntop.isShuttingDown()) then return(false) end

   vs_utils.set_status_scan(scan_type, host, ports_scan_param, id, nil,nil,nil, vs_utils.scan_status.scanning)

   -- Save on redis the scanning host to avoid inconsistent state on ntopng restarts
   local scanning_host = {scan_type = scan_type, host = host, ports = ports_scan_param, id = scan_id}
   save_scanning_host(scanning_host)

   -- Scan host
   local scan_module = vs_utils.load_module(scan_type)
   local now,result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports = scan_module:scan_host(host, ports, use_coroutines)

   if(ntop.isShuttingDown()) then
      return false
   end

   if (udp_ports ~= nil) then
      udp_ports = {ports = format_port_list_to_string(udp_ports), num_ports = #udp_ports}
   end

   if(tcp_ports ~= nil) then
      tcp_ports = {ports = format_port_list_to_string(tcp_ports), num_ports = #tcp_ports}
   end

   if scan_result and scan_result ~= vs_utils.scan_status.failed then
      scan_result = vs_utils.scan_status.ok

      ntop.incrCache(scanned_hosts_count_key)
   elseif(scan_result and scan_result == vs_utils.scan_status.failed) then
      scan_result = vs_utils.scan_status.failed
   end

   if debug_me then
      -- traceError(TRACE_NORMAL, TRACE_CONSOLE, "End scan Host ".. host .. ", result: " .. result .. "\n")
   end

   if (isAlreadyPresent({host= host, scan_type= scan_type})) then
      vs_utils.save_host_to_scan(scan_type, host, result, now, duration, scan_result,
                                 ports_scan_param, nil, num_open_ports, num_vulnerabilities_found, cve, scan_id, false, udp_ports, tcp_ports)
   end

   return true
end

-- **********************************************************

local function set_single_scan_in_progress()
   
   if (ntop.getCache(single_scan_key) ~= '1') then 
      ntop.setCache(single_scan_key, "1")
      ntop.setCache(single_scan_info_key, json.encode({
         cves = 0,
         udp_ports = 0,
         tcp_ports = 0,
         begin_epoch = os.time(),
         scanned_host = 1
      }))
   end

end

-- Function to update single host status
function vs_utils.set_status_scan(scan_type, host, ports, id, is_periodicity, is_all,is_single_scan, status)
   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
   local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
   local host_hash_value
   if(not isEmptyString(host_hash_value_string)) then

      host_hash_value = json.decode(host_hash_value_string) or {}
   else
      host_hash_value = {}
   end

   host_hash_value.is_ok_last_scan = status
   if (is_periodicity ~= nil and is_periodicity) then
      host_hash_value.is_periodicity = is_periodicity
   end

   if (is_all ~= nil and is_all) then
      host_hash_value.is_all = is_all
   end

   if (is_single_scan ~= nil and is_single_scan) then
      host_hash_value.is_single_scan = is_single_scan
      set_single_scan_in_progress()
   end

   ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))

   return true
end

-- **********************************************************

function vs_utils.schedule_ondemand_single_host_scan(scan_type, host, ports, scan_id, is_periodicity, is_all, is_single_scan)
   local scan = { scan_type = scan_type, host = host, ports = ports, id= scan_id}

 
   vs_utils.set_status_scan(scan_type, host, ports, scan_id, is_periodicity, is_all, is_single_scan, vs_utils.scan_status.scheduled)

   ntop.rpushCache(host_scan_queue_key, json.encode(scan))

   return true
end

-- **********************************************************

function vs_utils.schedule_ondemand_all_hosts_scan()
   local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()

   local is_scanning_almost_one = false
   if #host_to_scan_list > 0 then
      for _,scan_info in ipairs(host_to_scan_list) do
         vs_utils.schedule_ondemand_single_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, false, true, false)
         is_scanning_almost_one = true
      end
   end

   if (is_scanning_almost_one) then
      ntop.setCache(ondemand_scan_key , "1")
   end

   ntop.setCache(ondemand_scan_host_info_key, json.encode({
                       cves = 0,
                       udp_ports = 0,
                       tcp_ports = 0,
                       begin_epoch = os.time(),
                       scanned_host = 0
   }))

   return true
end

-- **********************************************************

-- periodicity can be set to "1day" "1week" "disabled"
function vs_utils.schedule_periodic_scan(periodicity)
   local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()

   if (#host_to_scan_list > 0 ) then

      local is_already_running = ntop.getCache(periodic_scan_key) == "1"
      if not is_already_running then

         local is_scanning_almost_one = false

         for _,scan_info in ipairs(host_to_scan_list) do
            local frequency = scan_info.scan_frequency
            if(frequency == periodicity) then
               vs_utils.schedule_ondemand_single_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, true, false, false)
               is_scanning_almost_one = true
            end
         end


         if is_scanning_almost_one then
            ntop.setCache(periodic_scan_key , "1")
            ntop.setCache(periodic_scan_type_key, periodicity)


            ntop.setCache(periodic_scan_host_info_key , json.encode({
                                cves = 0,
                                udp_ports = 0,
                                tcp_ports = 0,
                                begin_epoch = os.time(),
                                scanned_hosts = 0,
                                not_scanned_hosts = 0
            }))

         end
      end
   end
   return true
end

-- **********************************************************

-- Process a single host scan request that has been queued
function vs_utils.process_oldest_scheduled_scan(use_coroutines)
   if(ntop.isShuttingDown()) then return(false) end

   local elem = ntop.lpopCache(host_scan_queue_key)

   if((elem ~= nil) and (elem ~= "")) then
      if debug_me then
         traceError(TRACE_NORMAL,TRACE_CONSOLE, "Found vulnerability scan: ".. elem .. "\n")
      end

      local elem = json.decode(elem)

      if(use_coroutines) then
         if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Starting scan on host "..elem.host.."["..elem.scan_type .."]") end
         return(coroutine.create(function () vs_utils.scan_host(elem.scan_type, elem.host, elem.ports, elem.id, use_coroutines) end))
      else
         vs_utils.scan_host(elem.scan_type, elem.host, elem.ports, elem.id, use_coroutines)

         return true
      end
   else
      if(use_coroutines) then
         if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "No host to scan") end
         return nil
      else
         return false
      end
   end
end

-- **********************************************************

-- Process a single host scan request that has been queued
function vs_utils.process_all_scheduled_scans(max_num_scans, use_coroutines)
   local num = 0
   local co = {}

   if(max_num_scans == nil) then max_num_scans = 9999 end

   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Starting up to "..max_num_scans.." scans...") end

   while((max_num_scans > 0) and not(ntop.isShuttingDown())) do
      local res = vs_utils.process_oldest_scheduled_scan(use_coroutines)
      local do_inc = true

      if(use_coroutines) then
         if(res == nil) then
            break -- nothing to do
            do_inc = false
         else
            co[#co + 1] = res
         end
      else
         if(res == false) then
            break
         end
      end

      if(do_inc) then
         max_num_scans = max_num_scans - 1
         num = num + 1
      end
   end

   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Started "..num.." scans") end

   if(use_coroutines and (num > 0)) then
      -- See snmp_poll.lua

      while(not(ntop.isShuttingDown())) do
         local tot = #co
         local keep_on = false

         for i = 1, tot do
            if coroutine.status(co[i]) ~= "dead" then
               local rc, msg = coroutine.resume(co[i])

               -- Note that resume runs in protected mode.
               -- Therefore, if there is any error inside a coroutine, Lua will not show the error message,
               -- but instead will return it to the resume call.
               if not rc then
                  traceError(TRACE_NORMAL, TRACE_CONSOLE, msg or "Unknown error occurred")
               end

               keep_on = rc or keep_on
            end
         end -- for

         if(keep_on == false) then
            break
         end
      end -- while
   end

   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "All "..num.." scans are completed") end

   return num
end

-- **********************************************************

-- Example vs_utils.get_active_hosts("192.168.2.0", "24")
function vs_utils.get_active_hosts(host, cidr)
   local result = {}

   cidr = tonumber(cidr)

   if((cidr == 32) or (cidr == 128)
      or (host:find('.') == nil) -- not dots in IP, it looks symbolic
      or (string.sub(host, -1) ~= "0") -- last digit is not 0, so let's assume /32
   ) then
      result[#result+1] = host -- return it as is
   else
      local s = string.split(host, '%.')
      local net = s[1].."."..s[2].."."..s[3].."."
      local nmap = vs_utils.get_nmap_path()
      local command = nmap..' -sP -n ' .. net .. '1-254 | grep "Nmap scan report for" | cut -d " " -f 5'
      local out = ntop.execCmd(command)
      local l = lines(out)

      for _,h in pairs(l) do
         result[#result+1] = h
      end
   end

   return result
end

-- **********************************************************

-- Update all scan frequencies
function vs_utils.update_all_periodicity(scan_frequency)
   local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()

   for _,value in ipairs(host_to_scan_list) do
      local host_hash_key = vs_utils.get_host_hash_key(value.host, value.scan_type)
      local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
      if(not isEmptyString(host_hash_value_string)) then

         local host_hash_value = json.decode(host_hash_value_string)

         host_hash_value.scan_frequency = scan_frequency

         ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
      end

      local prefs_host_value_string = ntop.getHashCache(prefs_host_values_key, host_hash_key)

      if (not isEmptyString(prefs_host_value_string)) then
         local host_hash_value = json.decode(prefs_host_value_string)

         host_hash_value.scan_frequency = scan_frequency

         ntop.setHashCache(prefs_host_values_key, host_hash_key, json.encode(host_hash_value))
      end
   end

   return true
end

-- **********************************************************

function vs_utils.is_available()
   if (ntop.isnEdge()) then
      return false
   end
   local scan_modules = vs_utils.list_scan_modules()
   return (#scan_modules > 0)
end

-- **********************************************************

function vs_utils.runCommand(scan_command, use_coroutines)
   local result
   local debug_me = false

   if ntop.isWindows() then
      local handle = io.popen(scan_command)
      result = handle:read("*a")
      handle:close()
   else
      if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Started " .. scan_command) end

      if(use_coroutines) then
         if(true) then
            local job_id = ntop.execCmdAsync(scan_command)

            result = nil

            while((result == nil) and not(ntop.isShuttingDown())) do
               coroutine.yield()
               result = ntop.readResultCmdAsync(job_id)
               ntop.msleep(100)
            end

            if(debug_me) then tprint(result) end
         else
            coroutine.yield()
         end
      else
         result = ntop.execCmd(scan_command)
      end
   end

   return result
end

-- **********************************************************

function vs_utils.nmap_scan_host(command, host_ip, ports, use_coroutines, module_name)
   local scan_command

   if(ntop.isShuttingDown()) then
      return nil
   end

   -- IPv6 check
   if(string.contains(host_ip, ':')) then command = command .. " -6 " end

   if(not(isEmptyString(ports))) then command = command .. " -p " .. ports end

   scan_command = string.format("%s %s", command, host_ip)

   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Executing: "..scan_command.."\n") end

   local begin_epoch = os.time()
   local result = vs_utils.runCommand(scan_command, use_coroutines)
   local duration = os.time() - begin_epoch
   local scan_ok = true
   local num_open_ports
   local num_vulnerabilities_found
   local cve
   local tcp_ports
   local udp_ports

   result, num_open_ports, num_vulnerabilities_found, cve, udp_ports, tcp_ports = vs_utils.cleanup_nmap_result(result, module_name)
   return begin_epoch, result, duration, scan_ok, num_open_ports, num_vulnerabilities_found, cve, udp_ports, tcp_ports
end

-- **********************************************************

function vs_utils.nmap_check_host(host_ip, use_coroutines)
   local nmap =  vs_utils.get_nmap_path()
   local scan_command = nmap.." -sn" 

   -- IPv6 check
   if(string.contains(host_ip, ':')) then scan_command = scan_command .. " -6" end
   scan_command = string.format("%s %s",scan_command,host_ip)
   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Executing: "..scan_command.."\n") end

   local start_scan = os.time()
   local result = vs_utils.runCommand(scan_command, use_coroutines)
   local end_scan = os.time()
   local scan_duration = end_scan - start_scan



   local is_up = vs_utils.cleanup_nmap_check_host_result(result)
   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Host is up: "..tostring(is_up).."\n") end

   return is_up, scan_duration, start_scan, end_scan 

end
-- **********************************************************

-- Migrate old configurations
function vs_utils.migrate_keys()
   local old_hash_key = "ntopng.prefs.host_to_scan"
   local old_hosts = ntop.getHashKeysCache(old_hash_key) or {}

   for key,_ in pairs(old_hosts) do
      local hash_value_string = ntop.getHashCache(old_hash_key, key)
      local old_hash_value = json.decode(hash_value_string)
      if (old_hash_value) then
         local new_hash_value = {
            host = old_hash_value.host,
            host_name = old_hash_value.host_name,
            scan_type = old_hash_value.scan_type,
            scan_frequency = old_hash_value.scan_frequency,
            ports = old_hash_value.ports,
         }

         ntop.setHashCache(prefs_host_values_key, key,json.encode(new_hash_value))
      end
   end

   ntop.delCache(old_hash_key)

   local hosts = ntop.getHashKeysCache(host_to_scan_key) or {}
   local from_key = "tcp_openports"
   local to_key   = "tcp_portscan"

   for key, _ in pairs(hosts) do
      if(string.contains(key, from_key)) then
         value = ntop.getHashCache(host_to_scan_key, key)

         new_key   = key:gsub(from_key, to_key)
         new_value = value:gsub(from_key, to_key)

         ntop.setHashCache(host_to_scan_key, new_key, new_value)
         ntop.delHashCache(host_to_scan_key, key)
      end
   end
end

-- **********************************************************

-- init once
if(ntop.getCache("ntopng.prefs.vs.vs_slow_scan") == "1") then
   use_slow_scan = " -T polite --max-parallelism 1"
else
   use_slow_scan = ""
end

-- **********************************************************
function vs_utils.isVSConfiguredHost(ip)

end
-- **********************************************************

function vs_utils.retrieve_report_list(epoch)
   local sort_on = "DATE"
   return (vs_db_utils.retrieve_reports(sort_on,epoch))
end

-- **********************************************************

function vs_utils.retrieve_report(report_name)
   return (vs_db_utils.retrieve_report(report_name))
end

-- **********************************************************

function vs_utils.delete_report(epoch)
   return(vs_db_utils.delete_report(epoch))
end

-- **********************************************************

function vs_utils.edit_report(epoch, report_name)
   return(vs_db_utils.edit_report(epoch,report_name))
end

return vs_utils