--
-- (C) 2013-23 - ntop.org
--

--
-- This file implements some utility functions used by the REST API
-- in the vulnerability pages
--
--
-- https://geekflare.com/nmap-vulnerability-scan/
-- cd /usr/share/nmap/scripts/
-- git clone https://github.com/scipag/vulscan.git
-- ln -s `pwd`/scipag_vulscan /usr/share/nmap/scripts/vulscan
-- cd vulscan/utilities/updater/
-- chmod +x updateFiles.sh
-- ./updateFiles.sh
--
-- Example:
-- nmap -sV --script vulscan --script-args vulscandb=openvas.csv <target> -p 80,233
--
--
-- exploitdb.csv
-- osvdb.csv
-- securitytracker.csv
-- openvas.csv
-- scipvuldb.csv
-- xforce.csv
-- securityfocus.csv
-- cve.csv
--

-- **********************************************************

local dirs = ntop.getDirs()
package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path
package.path = dirs.installdir .. "/scripts/lua/pro/modules/?.lua;" .. package.path
package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/?.lua;" .. package.path
package.path = dirs.installdir .. "/scripts/lua/modules/recipients/?.lua;" .. package.path

require "lua_utils" -- used by tprint (debug)

local host_to_scan_key                    = "ntopng.vs.hosts.scanned_values"
local prefs_host_values_key               = "ntopng.prefs.vs.hosts_conf"

local periodic_scan_key                   = "ntopng.vs.periodic_scan"
local periodic_scan_type_key              = "ntopng.vs.periodic_scan_type"
local ondemand_scan_key                   = "ntopng.vs.scan_all"
local scanned_hosts_count_key             = "ntopng.prefs.host_to_scan.count_scanned"
local scanned_hosts_changes_key           = "ntopng.alerts.scanned_hosts_changes"
local host_in_scanning_hash_key           = "ntopng.vs.hosts.in_scanning"
local host_scan_queue_key                 = "ntopng.vs.scan_queue"

-- redis key for last scan report dates
local hosts_scan_last_report_dates        = "ntopng.vs.report_dates"

-- redis keys for periodic scan info
local periodic_scan_host_info_key         = "ntopng.vs.periodic_scan.info"

-- redis keys for scan all info
local ondemand_scan_host_info_key         = "ntopng.vs.scan_all.info"

local json = require("dkjson")
local format_utils = require("format_utils")
local recipients = require("recipients")
local cve_utils = require("cve_utils")

-- Enable debug with:
-- redis-cli set "ntopng.prefs.vs.debug_enabled" "1"
-- systemctl restart ntopng
local debug_me = ntop.getCache("ntopng.prefs.vs.debug_enabled") == "1"
local verbose = false

local vs_utils = {}

local use_slow_scan;

-- **********************************************************

function vs_utils.get_host_hash_key(host, scan_type)
   return string.format("%s-%s", host, scan_type)
end

-- **********************************************************

vs_utils.scan_status = {
   error       = 0,
   ok          = 1,
   scheduled   = 2,
   not_scanned = 3,
   scanning    = 4
}

vs_utils.ports_diff_case = {
   no_diff          = 2, -- case 1 or 2 (combined)
   ntopng_more_t_vs = 3,
   vs_more_t_ntopng = 4
}

-- **********************************************************

function vs_utils.get_nmap_path()
   local path = {
      "/usr/bin/nmap",
      "/usr/local/bin/nmap",
      "/opt/homebrew/bin/nmap"
   }

   for _,p in pairs(path) do
      if(ntop.exists(p)) then
	 return(p..use_slow_scan)
      end
   end

   return(nil)
end

-- **********************************************************

function vs_utils.is_nmap_installed()
   local module_path = {
      "/usr/share/nmap/scripts/",
      "/opt/homebrew/share/nmap/scripts/vulscan/",
      "/usr/local/share/nmap/scripts/vulscan",
   }
   local path = vs_utils.get_nmap_path()

   if(path ~= nil) then
      for _,m in pairs(module_path) do
	 if(ntop.exists(m)) then
	    return true
	 end
      end
   end

   return false
end

-- **********************************************************

local function get_report_path(scan_type, ip, all)
   local base_dir
   local ret = ""

   base_dir = dirs.workingdir .. "/" .. getSystemInterfaceId() .. "/vulnerability_scan"
   ntop.mkdir(base_dir)

   if (not all or all == nil) then
      ret = base_dir .. "/"..ip.."_"..scan_type..".txt"
   else
      ret = base_dir .. "/*.txt"
   end

   return(ret)
end

-- ##############################################

local function lines(str)
   local result = {}

   for line in str:gmatch '[^\n]+' do
      table.insert(result, line)
   end
   return result
end


-- ##############################################

local function format_port_list_to_string(ports)
   local scan_ports = ""

   if (ports ~= nil and #ports > 0) then
      for index,port in ipairs(ports) do
         if (index == 1) then
            scan_ports = ""..port
         else
            scan_ports = scan_ports .. ","..port
         end
      end
   end

   return scan_ports
end

-- ##############################################

local function find_port(port, port_list)
   local found = false

   for _,item in ipairs(port_list) do
      if (item == port) then
         found = true
         break
      end
   end
   return found
end

-- ##############################################

local function check_ports_diffences(num_old_ports, old_ports, num_new_ports, new_ports)
   local rsp = {
      trigger = true
   }
   if (num_old_ports == 0 and num_new_ports ~= 0) then
      rsp.open_ports = new_ports
      rsp.open_ports_num = num_new_ports
      rsp.closed_ports_num = 0
      rsp.case = 'new_ports'
   elseif(num_old_ports ~= 0 and num_new_ports == 0) then
      rsp.open_ports_num = 0
      rsp.closed_ports_num = num_old_ports
      rsp.closed_ports = old_ports
      rsp.case = 'ports_closed'
   elseif(num_old_ports ~= 0 and num_new_ports ~= 0) then
      local closed_ports = {}
      local open_ports = {}

      local diff = false
      for _, item in ipairs(old_ports) do
         local is_open = find_port(item, new_ports)
         if (not is_open) then
            closed_ports[#closed_ports+1] = item
            diff = true
         end
      end

      for _, item in ipairs(new_ports) do
         local is_open = find_port(item, old_ports)
         if (not is_open) then
            open_ports[#open_ports+1] = item
            diff = true
         end
      end


      if((not diff) and (num_old_ports == num_new_ports)) then
         rsp.trigger = false
      else
         rsp.open_ports = open_ports
         rsp.open_ports_num = #open_ports
         rsp.closed_ports = closed_ports
         rsp.closed_ports_num = #closed_ports

         if (#open_ports ~= 0 and #closed_ports == 0) then
            rsp.case = 'new_ports'
         elseif (#open_ports == 0 and #closed_ports ~= 0) then
            rsp.case = 'ports_closed'
         else
            rsp.case = 'ports_open_and_closed'
         end
      end
   else
      rsp.trigger = false
   end
   return rsp
end

-- ##############################################

local function split_port_list(data, is_tcp)

   if (is_tcp) then
      if (data.tcp_ports and data.tcp_ports.num_ports ~= 0) then
         return split(data.tcp_ports.ports,",")
      end
   else
      if(data.udp_ports and data.udp_ports.num_ports ~= 0) then
         return split(data.udp_ports.ports, ",")
      end
   end

   return {}
end

-- ##############################################

local function analyze_ports_diff(ports_difference)
   local rsp = {}
   if (ports_difference.trigger) then
      if (debug_me) then
         tprint("found ports differences")
         tprint(ports_difference)
      end
      rsp["open_ports"] = {
         num = ports_difference.open_ports_num,
         ports = format_port_list_to_string(ports_difference.open_ports)
      }
      rsp["closed_ports"] =  {
         num = ports_difference.closed_ports_num,
         ports = format_port_list_to_string(ports_difference.closed_ports)
      }
      rsp["ports_case"] = ports_difference.case

      if (debug_me) then
         tprint(ports_difference.case)
      end
   elseif (debug_me) then
      tprint("IS IT TRIGGERED: ")
      tprint(ports_difference.trigger)
   end

   rsp["triggered"] = ports_difference.trigger

   return rsp
end

-- ##############################################

-- This function checks the differences between an old and a new host scan
-- and return a table containing those differences
local function check_differences(host, host_name, scan_type, old_data, new_data)
   local rsp = {}

   -- security checks
   if host == nil or scan_type == nil then
      return nil
   end

   if tonumber(old_data.ports or 0) ~= tonumber(new_data.ports or 0) then
      rsp["num_ports"] = {
         old_num_ports = old_data.ports or 0,
         new_num_ports = new_data.ports or 0
      }
   end

   local num_cve_solved = 0
   local num_new_cve_issues = 0
   local cve_solved = {}
   local new_cve = {}

   -- Checking the solved vulnerabilities
   for _, cve in ipairs(old_data.cve or {}) do
      -- the old cves have the score
      cve = split(cve,"|")[1]
      -- If the new table does not contains the cve it means that it is solved
      if not (table.contains(new_data.cve or {}, cve)) then
         num_cve_solved = num_cve_solved + 1
         -- Add at most 5 cve
         if num_cve_solved <= 5 then
            cve_solved[#cve_solved + 1] = cve
         end
      end
   end

   -- Checking the new vulnerabilities
   for _, cve in ipairs(new_data.cve or {}) do
      -- If the new table does not contains the cve it means that it is solved
      if not (table.contains(old_data.cve or {}, cve)) then
         num_new_cve_issues = num_new_cve_issues + 1
         -- Add at most 5 cve
         if num_new_cve_issues <= 5 then
            new_cve[#new_cve + 1] = cve
         end
      end
   end

   -- Checking old_open_tcp_ports and new_open_tcp_ports
   local tcp_old_ports = {}
   local udp_old_ports = {}

   local tcp_new_ports = {}
   local udp_new_ports = {}

   if (scan_type == "tcp_portscan") then
      tcp_old_ports = split_port_list(old_data, true)
      tcp_new_ports = split_port_list(new_data, true)
      if (debug_me) then
         tprint("TCP OLD PORTS: ")
         tprint(tcp_old_ports)
         tprint("TCP NEW PORTS: ")
         tprint(tcp_new_ports)
      end

      local tcp_ports_differences = check_ports_diffences(  #tcp_old_ports, tcp_old_ports,
                                                            #tcp_new_ports, tcp_new_ports)
      local rsp_tcp_diff = analyze_ports_diff(tcp_ports_differences)
      if (rsp_tcp_diff.triggered) then
         rsp["tcp_open_ports"] = rsp_tcp_diff.open_ports
         rsp["tcp_closed_ports"] = rsp_tcp_diff.closed_ports
         rsp["tcp_ports_case"] = rsp_tcp_diff.ports_case

      end
      rsp["triggered"] = rsp_tcp_diff.triggered

      rsp["measurement"] = "ports_changes_detected"
   elseif (scan_type == "udp_portscan") then
      udp_old_ports = split_port_list(old_data, false)
      udp_new_ports = split_port_list(new_data, false)

      if (debug_me) then
         tprint("UDP OLD PORTS: ")
         tprint(udp_old_ports)
         tprint("UDP NEW PORTS")
         tprint(udp_new_ports)
      end

      local udp_ports_differences = check_ports_diffences(  #udp_old_ports, udp_old_ports,
                                                            #udp_new_ports, udp_new_ports)
      local rsp_udp_diff = analyze_ports_diff(udp_ports_differences)
      if (rsp_udp_diff.triggered) then
         rsp["udp_open_ports"] = rsp_udp_diff.open_ports
         rsp["udp_closed_ports"] = rsp_udp_diff.closed_ports
         rsp["udp_ports_case"] = rsp_udp_diff.ports_case
      end
      rsp["triggered"] = rsp_udp_diff.triggered

      rsp["measurement"] = "ports_changes_detected"

   end

   if num_cve_solved > 0 then
      rsp["num_cve_solved"] = num_cve_solved
      rsp["cve_solved"] = cve_solved
      rsp["measurement"] = "cve_changes_detected"
   end

   if num_new_cve_issues > 0 then
      rsp["num_new_cve_issues"] = num_new_cve_issues
      rsp["new_cve"] = new_cve
      rsp["measurement"] = "cve_changes_detected"
   end


   if table.empty(rsp) or rsp.triggered == false  then
      rsp = nil
   else
      rsp["host"] = host
      rsp["host_name"] = host_name
      rsp["scan_type"] = scan_type
   end

   return rsp
end

-- ##############################################

function vs_utils.cleanup_port(is_tcp, line)
   local splitted_line = {}

   local regex =  "([^/udp]+)"

   if (is_tcp) then
      regex =  "([^/tcp]+)"
   end
   for str in string.gmatch(line, regex) do
      table.insert(splitted_line, str)
   end

   return splitted_line[1]

end

-- Remove the first/last few lines that contain nmap information that change at each scan
function vs_utils.cleanup_nmap_result(scan_result, scan_type)
   if(scan_result ~= nil) then
      scan_result = scan_result:gsub("|", "")
      scan_result = scan_result:gsub("_", "")

      scan_result = lines(scan_result)

      for i=1,4 do
	 table.remove(scan_result, 1)
      end

      table.remove(scan_result, #scan_result)

      local num_open_ports = 0
      local num_vulnerabilities = 0
      local cve = {}
      local scan_out = {}
      local tcp_ports = {}
      local udp_ports = {}

      for _,l in pairs(scan_result) do
	 -- Ignore "open|filtered" ports
	 if((string.find(l, "open") ~= nil) and (string.find(l, "filtered") == nil)) then
	    local t = string.find(l, "/tcp ") or 0
	    local u = string.find(l, "/udp ") or 0

	    if (t > 0) then
	       num_open_ports = num_open_ports + 1
	       tcp_ports[#tcp_ports+1] = vs_utils.cleanup_port(true, l)
	    end

	    if(u > 0) then
	       num_open_ports = num_open_ports + 1
	       udp_ports[#udp_ports+1] = vs_utils.cleanup_port(false, l)
	    end
	 end

	 -- Escape XML/HTML code that might be present in the output
	 l = l:gsub("<", "&lt;")
	 l = l:gsub(">", "&gt;")

	 if(string.sub(l, 1, 2) == " [") then
	    local c = string.split(string.sub(l,3), "]")
	    local url = cve_utils.getDocURL(c[1], scan_type)

	    if(scan_type == "cve") then
		  l = '[<A HREF="'..url..'">'..c[1]..'</A>]'..c[2]
	    elseif(scan_type == "openvas") then
	       l = '[<A HREF="'..url..'">'..c[1]..'</A>]'..c[2]
	    end

	    table.insert(cve, c[1])
	    num_vulnerabilities = num_vulnerabilities + 1
	 end

	 table.insert(scan_out, l)
      end

      scan_result = table.concat(scan_out, "\n")
      return scan_result, num_open_ports, num_vulnerabilities, cve, udp_ports, tcp_ports
   else
      return "", 0, 0, 0, 0, 0
   end
end

-- **********************************************************

-- Remove the first/last few lines that contain nmap information that change at each scan
function vs_utils.cleanup_nmap_vulners_result(scan_result, scan_type)
   scan_result = scan_result:gsub("|_", "")
   scan_result = scan_result:gsub("|", "")

   scan_result = lines(scan_result)

   for i=1,4 do
      table.remove(scan_result, 1)
   end

   table.remove(scan_result, #scan_result)

   local num_open_ports = 0
   local num_vulnerabilities = 0
   local cve = {}
   local scan_out = {}

   for _,l in pairs(scan_result) do
      if(string.find(l, "open") ~= nil) then
	 local t = string.find(l, "/tcp ") or 0
	 local u = string.find(l, "/udp ") or 0

	 if((t > 0) or (u > 0)) then
	    num_open_ports = num_open_ports + 1
	 end
      end

      if(string.find(l, "https://vulners.com/") ~= nil) then
	 local c = string.split(l, "\t")
	 table.insert(cve, c[2])
	 num_vulnerabilities = num_vulnerabilities + 1
      end

      table.insert(scan_out, l)
   end

   scan_result = table.concat(scan_out, "\n")

   return scan_result, num_open_ports, num_vulnerabilities, cve
end

-- **********************************************************

-- Function to save host configuration
local function isAlreadyPresent(item)
   local hosts_details = vs_utils.retrieve_hosts_to_scan()

   for _,value in ipairs(hosts_details) do
      if (item.host == value.host and item.scan_type == value.scan_type ) then
         return true
      end
   end
   return false
end

-- **********************************************************

local function compare(a,b)

   local a_array = split(a,"|")
   local a_has_score = false
   if (#a_array > 1) then
      a_has_score = true
      a = a_array[2]
   end

   local b_array = split(b,"|")
   local b_has_score = false
   if (#b_array > 1) then
      b_has_score = true
      b = b_array[2]
   end

   if (a_has_score and b_has_score) then
      return a > b
   else
      return a_array[1] > b_array[1]
   end
end

-- **********************************************************

-- Function to format cve list with scores
local function get_cve_with_score(cve)
   local cve_with_score_list = {}
   local max_score = 0
   if(cve ~= nil) then
      for _,cve_id in ipairs(cve) do
	 local score = cve_utils.getCVEscore(cve_id)

	 local cve_formatted = cve
	 if(score ~= nil) then

	    if(max_score < score) then
	       max_score = score
	    end
	    cve_formatted = string.format("%s|%s",cve_id,score)
	 end
	 cve_with_score_list[#cve_with_score_list+1] = cve_formatted
      end
   end

   if next(cve_with_score_list) then
      table.sort(cve_with_score_list, compare)
   end

   return cve_with_score_list, max_score
end

-- **********************************************************

-- Function to remove scanning host
local function remove_scanning_host(host_info)
   local host_to_scan_hash_key = vs_utils.get_host_hash_key(host_info.host, host_info.scan_type)
   ntop.delHashCache(host_in_scanning_hash_key,host_to_scan_hash_key)
end

-- **********************************************************

-- Function to set the actual scanning host on a redis key
local function save_scanning_host(scan_info)
   local host_to_scan_hash_key = vs_utils.get_host_hash_key(scan_info.host, scan_info.scan_type)
   ntop.setHashCache(host_in_scanning_hash_key, host_to_scan_hash_key, json.encode(scan_info))
end

-- **********************************************************

-- Function to select correctly redis keys on periodic or scan all
local function get_counter_periodic_all_scan_keys(is_periodic)
   if (is_periodic) then
      return periodic_scan_host_info_key --host_periodic_scan_cve_num_key,host_periodic_scan_udp_ports_key,host_periodic_scan_tcp_ports_key
   else
      return ondemand_scan_host_info_key --host_scan_all_cve_num_key,host_scan_all_udp_ports_key,host_scan_all_tcp_ports_key
   end
end

-- **********************************************************

-- Function to update counters of periodically scan or scan all
-- @param is_periodic (true -> is a periodic scan, false -> is a scan all)
local function update_periodicity_or_all_scan_info(is_periodic, new_item)
   -- select correctly redis keys
   local redis_info_key = get_counter_periodic_all_scan_keys(is_periodic)

   local info_string = ntop.getCache(redis_info_key)
   local info_json = nil
   if (info_string ~= nil) then
      info_json = json.decode(info_string)
   end


   if (info_json == nil) then
      info_json = {}
   end

   if (new_item.num_vulnerabilities_found ~= nil) then

      if (info_json ~= {} and info_json.cves ~= nil) then
         info_json.cves = tonumber(info_json.cves) + new_item.num_vulnerabilities_found
      else
         info_json.cves = 0
      end
   end

   if (new_item.udp_ports ~= nil) then

      if (info_json ~= {} and info_json.udp_ports ~= nil) then
         info_json.udp_ports = tonumber(info_json.udp_ports) + new_item.udp_ports
      else
         info_json.udp_ports = 0
      end

   end

   if (new_item.tcp_ports ~= nil) then
      if (info_json ~= {} and info_json.tcp_ports ~= nil) then
         info_json.tcp_ports = tonumber(info_json.tcp_ports) + new_item.tcp_ports
      else
         info_json.tcp_ports = 0
      end
   end

   if (info_json ~= {} and info_json.scanned_hosts ~= nil) then
      info_json.scanned_hosts = tonumber(info_json.scanned_hosts) + 1
   else
      info_json.scanned_hosts = 1
   end

   if (info_json ~= {} and info_json.begin_epoch == nil) then
      info_json.begin_epoch = os.time()
   end

   ntop.setCache(redis_info_key, json.encode(info_json))
end

-- **********************************************************

-- Function to restore scanning host
function vs_utils.restore_host_to_scan()
   local hash_keys = ntop.getHashKeysCache(host_in_scanning_hash_key)

   if hash_keys then
      for k in pairs(hash_keys) do
         local hash_value_string = ntop.getHashCache(host_in_scanning_hash_key, k)

         if (not isEmptyString(hash_value_string)) then
            local host_info_to_restore = json.decode(hash_value_string)

            if (host_info_to_restore) then
               -- enqueue to scan
               ntop.lpushCache(host_scan_queue_key, hash_value_string)

               -- set status to scheduled
               vs_utils.set_status_scan(  host_info_to_restore.scan_type, host_info_to_restore.host, host_info_to_restore.ports,
                                          host_info_to_restore.id, host_info_to_restore.is_periodicity, host_info_to_restore.is_all,
                                          vs_utils.scan_status.scheduled)
            end
         end
      end
   end
end

-- **********************************************************

-- Function to restore backup config
function vs_utils.restore_config_backup(vs_backup)
   -- remove old hash entries
   ntop.delCache(host_to_scan_key)
   ntop.delCache(prefs_host_values_key)

   for _,item in ipairs(vs_backup) do
      -- restoring hash entries with status not scanned
      local host_hash_key = vs_utils.get_host_hash_key(item.host, item.scan_type)

      local item_to_restore = item
      ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(item_to_restore))
      ntop.setHashCache(prefs_host_values_key, host_hash_key, json.encode(item_to_restore))

   end
end

-- *********************************************************

-- Function to retrieve hosts keys for backups
function vs_utils.retrieve_hosts_backup()
   local hash_keys = ntop.getHashKeysCache(prefs_host_values_key)
   local rsp = {}
   if hash_keys then
      for k in pairs(hash_keys) do
         local hash_value_string = ntop.getHashCache(prefs_host_values_key, k)
         if (not isEmptyString(hash_value_string)) then
            local hash_value = json.decode(hash_value_string)
            rsp[#rsp+1] = hash_value
         end
      end
   end

   return rsp
end

-- **********************************************************

--Function to save
function vs_utils.add_host_pref(scan_type, host, ports, scan_frequency)

   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)

   local new_item = {
      host = host,
      host_name = host_name,
      scan_type = scan_type,
      ports = ports,
   }
   if not isEmptyString(scan_frequency) then
      new_item.scan_frequency = scan_frequency
   end

   local result = 1 -- success
   if (debug_me) then
      tprint("SAVING HOST: "..new_item.host)
   end

   --saved_hosts[#saved_hosts+1] = new_item
   ntop.setHashCache(prefs_host_values_key, host_hash_key, json.encode(new_item))

   return result
end

function vs_utils.edit_host_pref(scan_type, host, ports, scan_frequency)
   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
   local old_item_string = ntop.getHashCache(prefs_host_values_key,host_hash_key)
   if (not isEmptyString(old_item_string)) then
      local old_item = json.decode(old_item_string)
      old_item.ports = ports
      old_item.scan_frequency = scan_frequency

      ntop.setHashCache(prefs_host_values_key, host_hash_key, json.encode(old_item))
      return 1 --ok
   end
   return 2 -- not found
end

-- **********************************************************

-- Function to update host scan values
function vs_utils.save_host_to_scan(scan_type, host, scan_result, last_scan_time, last_duration,
				    is_ok_last_scan, ports, scan_frequency, num_open_ports,
				    num_vulnerabilities_found, cve, id, is_edit, udp_ports, tcp_ports)
   local checks = require "checks"
   local trigger_alert = checks.isCheckEnabled("active_monitoring", "vulnerability_scan")
      or checks.isCheckEnabled("system", "vulnerability_scan")
   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
   local old_data_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
   local old_data = json.decode(old_data_string)
   -- Getting the hostname, the only way is to scan all the interfaces and retrieve it
   local host_name = ntop.resolveName(host)
   if host_name == host then
      host_name = ""
   end
   -- In case the alert needs to be triggered, save the differences in order to lessen
   -- the info dropped on redis
   -- if is_ok_last_scan is nil then no prior scan was done, so do not trigger the alert
   if trigger_alert and old_data and (not is_edit)  then
      local already_scanned = (old_data.last_scan and old_data.last_scan.epoch)

      if already_scanned then

	 if(debug_me) then
	    tprint("ALREADY PRESENT-> CHECKING DIFF")
	 end

	 local old_cve_no_score = {}
	 for _,cve in ipairs(old_data.cve) do
	    old_cve_no_score[#old_cve_no_score+1] = split(cve,"|")[1]
	 end

	 local host_info_to_cache = check_differences(host, host_name,
						      scan_type,
						      {
							 vulnerabilities = old_data.num_vulnerabilities_found,
							 ports = old_data.num_open_ports,
							 cve = old_cve_no_score,
							 tcp_ports = {num_ports = old_data.tcp_ports, ports = old_data.tcp_ports_list },
							 udp_ports = {num_ports = old_data.udp_ports, ports = old_data.udp_ports_list}
						      },
						      {
							 vulnerabilities = num_vulnerabilities_found,
							 ports = num_open_ports,
							 cve = cve,
							 tcp_ports = tcp_ports,
							 udp_ports = udp_ports
	 })
         if host_info_to_cache then
            ntop.rpushCache(scanned_hosts_changes_key, json.encode(host_info_to_cache))
         end
      end
   end

   local epoch_id = 0
   if isEmptyString(id) then
      local key = "ntopng.prefs.last_host_id"
      local res = ntop.incrCache(key)
      epoch_id = res
   else
      epoch_id = id
   end

   if (isEmptyString(is_ok_last_scan)) then
      is_ok_last_scan = vs_utils.scan_status.not_scanned
   end

   local cve_formatted, max_score_cve = get_cve_with_score(cve)

   local new_item = {
      num_open_ports = num_open_ports,
      num_vulnerabilities_found = num_vulnerabilities_found,
      cve = cve_formatted,
      max_score_cve = max_score_cve,
      is_ok_last_scan = is_ok_last_scan,
      host_name = host_name
   }

   if tcp_ports ~= nil then
      new_item.tcp_ports = tcp_ports.num_ports
      new_item.tcp_ports_list = tcp_ports.ports
   end

   if udp_ports ~= nil then
      new_item.udp_ports = udp_ports.num_ports
      new_item.udp_ports_list = udp_ports.ports
   end

   if (udp_ports == nil and tcp_ports == nil) then
      new_item.tcp_ports = num_open_ports
   end

   if last_scan_time or last_duration then
      --local time_formatted = format_utils.formatEpoch(last_scan_time)
      if (last_duration == nil) or (last_duration <= 0) then
         last_duration = 1
      end

      last_duration = secondsToTime(last_duration)
      new_item.last_scan = {
	 epoch = last_scan_time,
	 --time  = time_formatted,
	 duration = last_duration
      }

      if is_ok_last_scan == vs_utils.scan_status.ok then
	 new_item.is_ok_last_scan = vs_utils.scan_status.ok
      end
   end

   if not isEmptyString(scan_frequency) then
      new_item.scan_frequency = scan_frequency
   elseif old_data and not isEmptyString(old_data.scan_frequency) then
      new_item.scan_frequency = old_data.scan_frequency
   end

   -- the is_periodicity param and is_all param are set outside the save_host_to_scan into the set_status method
   if (old_data and old_data.is_periodicity ~= nil) then
      new_item.is_periodicity = old_data.is_periodicity
   end

   if (old_data and old_data.is_all ~= nil) then
      new_item.is_all = old_data.is_all
   end

   if(scan_result ~= nil) then
      local handle = io.open(get_report_path(scan_type, host), "w")
      local result = handle:write(scan_result)
      handle:close()
   end

   local result = 1 -- success

   if (debug_me) then
      tprint("UPDATING HOST: "..host)
   end

   -- edit case
   ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(new_item))

   local counts = vs_utils.update_ts_counters()

   if (new_item.is_periodicity) then
      update_periodicity_or_all_scan_info(true, new_item)
   end

   if (new_item.is_all) then
      update_periodicity_or_all_scan_info(false, new_item)
   end

   remove_scanning_host({host=host, scan_type=scan_type, ports=ports})

   return result, new_item.id
end

-- **********************************************************

function vs_utils.update_ts_counters()
   local hosts_details = vs_utils.retrieve_hosts_to_scan()
   local count_cve         = 0
   local hosts_scanned
   local open_ports_count  = 0
   local hosts_count       = 0

   for _,item in ipairs(hosts_details) do

      hosts_count = hosts_count + 1

      if item.num_open_ports ~= nil then
         open_ports_count = open_ports_count + item.num_open_ports
      end
      if item.num_vulnerabilities_found ~= nil then
         count_cve = count_cve + item.num_vulnerabilities_found
      end

   end

   local count = ntop.getCache(scanned_hosts_count_key)
   if (not isEmptyString(count)) then
      hosts_scanned = tonumber(count)
   end

   local response = {
      cve_count = count_cve,
      scanned_hosts = hosts_scanned,
      open_ports = open_ports_count,
      hosts_count = hosts_count
   }

   return response
end

-- **********************************************************

-- Function to format num for emails
-- @param case: 0 - cve, 1 - udp, 2 - tcp
local function format_num_for_email(num, case)
   if (case == 0) then
      -- cve
      if (num == 0) then
         return(i18n("hosts_stats.page_scan_hosts.email.no_cves"))
      else
         local formatted_num = format_high_num_value_for_tables({num = num}, "num")
         return(i18n("hosts_stats.page_scan_hosts.email.num_cves", {num = formatted_num}))
      end
   elseif (case == 1) then
      -- udp
      if (num == 0) then
         return(i18n("hosts_stats.page_scan_hosts.email.no_udp"))
      else
         local formatted_num = format_high_num_value_for_tables({num = num}, "num")
         return(i18n("hosts_stats.page_scan_hosts.email.num_udp", {num = formatted_num}))
      end
   elseif (case == 2) then
      -- tcp
      if (num == 0) then
         return(i18n("hosts_stats.page_scan_hosts.email.no_tcp"))
      else
         local formatted_num = format_high_num_value_for_tables({num = num}, "num")
         return(i18n("hosts_stats.page_scan_hosts.email.num_tcp", {num = formatted_num}))
      end

   elseif (case == 3) then
      -- scanned_hosts

      if (num == 0) then
         return(i18n("hosts_stats.page_scan_hosts.email.no_scanned_hosts"))
      else
         local formatted_num = format_high_num_value_for_tables({num = num}, "num")
         return(i18n("hosts_stats.page_scan_hosts.email.num_scanned_hosts", {num = formatted_num}))
      end
   end
end

-- **********************************************************

-- Function to send notification after a periodic scan
-- @param is_periodic (true -> is a periodic scan message, false -> is an all scan message)
-- @param periodicity (can be nil in case of scan all)
function vs_utils.notify_scan_results(is_periodic, periodicity)
   local notification_message = ""

   local info_redis_key = get_counter_periodic_all_scan_keys(is_periodic)

   local info_string = ntop.getCache(info_redis_key)

   local info_json = nil
   if (info_string ~= nil) then
      info_json = json.decode(info_string) or {}
   else
      info_json = {
         cves = 0,
         udp_ports = 0,
         tcp_ports = 0
      }
   end
   local cve_num = tonumber(info_json.cves) or 0
   local udp_ports = tonumber(info_json.udp_ports) or 0
   local tcp_ports = tonumber(info_json.tcp_ports) or 0
   local scanned_hosts = tonumber(info_json.scanned_hosts) or 0

   local title = i18n("hosts_stats.page_scan_hosts.email.vulnerability_scan_report_title",{host = getHttpHost()})
   local duration = 0
   local duration_label = ''

   local begin_epoch_t = tonumber(info_json.begin_epoch)
   local end_epoch_t   = os.time()

   if (info_json.begin_epoch ~= nil) then
      duration = end_epoch_t - begin_epoch_t
      duration_label = secondsToTime(duration)
   end

   local start_date_formatted = formatEpoch(begin_epoch_t)
   local end_date_formatted   = formatEpoch(end_epoch_t)
   ntop.setCache(hosts_scan_last_report_dates, json.encode({start_date = start_date_formatted, end_date = end_date_formatted}))

   if (periodicity and periodicity == "1day") then
      notification_message = i18n("hosts_stats.page_scan_hosts.email.periodicity_scan_1_day_ended", {
				     cves = format_num_for_email(cve_num,0),
				     udp_ports = format_num_for_email(udp_ports,1),
				     tcp_ports = format_num_for_email(tcp_ports,2),
                 scanned_hosts = format_num_for_email(scanned_hosts, 3),
				     url = getHttpHost() .. ntop.getHttpPrefix() .. "/lua/pro/reportng.lua?report_template=vs_result",
				     duration = duration_label,
                 start_date = start_date_formatted,
                 end_date = end_date_formatted
      })
   elseif (periodicity and periodicity == "1week") then
      notification_message = i18n("hosts_stats.page_scan_hosts.email.periodicity_scan_1_week_ended", {
				     cves = format_num_for_email(cve_num,0),
				     udp_ports = format_num_for_email(udp_ports,1),
				     tcp_ports = format_num_for_email(tcp_ports,2),
                 scanned_hosts = format_num_for_email(scanned_hosts, 3),
				     url = getHttpHost() .. ntop.getHttpPrefix() .. "/lua/pro/reportng.lua?report_template=vs_result",
				     duration = duration_label,
                 start_date = start_date_formatted,
                 end_date = end_date_formatted
      })
   else
      -- on demand scan
      notification_message = i18n("hosts_stats.page_scan_hosts.email.scan_all_ended", {
				     cves = format_num_for_email(cve_num,0),
				     udp_ports = format_num_for_email(udp_ports,1),
				     tcp_ports = format_num_for_email(tcp_ports,2),
                 scanned_hosts = format_num_for_email(scanned_hosts, 3),
				     url = getHttpHost() .. ntop.getHttpPrefix() .. "/lua/pro/reportng.lua?report_template=vs_result",
				     duration = duration_label,
                 start_date = start_date_formatted,
                 end_date = end_date_formatted,

      })
   end

   if verbose then
      traceError(TRACE_NORMAL,TRACE_CONSOLE, "Vulnerability Scan completed. Sending " .. title .."\n")
   end

   recipients.sendMessageByNotificationType({periodicity = periodicity, success = true, message = notification_message, title = title}, "vulnerability_scans")

   ntop.setCache(info_redis_key,json.encode({
		       cves = 0,
		       udp_ports = 0,
		       tcp_ports = 0,
		       begin_epoch = 0,
		       scanned_hosts = 0
   }))
end

-- **********************************************************

-- Function to retrieve dates of scan all or periodic scan
function vs_utils.get_scan_all_dates()
   return ntop.getCache(hosts_scan_last_report_dates)
end

-- **********************************************************

-- Function to verify if periodic scan is ended
function vs_utils.is_periodic_scan_completed()
   local periodicity_scan_in_progress = ntop.getCache(periodic_scan_key) == "1"

   if (periodicity_scan_in_progress) then
      local hosts_details = vs_utils.retrieve_hosts_to_scan()
      for _,item in ipairs(hosts_details) do

         -- verify status of in progress periodic scanning
         if(item.is_periodicity and (item.is_ok_last_scan == vs_utils.scan_status.scheduled or item.is_ok_last_scan == vs_utils.scan_status.scanning)) then
            return false
         end
      end

      ntop.setCache(periodic_scan_key, "0")

      local periodicity = ntop.getCache(periodic_scan_type_key)

      for _,item in ipairs(hosts_details) do
         local host_hash_key = vs_utils.get_host_hash_key(item.host, item.scan_type)
         local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
         if(not isEmptyString(host_hash_value_string)) then

            local host_hash_value = json.decode(host_hash_value_string)

            host_hash_value.is_periodicity = false

            ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
         end
      end

      return true, periodicity
   end

   return false
end

-- **********************************************************

-- Function to verify if scan all is ended
function vs_utils.is_ondemand_scan_completed()
   local scan_all_in_progress = ntop.getCache(ondemand_scan_key) == "1"

   if (scan_all_in_progress) then
      local hosts_details = vs_utils.retrieve_hosts_to_scan()
      for _,item in ipairs(hosts_details) do

         -- verify status of in progress periodic scanning
         if(item.is_all and (item.is_ok_last_scan == vs_utils.scan_status.scheduled or item.is_ok_last_scan == vs_utils.scan_status.scanning)) then
            return false
         end
      end

      ntop.setCache(ondemand_scan_key, "0")

      for _,item in ipairs(hosts_details) do
         local host_hash_key = vs_utils.get_host_hash_key(item.host, item.scan_type)
         local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
         if(not isEmptyString(host_hash_value_string)) then

            local host_hash_value = json.decode(host_hash_value_string)

            host_hash_value.is_all = false

            ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
         end
      end

      return true
   end

   return false
end

-- **********************************************************

-- Function to enable periodic scan end check on callbacks
function vs_utils.is_periodic_scan_running()
   return ntop.getCache(periodic_scan_key) == "1"
end

-- **********************************************************

-- Function to retrieve a specific host scan info
function vs_utils.retrieve_host(host)
   local hosts_scanned = ntop.getHashKeysCache(host_to_scan_key) or {}
   for key, _ in pairs(hosts_scanned) do
      if key:find(host) and key:find("cve") then
         local hash_value_string = ntop.getHashCache(host_to_scan_key, key)
         local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,key)
         local hash_value = json.decode(hash_prefs_string)
         if (not isEmptyString(hash_value_string)) then
            -- hash value found
            hash_value = json.decode(hash_value_string)
            local hash_pref_value = json.decode(hash_prefs_string) or {}
            for k,value in pairs(hash_pref_value) do
               if (k ~= 'is_ok_last_scan') then
                  hash_value[k] = value
               end
            end

         end
         return hash_value
      end
   end
   return nil
end

-- **********************************************************

-- Function to retrieve hosts list to scan
function vs_utils.retrieve_hosts_to_scan()
   local hash_keys = ntop.getHashKeysCache(prefs_host_values_key)
   local rsp = {}
   if hash_keys then
      for k in pairs(hash_keys) do
         local hash_value_string = ntop.getHashCache(host_to_scan_key, k)
         local hash_prefs_string = ntop.getHashCache(prefs_host_values_key,k)
         local hash_value = json.decode(hash_prefs_string)

         if (not isEmptyString(hash_value_string)) then
            -- hash value found

            hash_value = json.decode(hash_value_string)
            local hash_pref_value = json.decode(hash_prefs_string) or {}

               for key,value in pairs(hash_pref_value) do
                  if (key ~= 'is_ok_last_scan') then
                     hash_value[key] = value
                  end
               end
         else
            -- hash value not found
            ntop.setHashCache(host_to_scan_key, k, hash_prefs_string)

         end

         rsp[#rsp+1] = hash_value
      end
   end

   return rsp
end

-- **********************************************************

-- Function to retrieve hosts list to scan just for status_info
function vs_utils.check_in_progress_status()
   local hash_keys = ntop.getHashKeysCache(host_to_scan_key)
   local total_in_progress = 0
   local total = 0

   if hash_keys then
      for k in pairs(hash_keys) do
         local hash_value_string = ntop.getHashCache(host_to_scan_key, k)

         if (not isEmptyString(hash_value_string)) then
            local hash_value = json.decode(hash_value_string)
            -- Check IN PROGRESS --> FIX ME with enums
            if hash_value and (hash_value.is_ok_last_scan == vs_utils.scan_status.scheduled or hash_value.is_ok_last_scan == vs_utils.scan_status.scanning) then
               total_in_progress = total_in_progress + 1
            end
            total = total + 1
         end
      end
   end

   return total, total_in_progress
end

-- **********************************************************

-- Function to retrieve last host scan result
function vs_utils.retrieve_hosts_scan_result(scan_type, host)
   local path = get_report_path(scan_type, host)

   if(ntop.exists(path)) then
      local handle = io.open(path, "r")
      local result = handle:read("*a")
      handle:close()

      return result
   else
      return("Unable to read file "..path)
   end
end

-- **********************************************************

-- Function to delete host to scan
function vs_utils.delete_host_to_scan(host, scan_type, all)
   if all then
      ntop.delCache(prefs_host_values_key)
      ntop.delCache(host_to_scan_key)
      ntop.delCache(host_scan_queue_key)
      ntop.delCache(host_in_scanning_hash_key)
      ntop.delCache(periodic_scan_key)
      ntop.delCache(periodic_scan_type_key)

      local path_to_s_result = get_report_path(scan_type, host, true)
      os.execute("rm -f "..path_to_s_result)
   else
      local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
      local path_to_s_result = get_report_path(scan_type, host, false)
      os.remove(path_to_s_result)
      ntop.delHashCache(host_to_scan_key, host_hash_key)
      ntop.delHashCache(prefs_host_values_key, host_hash_key)

      -- Remove this host from active schedules
      local elems = {}
      while(true) do
	 local e = ntop.lpopCache(host_scan_queue_key)

	 if(e == nil) then
	    break
	 else
	    local r = json.decode(e)
	    if(not((r.scan_type == "cve") and (r.host == "127.0.0.1"))) then
	       table.insert(elems, e)
	    end
	 end
      end

      for _,i in pairs(elems) do
	 ntop.lpushCache(host_scan_queue_key, i)
      end
   end

   return true
end

-- **********************************************************

-- Function to delete host to scan by id
function vs_utils.delete_host_to_scan_by_id(id)
   local hosts_details = vs_utils.retrieve_hosts_to_scan()
   local host_to_delete = {}
   local id_number = tonumber(id)

   for _,value in ipairs(hosts_details) do
      if(tonumber(value.id) == id_number ) then
         host_to_delete.host = value.host
         host_to_delete.scan_type = value.scan_type
         break
      end
   end

   local host_hash_key = vs_utils.get_host_hash_key(host_to_delete.host, host_to_delete.scan_type)
   local path_to_s_result = get_report_path(host_to_delete.scan_type, host_to_delete.host, false)
   os.remove(path_to_s_result)
   ntop.delHashCache(host_to_scan_key, host_hash_key)
   ntop.delHashCache(prefs_host_values_key, host_hash_key)


   return true
end

-- **********************************************************

-- Function to retrieve scan types list
function vs_utils.retrieve_scan_types()
   local scan_types = vs_utils.list_scan_modules()
   local ret = {}

   for _,scan_type in ipairs(scan_types) do
      table.insert(ret, { id = scan_type, label = i18n("hosts_stats.page_scan_hosts.scan_type_list."..scan_type) })
   end

   return ret
end

-- **********************************************************

function vs_utils.list_scan_modules()
   local dirs = ntop.getDirs()
   local basedir = dirs.scriptdir .. "/lua/modules/vulnerability_scan/modules"
   local modules = {}

   for name in pairs(ntop.readdir(basedir)) do
      if(ends(name, ".lua")) then
	 name = string.sub(name, 1, string.len(name)-4) -- remove .lua trailer
	 local m = vs_utils.load_module(name)

	 if(m:is_enabled()) then
	    table.insert(modules, name)
	 end
      end
   end

   return(modules)
end

-- **********************************************************

function vs_utils.load_module(name)
   package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/modules/?.lua;".. package.path

   return(require(name):new())
end

-- **********************************************************

function vs_utils.discover_open_ports(host)
   local result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports, scan_ports, network_alert_store,now

   local scan_module = vs_utils.load_module("tcp_portscan")
   now,result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports = scan_module:scan_host(host, ports)

   -- FIX ME -> only tcp for now

   return format_port_list_to_string(tcp_ports)
end

-- **********************************************************

-- Function to exec single host scan
function vs_utils.scan_host(scan_type, host, ports, scan_id, use_coroutines)
   if(ntop.isShuttingDown()) then return(false) end

   if(use_coroutines == nil) then use_coroutines = false end

   if debug_me then
      if (ports ~= nil) then
         traceError(TRACE_NORMAL,TRACE_CONSOLE, "Scanning Host ".. host .. " on Ports: " .. ports .. "\n")
      else
         traceError(TRACE_NORMAL,TRACE_CONSOLE, "Scanning Host ".. host.."\n")
      end
   end

   -- to save on redis the user input
   local ports_scan_param = ports

   if(string.contains(scan_type, '_portscan')) then
      -- Nothing to do
   else
      if (isEmptyString(ports)) then
	 ports = vs_utils.discover_open_ports(host)
      end
   end

   if(ntop.isShuttingDown()) then return(false) end

   vs_utils.set_status_scan(scan_type, host, ports_scan_param, id, nil,nil, vs_utils.scan_status.scanning)

   -- Save on redis the scanning host to avoid inconsistent state on ntopng restarts
   local scanning_host = {scan_type = scan_type, host = host, ports = ports_scan_param, id = scan_id}
   save_scanning_host(scanning_host)

   -- Scan host
   local scan_module = vs_utils.load_module(scan_type)
   local now,result,duration,scan_result,num_open_ports,num_vulnerabilities_found, cve, udp_ports, tcp_ports = scan_module:scan_host(host, ports, use_coroutines)

   if(ntop.isShuttingDown()) then
      return false
   end

   if (udp_ports ~= nil) then
      udp_ports = {ports = format_port_list_to_string(udp_ports), num_ports = #udp_ports}
   end

   if(tcp_ports ~= nil) then
      tcp_ports = {ports = format_port_list_to_string(tcp_ports), num_ports = #tcp_ports}
   end

   if scan_result then
      scan_result = vs_utils.scan_status.ok

      ntop.incrCache(scanned_hosts_count_key)
   end

   if debug_me then
      traceError(TRACE_NORMAL, TRACE_CONSOLE, "End scan Host ".. host .. ", result: " .. result .. "\n")
   end

   if (isAlreadyPresent({host= host, scan_type= scan_type})) then
      vs_utils.save_host_to_scan(scan_type, host, result, now, duration, scan_result,
				 ports_scan_param, nil, num_open_ports, num_vulnerabilities_found, cve, scan_id, false, udp_ports, tcp_ports)
   end

   return true
end

-- **********************************************************

-- Function to update single host status
function vs_utils.set_status_scan(scan_type, host, ports, id, is_periodicity, is_all, status)
   local host_hash_key = vs_utils.get_host_hash_key(host, scan_type)
   local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
   local host_hash_value
   if(not isEmptyString(host_hash_value_string)) then

      host_hash_value = json.decode(host_hash_value_string) or {}
   else
      host_hash_value = {}
   end

   host_hash_value.is_ok_last_scan = status
   if (is_periodicity ~= nil) then
      host_hash_value.is_periodicity = is_periodicity
   end

   if (is_all ~= nil) then
      host_hash_value.is_all = is_all
   end

   ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))

   return true
end

-- **********************************************************

function vs_utils.schedule_ondemand_single_host_scan(scan_type, host, ports, scan_id, is_periodicity, is_all)
   local scan = { scan_type = scan_type, host = host, ports = ports, id= scan_id}
   vs_utils.set_status_scan(scan_type, host, ports, scan_id, is_periodicity, is_all, vs_utils.scan_status.scheduled)

   ntop.rpushCache(host_scan_queue_key, json.encode(scan))

   return true
end

-- **********************************************************

function vs_utils.schedule_ondemand_all_hosts_scan()
   local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()

   local is_scanning_almost_one = false
   if #host_to_scan_list > 0 then
      for _,scan_info in ipairs(host_to_scan_list) do
	 vs_utils.schedule_ondemand_single_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, false, true)
	 is_scanning_almost_one = true
      end
   end

   if (is_scanning_almost_one) then
      ntop.setCache(ondemand_scan_key , "1")
   end

   ntop.setCache(ondemand_scan_host_info_key, json.encode({
		       cves = 0,
		       udp_ports = 0,
		       tcp_ports = 0,
		       begin_epoch = os.time(),
		       scanned_host = 0
   }))

   return true
end

-- **********************************************************

-- periodicity can be set to "1day" "1week" "disabled"
function vs_utils.schedule_periodic_scan(periodicity)
   local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()

   if (#host_to_scan_list > 0 ) then

      local is_already_running = ntop.getCache(periodic_scan_key) == "1"
      if not is_already_running then

         local is_scanning_almost_one = false

	 for _,scan_info in ipairs(host_to_scan_list) do
	    local frequency = scan_info.scan_frequency
	    if(frequency == periodicity) then
	       vs_utils.schedule_ondemand_single_host_scan(scan_info.scan_type, scan_info.host, scan_info.ports, scan_info.id, true, false)
	       is_scanning_almost_one = true
	    end
	 end


         if is_scanning_almost_one then
            ntop.setCache(periodic_scan_key , "1")
            ntop.setCache(periodic_scan_type_key, periodicity)


            ntop.setCache(periodic_scan_host_info_key , json.encode({
				cves = 0,
				udp_ports = 0,
				tcp_ports = 0,
				begin_epoch = os.time(),
            scanned_hosts = 0
            }))

         end
      end
   end
   return true
end

-- **********************************************************

-- Process a single host scan request that has been queued
function vs_utils.process_oldest_scheduled_scan(use_coroutines)
   if(ntop.isShuttingDown()) then return(false) end

   local elem = ntop.lpopCache(host_scan_queue_key)

   if((elem ~= nil) and (elem ~= "")) then
      if debug_me then
         traceError(TRACE_NORMAL,TRACE_CONSOLE, "Found vulnerability scan: ".. elem .. "\n")
      end

      local elem = json.decode(elem)

      if(use_coroutines) then
	 if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Starting scan on host "..elem.host.."["..elem.scan_type .."]") end
	 return(coroutine.create(function () vs_utils.scan_host(elem.scan_type, elem.host, elem.ports, elem.id, use_coroutines) end))
      else
	 vs_utils.scan_host(elem.scan_type, elem.host, elem.ports, elem.id, use_coroutines)

	 return true
      end
   else
      if(use_coroutines) then
	 if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "No host to scan") end
	 return nil
      else
	 return false
      end
   end
end

-- **********************************************************

-- Process a single host scan request that has been queued
function vs_utils.process_all_scheduled_scans(max_num_scans, use_coroutines)
   local num = 0
   local co = {}

   if(max_num_scans == nil) then max_num_scans = 9999 end

   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Starting up to "..max_num_scans.." scans...") end

   while((max_num_scans > 0) and not(ntop.isShuttingDown())) do
      local res = vs_utils.process_oldest_scheduled_scan(use_coroutines)
      local do_inc = true

      if(use_coroutines) then
	 if(res == nil) then
	    break -- nothing to do
	    do_inc = false
	 else
	    co[#co + 1] = res
	 end
      else
	 if(res == false) then
	    break
	 end
      end

      if(do_inc) then
	 max_num_scans = max_num_scans - 1
	 num = num + 1
      end
   end

   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Started "..num.." scans") end

   if(use_coroutines and (num > 0)) then
      -- See snmp_poll.lua

      while(not(ntop.isShuttingDown())) do
	 local tot = #co
	 local keep_on = false

	 for i = 1, tot do
	    if coroutine.status(co[i]) ~= "dead" then
	       local rc, msg = coroutine.resume(co[i])

	       -- Note that resume runs in protected mode.
	       -- Therefore, if there is any error inside a coroutine, Lua will not show the error message,
	       -- but instead will return it to the resume call.
	       if not rc then
		  traceError(TRACE_NORMAL, TRACE_CONSOLE, msg or "Unknown error occurred")
	       end

	       keep_on = rc or keep_on
	    end
	 end -- for

	 if(keep_on == false) then
	    break
	 end
      end -- while
   end

   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "All "..num.." scans are completed") end

   return num
end

-- **********************************************************

-- Example vs_utils.get_active_hosts("192.168.2.0", "24")
function vs_utils.get_active_hosts(host, cidr)
   local result = {}

   cidr = tonumber(cidr)

   if((cidr == 32) or (cidr == 128)
      or (host:find('.') == nil) -- not dots in IP, it looks symbolic
      or (string.sub(host, -1) ~= "0") -- last digit is not 0, so let's assume /32
   ) then
      result[#result+1] = host -- return it as is
   else
      local s = string.split(host, '%.')
      local net = s[1].."."..s[2].."."..s[3].."."
      local nmap = vs_utils.get_nmap_path()
      local command = nmap..' -sP -n ' .. net .. '1-254 | grep "Nmap scan report for" | cut -d " " -f 5'
      local out = ntop.execCmd(command)
      local l = lines(out)

      for _,h in pairs(l) do
	 result[#result+1] = h
      end
   end

   return result
end

-- **********************************************************

-- Update all scan frequencies
function vs_utils.update_all_periodicity(scan_frequency)
   local host_to_scan_list = vs_utils.retrieve_hosts_to_scan()

   for _,value in ipairs(host_to_scan_list) do
      local host_hash_key = vs_utils.get_host_hash_key(value.host, value.scan_type)
      local host_hash_value_string = ntop.getHashCache(host_to_scan_key, host_hash_key)
      if(not isEmptyString(host_hash_value_string)) then

         local host_hash_value = json.decode(host_hash_value_string)

         host_hash_value.scan_frequency = scan_frequency

         ntop.setHashCache(host_to_scan_key, host_hash_key, json.encode(host_hash_value))
      end

      local prefs_host_value_string = ntop.getHashCache(prefs_host_values_key, host_hash_key)

      if (not isEmptyString(prefs_host_value_string)) then
         local host_hash_value = json.decode(prefs_host_value_string)

         host_hash_value.scan_frequency = scan_frequency

         ntop.setHashCache(prefs_host_values_key, host_hash_key, json.encode(host_hash_value))
      end
   end

   return true
end

-- **********************************************************

function vs_utils.is_available()
   local scan_modules = vs_utils.list_scan_modules()
   return ((#scan_modules > 0) and (not ntop.isnEdge()))
end

-- **********************************************************

function vs_utils.format_port_label(port, service_name, protocol)
   if (isEmptyString(service_name)) then
      return string.format("%s/%s",port,protocol)
   else
      return string.format("%s/%s (%s)",port,protocol,service_name)
   end
end

-- **********************************************************
-- Retrieves detected ports by ntopng

function vs_utils.retrieve_detected_ports(host)
   local host_info = interface.getHostInfo(host)

   local tcp_ports_detected = {}
   local udp_ports_detected = {}
   local host_in_mem = false
   if (host_info and host_info.used_ports and host_info.used_ports.local_server_ports) then
      for port, l7_proto in pairs(host_info.used_ports.local_server_ports) do
         local port_details = split(port, ":")
         local id_port = port_details[2]
         local l4_proto = port_details[1]

         if (l4_proto == 'tcp') then
            tcp_ports_detected[#tcp_ports_detected+1] = id_port
         end

         if (l4_proto == 'udp') then
            udp_ports_detected[#udp_ports_detected+1] = id_port
         end
      end
      host_in_mem = true
   end

   return tcp_ports_detected, host_in_mem, udp_ports_detected
end

-- **********************************************************

-- Search port in ports list
local function find_port(port_to_find, port_list)
   for _, port in ipairs(port_list) do
      if(port_to_find == tonumber(port)) then
         return true
      end
   end
   return false
end

-- **********************************************************

-- Compare vs ports and ntopng detected ports
function vs_utils.compare_ports(vs_scan_port_string_list, ntopng_ports)
   local vs_scan_ports = split(vs_scan_port_string_list, ",")

   local ports_unused = {}
   local filtered_ports = {}
   -- check vs_scan_ports with ntopng_ports
   local not_found_a_port = false
   for _,vs_port in ipairs(vs_scan_ports) do
      local find_actual_port = find_port(tonumber(vs_port), ntopng_ports)

      if (not find_actual_port) then
         not_found_a_port = true
         ports_unused[#ports_unused+1] = vs_port
      end

   end

   local diff_case

   if (not_found_a_port) then
      diff_case = vs_utils.ports_diff_case.vs_more_t_ntopng
   end

   if (#vs_scan_ports == #ntopng_ports) then
      diff_case = vs_utils.ports_diff_case.no_diff
   else
      local filtered = false

      for _,ntop_port in ipairs(ntopng_ports) do
         if (not find_port(tonumber(ntop_port), vs_scan_ports)) then
            filtered = true
            filtered_ports[#filtered_ports+1] = ntop_port
         end

      end
      if (filtered) then
         diff_case = vs_utils.ports_diff_case.ntopng_more_t_vs
      end
   end

   return ports_unused, filtered_ports, diff_case
end

-- **********************************************************

function vs_utils.runCommand(scan_command, use_coroutines)
   local result
   local debug_me = false

   if ntop.isWindows() then
      local handle = io.popen(scan_command)
      result = handle:read("*a")
      handle:close()
   else
      if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Started " .. scan_command) end

      if(use_coroutines) then
	 if(true) then
	    local job_id = ntop.execCmdAsync(scan_command)

	    result = nil

	    while((result == nil) and not(ntop.isShuttingDown())) do
	       coroutine.yield()
	       result = ntop.readResultCmdAsync(job_id)
	       ntop.msleep(100)
	    end

	    if(debug_me) then tprint(result) end
	 else
	    coroutine.yield()
	 end
      else
	 result = ntop.execCmd(scan_command)
      end
   end

   return result
end

-- **********************************************************

function vs_utils.nmap_scan_host(command, host_ip, ports, use_coroutines, module_name)
   local scan_command

   if(ntop.isShuttingDown()) then
      return nil
   end

   -- IPv6 check
   if(string.contains(host_ip, ':')) then command = command .. " -6 " end

   if(not(isEmptyString(ports))) then command = command .. " -p " .. ports end

   scan_command = string.format("%s %s", command, host_ip)

   if(debug_me) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Executing: "..scan_command.."\n") end

   local begin_epoch = os.time()
   local result = vs_utils.runCommand(scan_command, use_coroutines)
   local duration = os.time() - begin_epoch
   local scan_ok = true
   local num_open_ports
   local num_vulnerabilities_found
   local cve
   local tcp_ports
   local udp_ports

   result, num_open_ports, num_vulnerabilities_found, cve, udp_ports, tcp_ports = vs_utils.cleanup_nmap_result(result, module_name)
   return begin_epoch, result, duration, scan_ok, num_open_ports, num_vulnerabilities_found, cve, udp_ports, tcp_ports
end

-- **********************************************************

-- Migrate old configurations
function vs_utils.migrate_keys()
   local old_hash_key = "ntopng.prefs.host_to_scan"
   local old_hosts = ntop.getHashKeysCache(old_hash_key) or {}

   for key,_ in pairs(old_hosts) do
      local hash_value_string = ntop.getHashCache(old_hash_key, key)
      local old_hash_value = json.decode(hash_value_string)
      if (old_hash_value) then
         local new_hash_value = {
            host = old_hash_value.host,
            host_name = old_hash_value.host_name,
            scan_type = old_hash_value.scan_type,
            scan_frequency = old_hash_value.scan_frequency,
            ports = old_hash_value.ports,
         }

         ntop.setHashCache(prefs_host_values_key, key,json.encode(new_hash_value))
      end
   end

   ntop.delCache(old_hash_key)

   local hosts = ntop.getHashKeysCache(host_to_scan_key) or {}
   local from_key = "tcp_openports"
   local to_key   = "tcp_portscan"

   for key, _ in pairs(hosts) do
      if(string.contains(key, from_key)) then
	 value = ntop.getHashCache(host_to_scan_key, key)

	 new_key   = key:gsub(from_key, to_key)
	 new_value = value:gsub(from_key, to_key)

	 ntop.setHashCache(host_to_scan_key, new_key, new_value)
	 ntop.delHashCache(host_to_scan_key, key)
      end
   end
end

-- **********************************************************

-- init once
if(ntop.getCache("ntopng.prefs.vs.vs_slow_scan") == "1") then
   use_slow_scan = " -T polite --max-parallelism 1"
else
   use_slow_scan = ""
end

-- **********************************************************

return vs_utils