ntopng/scripts/lua/modules/lists_utils.lua

--
-- (C) 2019 - ntop.org
--

local lists_utils = {}

local dirs = ntop.getDirs()
local os_utils = require("os_utils")
local categories_utils = require("categories_utils")
local json = require("dkjson")
local alerts_api = require("alerts_api")

-- ##############################################

local CUSTOM_CATEGORY_MINING = 99
local CUSTOM_CATEGORY_MALWARE = 100
local CUSTOM_CATEGORY_ADVERTISEMENT = 101

local DEFAULT_UPDATE_INTERVAL = 86400
local MAX_LIST_ERRORS = 3

local is_nedge = ntop.isnEdge()

-- supported formats: ip, domain, hosts
--
-- Examples:
--    [ip] 1.2.3.4
--    [ip] 1.2.3.0/24
--    [domain] amalwaredomain.com
--    [hosts] 127.0.0.1   amalwaredomain.com
--    [hosts] 127.0.0.1   1.2.3.4
--
local BUILTIN_LISTS = {
   ["ntop IP Malware Meltdown"] = {
      url = "http://blacklists.ntop.org/blacklist-ip.txt",
      category = CUSTOM_CATEGORY_MALWARE,
      format = "ip",
      enabled = false,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["ntop Host Malware Meltdown"] = {
      url = "http://blacklists.ntop.org/blacklist-hostnames.txt",
      category = CUSTOM_CATEGORY_MALWARE,
      format = "host",
      enabled = false,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["Emerging Threats"] = {
      url = "https://rules.emergingthreats.net/fwrules/emerging-Block-IPs.txt",
      category = CUSTOM_CATEGORY_MALWARE,
      format = "ip",
      enabled = true,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["Cisco Talos Intelligence"] = {
      url = "https://talosintelligence.com/documents/ip-blacklist",
      category = CUSTOM_CATEGORY_MALWARE,
      format = "ip",
      enabled = true,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["Ransomware Domain Blocklist"] = {
      url = "https://ransomwaretracker.abuse.ch/downloads/RW_DOMBL.txt",
      category = CUSTOM_CATEGORY_MALWARE,
      format = "domain",
      enabled = true,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["Ransomware IP Blocklist"] = {
      url = "https://ransomwaretracker.abuse.ch/downloads/RW_IPBL.txt",
      category = CUSTOM_CATEGORY_MALWARE,
      format = "ip",
      enabled = false, -- Medium False Positive rate
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["Feodo Tracker Botnet C2 IP Blocklist"] = {
      url = "https://feodotracker.abuse.ch/downloads/ipblocklist.txt",
      category = CUSTOM_CATEGORY_MALWARE,
      format = "ip",
      enabled = true,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["SSLBL Botnet C2 IP Blacklist"] = {
      url = "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt",
      category = CUSTOM_CATEGORY_MALWARE,
      format = "ip",
      enabled = true,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["MalwareDomainList Hosts"] = {
      url = "https://www.malwaredomainlist.com/hostslist/hosts.txt",
      category = CUSTOM_CATEGORY_MALWARE,
      format = "hosts",
      enabled = false,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["Anti-WebMiner"] = {
      url = "https://raw.githubusercontent.com/greatis/Anti-WebMiner/master/hosts",
      category = CUSTOM_CATEGORY_MINING,
      format = "hosts",
      enabled = false,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["NoCoin Filter List"] = {
      url = "https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt",
      category = CUSTOM_CATEGORY_MINING,
      format = "hosts",
      enabled = true,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["Disconnect.me Simple Ad List"] = {
      url = "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt",
      category = CUSTOM_CATEGORY_ADVERTISEMENT,
      format = "domain",
      enabled = is_nedge,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["hpHosts Ad and Tracking"] = {
      url = "https://hosts-file.net/ad_servers.txt",
      category = CUSTOM_CATEGORY_ADVERTISEMENT,
      format = "hosts",
      enabled = is_nedge,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["AdAway default blocklist"] = {
      url = "https://adaway.org/hosts.txt",
      category = CUSTOM_CATEGORY_ADVERTISEMENT,
      format = "hosts",
      enabled = is_nedge,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }, ["SSLBL JA3"] = {
      url = "https://sslbl.abuse.ch/blacklist/ja3_fingerprints.csv",
      format = "ja3_suricata_csv",
      category = CUSTOM_CATEGORY_MALWARE,
      enabled = true,
      update_interval = DEFAULT_UPDATE_INTERVAL,
   }
}

-- ##############################################

-- NOTE: metadata and status are handled as separate keys.
-- Metadata can only be updated by the gui, whereas status can only be
-- updated by housekeeping. This avoid concurrency issues.
local METADATA_KEY = "ntopng.prefs.category_lists.metadata"
local STATUS_KEY = "ntopng.prefs.category_lists.status"


local function loadListsFromRedis()
   local lists_metadata = ntop.getPref(METADATA_KEY)
   local lists_status = ntop.getPref(STATUS_KEY)

   if isEmptyString(lists_status) then
      return {}
   end

   local status = json.decode(lists_status)
   local lists = {}

   if not isEmptyString(lists_metadata) then
      lists = json.decode(lists_metadata)
   end

   lists = table.merge(BUILTIN_LISTS, lists)

   if((lists == nil) or (status == nil)) then
      return {}
   end

   for list_name, list in pairs(lists) do
      if status[list_name] then
	 list.status = status[list_name]
      end
   end

   return lists
end

-- ##############################################

local function saveListsStatusToRedis(lists)
   local status = {}

   for list_name, list in pairs(lists or {}) do
      status[list_name] = list.status
   end

   ntop.setPref(STATUS_KEY, json.encode(status))
end

-- ##############################################

local function saveListsMetadataToRedis(lists)
   local metadata = {}

   for list_name, list in pairs(lists or {}) do
      local default_prefs = BUILTIN_LISTS[list_name]
      local meta = {}
      local has_custom_pref = false

      -- Only save the preferences that differ from the default configuration
      for key, val in pairs(list) do
         if((key ~= "status") and (default_prefs[key] ~= val)) then
            meta[key] = val
            has_custom_pref = true
         end
      end

      if(has_custom_pref) then
         metadata[list_name] = meta
      end
   end

   ntop.setPref(METADATA_KEY, json.encode(metadata))
end

-- ##############################################

function lists_utils.getCategoryLists()
   -- TODO add support for user defined urls
   local lists = {}
   local redis_lists = loadListsFromRedis()

   local default_status = {last_update=0, num_hosts=0, last_error=false, num_errors=0}

   for key, default_values in pairs(BUILTIN_LISTS) do
      local list = table.merge(default_values, redis_lists[key] or {status = {}})
      list.status = table.merge(default_status, list.status)
      lists[key] = list
   end

   return lists
end

-- ##############################################

function lists_utils.editList(list_name, metadata_override)
   local lists = lists_utils.getCategoryLists()
   local list = lists[list_name]

   if not list then
      return false
   end

   list = table.merge(list, metadata_override)
   lists[list_name] = list

   saveListsMetadataToRedis(lists)

   -- Trigger a reload, for example for disabled lists
   lists_utils.downloadLists()
end

-- ##############################################

-- Force a single list reload
function lists_utils.updateList(list_name)
   ntop.setCache("ntopng.cache.category_lists.update." .. list_name, "1")
   lists_utils.downloadLists()
end

-- ##############################################

local function initListCacheDir()
   ntop.mkdir(os_utils.fixPath(string.format("%s/category_lists", dirs.workingdir)))
end

local function getListCacheFile(list_name, downloading)
   local f = string.format("%s/category_lists/%s.txt", dirs.workingdir, list_name)

   if downloading then
      f = string.format("%s.new", f)
   end

   return os_utils.fixPath(f)
end

-- ##############################################

local function getNextListUpdate(list)
   local interval

   if(list.status.last_error and (list.status.num_errors < MAX_LIST_ERRORS)) then
      -- When the download fails, retry next hour
      interval = 3600
   else
      interval = list.update_interval
   end

   local next_update

   -- align if possible
   if interval == 3600 then
      next_update = ntop.roundTime(list.status.last_update, 3600, false)
   elseif interval == 86400 then
      next_update = ntop.roundTime(list.status.last_update, 86400, true --[[ UTC align ]])
   else
      next_update = list.status.last_update + interval
   end

   return next_update
end

-- Returns true if the given list should be updated
function lists_utils.shouldUpdate(list_name, list, now)
   local list_file = getListCacheFile(list_name, false)
   local next_update = getNextListUpdate(list)

   -- note: num_errors is used to avoid retying downloading the same list again when
   -- the file does not exist
   return(list.enabled and
	     ((now >= next_update) or
		   (not ntop.exists(list_file) and (list.status.num_errors < MAX_LIST_ERRORS)) or
		   (ntop.getCache("ntopng.cache.category_lists.update." .. list_name) == "1")))
end

-- ##############################################

-- Check if the lists require an update
-- Returns true after all the lists are processed, false otherwise
local function checkListsUpdate(timeout)
   local lists = lists_utils.getCategoryLists()
   local begin_time = os.time()
   local now = begin_time

   initListCacheDir()

   for list_name, list in pairsByKeys(lists) do
      local list_file = getListCacheFile(list_name, false)

      if lists_utils.shouldUpdate(list_name, list, now) then
	 local temp_fname = getListCacheFile(list_name, true)

	 traceError(TRACE_INFO, TRACE_CONSOLE, string.format("Updating list '%s'...", list_name))
	 local started_at = os.time()
	 local res = ntop.httpFetch(list.url, temp_fname, timeout)

	 if(res and (res["RESPONSE_CODE"] == 200)) then
	    -- download was successful, replace the original file
	    os.rename(temp_fname, list_file)
	    list.status.last_error = false
	    list.status.num_errors = 0
	 else
	    -- failure
	    local respcode = 0
	    local last_error = i18n("delete_data.msg_err_unknown")

	    if res and res["ERROR"] then
	       last_error = res["ERROR"]
	    elseif res and res["RESPONSE_CODE"] ~= nil then
	       respcode = ternary(res["RESPONSE_CODE"], res["RESPONSE_CODE"], "-")

	       if res["IS_PARTIAL"] then
		  last_error = i18n("category_lists.connection_time_out", {duration=(os.time() - started_at)})
	       else
		  last_error = i18n("category_lists.server_returned_error")
	       end

	       if(respcode > 0) then
		  last_error = last_error .. i18n("category_lists.http_code", {err_code = respcode})
	       end
	    end

	    list.status.last_error = last_error
	    list.status.num_errors = list.status.num_errors + 1

	    alerts_api.store(
	       alerts_api.categoryListsEntity(list_name),
	       alerts_api.listDownloadFailedType(list_name, last_error)
	    )
	 end

	 now = os.time()
	 -- set last_update even on failure to avoid blocking on the same list again
	 list.status.last_update = now
	 ntop.delCache("ntopng.cache.category_lists.update." .. list_name)

	 if now-begin_time >= timeout then
	    -- took too long, will resume on next housekeeping execution
	    break
	 end
      end
   end

   -- update lists state
   saveListsStatusToRedis(lists)

   if now-begin_time >= timeout then
      -- Still in progress, do not mark as finished yet
      return false
   else
      return true
   end
end

-- ##############################################

local function loadListItem(host, category, user_custom_categories)
   category = tonumber(category)

   -- Checking for "whitelisted hosts" (Format: !<host>)
   if string.sub(host, 1, 1) == "!" then
      return false
   end

   if category ~= nil then
      --traceError(TRACE_NORMAL, TRACE_CONSOLE, host .. " -> " .. category)

      -- Checking for "whitelisted hosts"
      if user_custom_categories[category] ~= nil then
	 local hosts_map = swapKeysValues(user_custom_categories[category])
	 if hosts_map["!"..host] ~= nil then
	    return false
	 end
      end

      if isIPv4(host) or isIPv4Network(host) then
	 ntop.loadCustomCategoryIp(host, category)
	 return true
      else
	 ntop.loadCustomCategoryHost(host, category)
	 return true
      end
   end

   return false
end

-- ##############################################

local function parse_hosts_line(line)
   local words = string.split(line, "%s+")
   local host = nil

   if(words and (#words == 2)) then
      host = words[2]

      if((host == "localhost") or (host == "127.0.0.1") or (host == "::1")) then
	 host = nil
      end
   else
      -- invalid host
      host = nil
   end

   return(host)
end

-- ##############################################

local function handle_ja3_suricata_csv_line(line)
   local parts = string.split(line, ",")

   if((parts ~= nil) and (#parts >= 1)) then
      local md5_hash = parts[1]

      if(string.len(md5_hash) == 32) then
	 ntop.loadMaliciousJA3Hash(string.lower(md5_hash))
	 return(true)
      end
   end

   return(false)
end

-- ##############################################

-- Loads hosts from a list file on disk
local function loadFromListFile(list_name, list, user_custom_categories)
   local list_fname = getListCacheFile(list_name)
   local num_lines = 0
   local f = io.open(list_fname, "r")

   if f == nil then
      if list.status.num_hosts > 0 then
	 -- avoid generating warnings during first startup
	 traceError(TRACE_WARNING, TRACE_CONSOLE, string.format("Could not find '%s'...", list_fname))
      end

      return 0
   end

   traceError(TRACE_INFO, TRACE_CONSOLE, string.format("Loading list '%s'...", list_fname))

   for line in f:lines() do
      if ntop.isShutdown() then
	 break
      end
      local trimmed = line:match("^%s*(.-)%s*$")

      if((string.len(trimmed) > 0) and not(string.starts(trimmed, "#"))) then
	 local host = trimmed

	 if list.format == "hosts" then
	    host = parse_hosts_line(trimmed)
	 elseif list.format == "ja3_suricata_csv" then
	    -- handled differently
	    if handle_ja3_suricata_csv_line(trimmed) then
	       num_lines = num_lines + 1
	    end
	    host = nil
	 end

	 if host then
	    if loadListItem(host, list.category, user_custom_categories) then
	       num_lines = num_lines + 1
	    end
	 end
      end
   end

   f:close()
   return num_lines
end

-- ##############################################

-- NOTE: this must be executed in the same thread as checkListsUpdate
local function reloadListsNow()
   local user_custom_categories = categories_utils.getAllCustomCategoryHosts()
   local lists = lists_utils.getCategoryLists()
   local start_t = os.time()

   -- Load hosts from cached URL lists
   for list_name, list in pairsByKeys(lists) do
      if list.enabled then
	 local new_hosts = loadFromListFile(list_name, list, user_custom_categories)

	 if new_hosts > 0 then
	    list.status.num_hosts = new_hosts
	 end
      end
   end

   -- update lists state
   saveListsStatusToRedis(lists)

   -- Load user-customized categories
   for category_id, hosts in pairs(user_custom_categories) do
      for _, host in ipairs(hosts) do
	 if ntop.isShutdown() then
	    break
	 end
	 loadListItem(host, category_id, user_custom_categories)
      end
   end

   local end_t = os.time()
   traceError(TRACE_INFO, TRACE_CONSOLE, string.format("Lists loaded in %d seconds", (end_t - start_t)))

   -- Reload into memory
   ntop.reloadCustomCategories()
   ntop.reloadJA3Hashes()
end

-- ##############################################

-- This avoids waiting for lists reload
function lists_utils.reloadLists()
   ntop.setCache("ntopng.cache.reload_lists_utils", "1")
end

-- This is necessary to avoid concurrency issues
function lists_utils.downloadLists()
   ntop.setCache("ntopng.cache.download_lists_utils", "1")
end

-- ##############################################

-- This is run in housekeeping.lua
function lists_utils.checkReloadLists()
   local reload_now = (ntop.getCache("ntopng.cache.reload_lists_utils") == "1")

   if ntop.getCache("ntopng.cache.download_lists_utils") == "1" then
      if checkListsUpdate(60 --[[ timeout ]]) then
	 ntop.delCache("ntopng.cache.download_lists_utils")
	 -- lists where possibly updated, reload
	 reload_now = true
      end
   end

   if reload_now then
      reloadListsNow()
      ntop.delCache("ntopng.cache.reload_lists_utils")
   end

   -- Possibly reload hosts blacklist status
   ntop.checkReloadHostBlacklist()
end

-- ##############################################

return lists_utils