-- -- (C) 2013-24 - ntop.org -- -- -- This script is executed once at startup similar to /etc/rc.local on Unix -- local dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path package.path = dirs.installdir .. "/scripts/lua/modules/pools/?.lua;" .. package.path package.path = dirs.installdir .. "/scripts/lua/modules/vulnerability_scan/?.lua;" .. package.path -- Important: load this before any other alert related module require "prefs_utils" local checks = require "checks" local radius_handler = require "radius_handler" checks.loadChecks() local recipients = require "recipients" recipients.initialize() local alert_utils = require "alert_utils" local discover_utils = require "discover_utils" local host_pools_nedge = require "host_pools_nedge" local os_utils = require "os_utils" local lists_utils = require "lists_utils" local recovery_utils = require "recovery_utils" local delete_data_utils = require "delete_data_utils" local ts_utils = require "ts_utils" local presets_utils = require "presets_utils" local blog_utils = require("blog_utils") local vs_utils = require "vs_utils" local drop_host_pool_utils = require "drop_host_pool_utils" local json = require "dkjson" -- ################################################################## traceError(TRACE_NORMAL, TRACE_CONSOLE, "Processing startup.lua: please hold on...") if ntop.isPro() then package.path = dirs.installdir .. "/pro/scripts/callbacks/system/?.lua;" .. package.path require("startup") end -- ################################################################## local function migrate_script_config(conf, proto_name) local config = conf.config.flow["unexpected_" .. proto_name] local ret = false if (config ~= nil) then if (config.all.enabled) then local value = config.all.script_conf.items if (value ~= nil) then -- tprint(value) local new_key = "ntopng.prefs.nw_config_" .. proto_name .. "_list" local new_value = "" for k, v in pairs(value) do if (k == 1) then new_value = v else new_value = new_value .. "," .. v end end if (new_value ~= "") then -- tprint(new_key.." = "..new_value) ntop.setCache("ntopng.prefs.nw_config_" .. proto_name .. "_list", new_value) ret = true end config.all.script_conf.items = nil -- remove value from config end end end return ret end local function migrate_unexpected_proto_config() local key = "ntopng.prefs.checks.configset_v1" local conf = json.decode(ntop.getCache(key)) local modified local rc = false rc = rc or migrate_script_config(conf, "dns") rc = rc or migrate_script_config(conf, "ntp") rc = rc or migrate_script_config(conf, "dhcp") rc = rc or migrate_script_config(conf, "smtp") if (rc == true) then -- something has been migrated ntop.setCache(key, json.encode(conf)) end end -- ################################################################## if ntop.isAppliance() then package.path = dirs.installdir .. "/scripts/lua/modules/system_config/?.lua;" .. package.path -- Discard any pending, unfinished, unsaved configuration local appliance_config = require("appliance_config"):create(true):discard() -- Load the actual valid configuration appliance_config = require("appliance_config"):create(false) -- Apply some config prefs local vlan_trunk = appliance_config:isBridgeOverVLANTrunkEnabled() ntop.setPref("ntopng.prefs.enable_vlan_trunk_bridge", ternary(vlan_trunk, "1", "0")) -- Load possibly changed prefs ntop.reloadPreferences() end if ntop.isnEdge() then host_pools_nedge.initPools() radius_handler.deleteAllKeys() end if (ntop.isPro()) then shaper_utils = require "shaper_utils" shaper_utils.initShapers() end -- ################################################################## local has_pcap_dump_interface = false local function cleanupIfname(ifname, ifid) interface.select(ifname) local ifid = getInterfaceId(ifname) if interface.isPcapDumpInterface() then has_pcap_dump_interface = true end local alerts_status_path = os_utils.fixPath( dirs.workingdir .. "/" .. ifid .. "/json/") ntop.rmdir(alerts_status_path) -- Remove network discovery request on startup discover_utils.clearNetworkDiscovery(ifid) -- Clean old InfluxDB export cache local export_dir = os_utils.fixPath(dirs.workingdir .. "/" .. ifid .. "/ts_export") ntop.rmdir(export_dir) -- Clean redis queue used to push host filters to pfring ntop.delCache("pfring." .. ifname .. ".filter.host.queue") -- Check the preference if the interface is mirrored or not -- In case it is, force the serialization key to IP local is_mirrored_traffic = false if not ntop.isnEdge() and is_packet_interface then local is_mirrored_traffic_pref = string.format( "ntopng.prefs.ifid_%d.is_traffic_mirrored", interface.getId()) is_mirrored_traffic = ternary(ntop.getPref(is_mirrored_traffic_pref) == '1', true, false) end -- If it's mirrored or if it's a ZMQ Interface, force the serialization key to IP if is_mirrored_traffic or interface.isZMQInterface() then local serialize_by_mac_key = string.format( "ntopng.prefs.ifid_%u.serialize_local_broadcast_hosts_as_macs", interface.getId()) -- In case of mirrored traffic, force the key to IP ntop.setPref(serialize_by_mac_key, false) interface.updateLbdIdentifier() end end -- Remove the json dumps previously needed for alerts generation for ifid, ifname in pairs(interface.getIfNames()) do cleanupIfname(ifname, ifid) end cleanupIfname(getSystemInterfaceName(), getSystemInterfaceId()) -- Also flush the export queue ntop.delCache("ntopng.influx_file_queue") -- Also flush the alert queue ntop.delCache("ntopng.trace_error.alert_queue") -- ################################################################## local recording_utils = require "recording_utils" recording_utils.checkAvailable() -- ################################################################## local companion_interface_utils = require "companion_interface_utils" companion_interface_utils.initCompanions() -- ################################################################## lists_utils.startup() -- ################################################################## traceError(TRACE_NORMAL, TRACE_CONSOLE, "Initializing device polices...") -- Initialize device policies (presets) -- NOTE: Must go after lists_utils initialization and reload -- as new custom protocols can be set by lists utils presets_utils.init() presets_utils.reloadAllDevicePolicies() -- TODO: migrate custom re-arm settings -- this will retrieve host pools and policers configurtions via HTTP if enabled if ntop.isnEdge() then local http_bridge_conf_utils = require "http_bridge_conf_utils" http_bridge_conf_utils.configureBridge() end traceError(TRACE_NORMAL, TRACE_CONSOLE, "Initializing alerts...") alert_utils.notify_ntopng_start() if not recovery_utils.check_clean_shutdown() then package.path = dirs.installdir .. "/scripts/callbacks/system/?.lua;" .. package.path end recovery_utils.unmark_clean_shutdown() traceError(TRACE_NORMAL, TRACE_CONSOLE, "Initializing timeseries...") -- Need to run setup at startup since the schemas may be changed ts_utils.setupAgain() -- Migrate "iface:ndpi_categories" under the correct folder if (ntop.getCache("ntopng.cache.rrd_category_migration") ~= "1") then for ifid, ifname in pairs(delete_data_utils.list_all_interfaces()) do ntop.mkdir(dirs.workingdir .. "/" .. ifid .. "/rrd/ndpi_categories") for cat_name, cat_id in pairs(interface.getnDPICategories()) do local old_path = os_utils.fixPath( dirs.workingdir .. "/" .. ifid .. "/rrd/" .. cat_name .. ".rrd") if ntop.exists(old_path) then local new_path = os_utils.fixPath( dirs.workingdir .. "/" .. ifid .. "/rrd/ndpi_categories/" .. cat_name .. ".rrd") traceError(TRACE_INFO, TRACE_CONSOLE, "Migrating Category RRD: " .. old_path) os.rename(old_path, new_path) end end end -- do not perform migration again ntop.setCache("ntopng.cache.rrd_category_migration", "1") end -- Clear the unused DHCP cache keys for ifid, ifname in pairs(delete_data_utils.list_all_interfaces()) do ntop.delCache("ntopng.dhcp." .. ifid .. ".cache") end -- Remove notification cache local notifications = ntop.getKeysCache("ntopng.cache.alerts.notification.*") or {} for k, _ in pairs(notifications) do ntop.delCache(k) end if (has_pcap_dump_interface) then -- Load the lists at the very beginning in order to avoid misclassification -- when reading from PCAP dump. This can take some time. traceError(TRACE_NORMAL, TRACE_CONSOLE, "Loading category lists...") lists_utils.checkReloadLists() traceError(TRACE_NORMAL, TRACE_CONSOLE, "Loading category lists done") end -- Show the warning at most 1 time per run ntop.delCache("ntopng.cache.rrd_format_change_warning_shown") -- Check if there is a local file to run local local_startup_file = "/usr/share/ntopng/local/scripts/callbacks/system/startup.lua" if (ntop.exists(local_startup_file)) then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Running " .. local_startup_file) dofile(local_startup_file) end if (ntop.isPro()) then if ntop.isClickHouseEnabled() then -- Check if DB column types are good or need to be fixed interface.select(getSystemInterfaceId()) local db_migration_required = false local res = interface.execSQLQuery("DESCRIBE flows") if res and type(res) == "table" then for _, row in ipairs(res) do if row['name'] == 'SRC_NETWORK_ID' or row['name'] == 'DST_NETWORK_ID' then if row['type'] ~= 'UInt32' then db_migration_required = true end end end end if db_migration_required then traceError(TRACE_NORMAL, TRACE_CONSOLE, "Migrating DB column types...") interface.execSQLQuery( "ALTER TABLE flows MODIFY COLUMN SRC_NETWORK_ID UInt32") interface.execSQLQuery( "ALTER TABLE flows MODIFY COLUMN DST_NETWORK_ID UInt32") interface.execSQLQuery( "ALTER TABLE hourly_flows MODIFY COLUMN SRC_NETWORK_ID UInt32") interface.execSQLQuery( "ALTER TABLE hourly_flows MODIFY COLUMN DST_NETWORK_ID UInt32") interface.execSQLQuery( "ALTER TABLE flows UPDATE SRC_NETWORK_ID = 4294967295 WHERE SRC_NETWORK_ID = 65535") interface.execSQLQuery( "ALTER TABLE flows UPDATE DST_NETWORK_ID = 4294967295 WHERE DST_NETWORK_ID = 65535") interface.execSQLQuery( "ALTER TABLE hourly_flows UPDATE SRC_NETWORK_ID = 4294967295 WHERE SRC_NETWORK_ID = 65535") interface.execSQLQuery( "ALTER TABLE hourly_flows UPDATE DST_NETWORK_ID = 4294967295 WHERE DST_NETWORK_ID = 65535") end -- Import ClickHouse dumps if any local silence_import_warnings = true traceError(TRACE_NORMAL, TRACE_CONSOLE, "Importing ClickHouse dumps...") ntop.importClickHouseDumps(silence_import_warnings) end -- In case of the alert enabled, clean the list of all the elements drop_host_pool_utils.clean_list() end -- Fetch latest ntop blog posts if not ntop.isnEdge() then -- Note: On nEdge they are fetched in a dayly/delayed callback as connectivity -- may be not yet up at this stage blog_utils.fetchLatestPosts() end if ntop.isnEdge() then interface.select(tostring(interface.getFirstInterfaceId())) host_pools_nedge.startupCheckResetPoolsQuotas() interface.select(getSystemInterfaceId()) end -- Cleanup old influxdb files (if any) local influxdb_dir = dirs.workingdir .. "/tmp/influxdb" if (ntop.exists(influxdb_dir)) then local files = ntop.readdir(influxdb_dir) if (files ~= nil) then for _, name in pairs(files) do if (ends(name, ".tmp") == true) then local fname = influxdb_dir .. "/" .. name -- io.write("[InfluxDB] Deleting file "..fname.."\n") ntop.unlink(fname) end end end end -- Vulnerability scan activities vs_utils.migrate_keys() vs_utils.restore_host_to_scan() -- migrate unexpected dns/ntp/dhcp/smtp scripts to /lua/admin/network_configuration.lua migrate_unexpected_proto_config() -- Reload Alert Exclusions ntop.reloadAlertExclusions() -- Removing limits exceeded key, it's used just for the badges in the gui ntop.delCache("ntopng.limits.exporters") -- initialization of mitre attack matrix informations local mitre_utils = require "mitre_utils" local mitre_table = mitre_utils.insertDBMitreInfo() ntop.reloadServersConfiguration() ntop.reloadASNConfiguration() ts_utils.runFirstSetup() traceError(TRACE_NORMAL, TRACE_CONSOLE, "Completed startup.lua")