diff --git a/include/AlertsManager.h b/include/AlertsManager.h index e95c9e644a..4080147053 100644 --- a/include/AlertsManager.h +++ b/include/AlertsManager.h @@ -24,7 +24,7 @@ #include "ntop_includes.h" -class Host; +//class Host; class AlertsManager : protected StoreManager { private: @@ -40,20 +40,34 @@ class AlertsManager : protected StoreManager { int releaseAlert(AlertEntity alert_entity, const char *alert_entity_value, const char *engaged_alert_id, AlertType alert_type, AlertLevel alert_severity, const char *alert_json); + int storeAlert(AlertEntity alert_entity, const char *alert_entity_value, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json); int engageReleaseHostAlert(Host *h, const char *engaged_alert_id, AlertType alert_type, AlertLevel alert_severity, const char *alert_json, bool engage); - /* */ + int engageReleaseNetworkAlert(const char *cidr, + const char *engaged_alert_id, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json, + bool engage); + int engageReleaseInterfaceAlert(NetworkInterface *n, + const char *engaged_alert_id, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json, + bool engage); public: AlertsManager(int interface_id, const char *db_filename); ~AlertsManager() {}; +#ifdef NOTUSED int storeAlert(AlertType alert_type, AlertLevel alert_severity, const char *alert_json); int storeAlert(lua_State *L, int index); +#endif + /* + ========== HOST alerts API ========= + */ inline int engageHostAlert(Host *h, const char *engaged_alert_id, AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { @@ -64,8 +78,55 @@ class AlertsManager : protected StoreManager { AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { return engageReleaseHostAlert(h, engaged_alert_id, alert_type, alert_severity, alert_json, false /* release */); }; + int storeHostAlert(Host *h, AlertType alert_type, AlertLevel alert_severity, const char *alert_json); + /* + ========== FLOW alerts API ========= + */ + inline int storeFlowAlert(Flow *f, AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { + return storeAlert(alert_entity_flow, ""/* TODO: possibly add an unique id for flows */, + alert_type, alert_severity, alert_json); + }; + + /* + ========== NETWORK alerts API ====== + */ + inline int engageNetworkAlert(const char *cidr, + const char *engaged_alert_id, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { + return engageReleaseNetworkAlert(cidr, engaged_alert_id, alert_type, alert_severity, alert_json, true /* engage */); + }; + inline int releaseNetworkAlert(const char *cidr, + const char *engaged_alert_id, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { + return engageReleaseNetworkAlert(cidr, engaged_alert_id, alert_type, alert_severity, alert_json, false /* release */); + }; + int storeNetworkAlert(const char *cidr, AlertType alert_type, AlertLevel alert_severity, const char *alert_json); + + /* + ========== INTERFACE alerts API ====== + */ + inline int engageInterfaceAlert(NetworkInterface *n, + const char *engaged_alert_id, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { + return engageReleaseInterfaceAlert(n, engaged_alert_id, alert_type, alert_severity, alert_json, true /* engage */); + }; + inline int releaseInterfaceAlert(NetworkInterface *n, + const char *engaged_alert_id, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { + return engageReleaseInterfaceAlert(n, engaged_alert_id, alert_type, alert_severity, alert_json, false /* release */); + }; + int storeInterfaceAlert(NetworkInterface *n, AlertType alert_type, AlertLevel alert_severity, const char *alert_json); + + + int getAlerts(lua_State* vm, patricia_tree_t *allowed_hosts, + u_int32_t start_offset, u_int32_t end_offset, + bool engaged); + int getNumAlerts(bool engaged); + int deleteAlerts(bool engaged, const int *rowid); + /* Following are the legacy methods that were formally global to the whole ntopng */ +#ifdef NOTUSED /** * @brief Queue an alert in redis * @@ -102,6 +163,7 @@ class AlertsManager : protected StoreManager { * */ int flushAllQueuedAlerts(); +#endif }; #endif /* _ALERTS_MANAGER_H_ */ diff --git a/include/CollectorInterface.h b/include/CollectorInterface.h index e10bab0490..d2a142532a 100644 --- a/include/CollectorInterface.h +++ b/include/CollectorInterface.h @@ -44,7 +44,7 @@ class CollectorInterface : public ParserInterface { CollectorInterface(const char *_endpoint); ~CollectorInterface(); - inline const char* get_type() { return(CONST_INTERFACE_TYPE_VIEW); }; + inline const char* get_type() { return(CONST_INTERFACE_TYPE_ZMQ); }; inline bool is_ndpi_enabled() { return(false); }; inline char* getEndpoint(u_int8_t id) { return((id < num_subscribers) ? subscriber[id].endpoint : (char*)""); }; diff --git a/include/DB.h b/include/DB.h index 2d9c625b31..75e4328074 100644 --- a/include/DB.h +++ b/include/DB.h @@ -37,6 +37,8 @@ class DB { virtual int exec_sql_query(lua_State *vm, char *sql, bool limit_rows); virtual void startDBLoop(); virtual bool createDBSchema() {return false; /* override in non-schemaless subclasses */}; + virtual void updateStats(const struct timeval *tv) {}; + virtual void lua(lua_State* vm) const {}; }; #endif /* _DB_CLASS_H_ */ diff --git a/include/ElasticSearch.h b/include/ElasticSearch.h index da4fcae122..32f8a0d775 100644 --- a/include/ElasticSearch.h +++ b/include/ElasticSearch.h @@ -31,7 +31,10 @@ class ElasticSearch { struct string_list *head, *tail; pthread_rwlock_t listMutex; bool reportDrops; + struct timeval lastUpdateTime; u_int32_t elkDroppedFlowsQueueTooLong; + u_int64_t elkExportedFlows, elkLastExportedFlows; + float elkExportRate; public: ElasticSearch(); ~ElasticSearch(); @@ -40,6 +43,9 @@ class ElasticSearch { void pushEStemplate(); void indexESdata(); void startFlowDump(); + + void updateStats(const struct timeval *tv); + void lua(lua_State* vm) const; }; diff --git a/include/Host.h b/include/Host.h index 3547e4b221..0100cd5b26 100644 --- a/include/Host.h +++ b/include/Host.h @@ -54,6 +54,7 @@ class Host : public GenericHost { CountMinSketch *sent_to_sketch, *rcvd_from_sketch; #endif AlertCounter *syn_flood_attacker_alert, *syn_flood_victim_alert; + bool flow_flood_attacker_alert, flow_flood_victim_alert; TrafficStats tcp_sent, tcp_rcvd; TrafficStats udp_sent, udp_rcvd; TrafficStats icmp_sent, icmp_rcvd; @@ -96,7 +97,7 @@ class Host : public GenericHost { public: Host(NetworkInterface *_iface); - Host(NetworkInterface *_iface, char *ipAddress); + Host(NetworkInterface *_iface, char *ipAddress, u_int16_t _vlanId); Host(NetworkInterface *_iface, u_int8_t mac[6], u_int16_t _vlanId); Host(NetworkInterface *_iface, u_int8_t mac[6], u_int16_t _vlanId, IpAddress *_ip); ~Host(); diff --git a/include/MySQLDB.h b/include/MySQLDB.h index 3913905798..d63f36f8e4 100644 --- a/include/MySQLDB.h +++ b/include/MySQLDB.h @@ -28,8 +28,11 @@ class MySQLDB : public DB { private: MYSQL mysql; bool db_operational; + struct timeval lastUpdateTime; + u_int32_t mysqlDroppedFlowsQueueTooLong; - u_int64_t mysqlExportedFlows; + u_int64_t mysqlExportedFlows, mysqlLastExportedFlows; + float mysqlExportRate; static volatile bool db_created; pthread_t queryThreadLoop; @@ -47,9 +50,12 @@ class MySQLDB : public DB { bool createDBSchema(); static volatile bool isDbCreated() {return db_created;}; inline u_int32_t numDroppedFlows() const { return mysqlDroppedFlowsQueueTooLong; }; + inline float exportRate() const { return mysqlExportRate; }; bool dumpFlow(time_t when, bool partial_dump, Flow *f, char *json); int exec_sql_query(lua_State *vm, char *sql, bool limitRows); void startDBLoop(); + void updateStats(const struct timeval *tv); + void lua(lua_State* vm) const; }; #endif /* _MYSQL_DB_CLASS_H_ */ diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index 6b2fd05f28..b532e93483 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -218,7 +218,7 @@ class NetworkInterface { inline int get_datalink() { return(pcap_datalink_type); }; inline void set_datalink(int l) { pcap_datalink_type = l; }; inline int isRunning() { return running; }; - bool restoreHost(char *host_ip); + bool restoreHost(char *host_ip, u_int16_t vlan_id); u_int printAvailableInterfaces(bool printHelp, int idx, char *ifname, u_int ifname_len); void findFlowHosts(u_int16_t vlan_id, u_int8_t src_mac[6], IpAddress *_src_ip, Host **src, diff --git a/include/Prefs.h b/include/Prefs.h index 282395397e..9b08773853 100644 --- a/include/Prefs.h +++ b/include/Prefs.h @@ -42,13 +42,13 @@ class Prefs { char *deferred_interfaces_to_register[MAX_NUM_INTERFACES], *cli; char *http_binding_address, *https_binding_address; Ntop *ntop; - bool enable_dns_resolution, sniff_dns_responses, disable_host_persistency, + bool enable_dns_resolution, sniff_dns_responses, enable_idle_local_hosts_cache, categorization_enabled, resolve_all_host_ip, change_user, daemonize, enable_auto_logout, use_promiscuous_mode, disable_alerts, enable_ixia_timestamps, enable_vss_apcon_timestamps, enable_users_login, disable_localhost_login, online_license_check; LocationPolicy dump_hosts_to_db, sticky_hosts; - u_int non_local_host_max_idle, local_host_max_idle, flow_max_idle; + u_int non_local_host_max_idle, local_host_cache_duration, local_host_max_idle, flow_max_idle; u_int16_t intf_rrd_raw_days, intf_rrd_1min_days, intf_rrd_1h_days, intf_rrd_1d_days; u_int16_t other_rrd_raw_days, other_rrd_1min_days, other_rrd_1h_days, other_rrd_1d_days; u_int16_t housekeeping_frequency; @@ -148,8 +148,8 @@ class Prefs { inline char* get_instance_name() { return(instance_name); }; inline bool are_alerts_disabled() { return(disable_alerts); }; inline void set_alerts_status(bool enabled) { if(enabled) disable_alerts = false; else disable_alerts = true; }; - inline bool is_host_persistency_enabled() { return(disable_host_persistency ? false : true); }; - inline bool do_auto_logout() { return(enable_auto_logout); }; + inline bool is_idle_local_host_cache_enabled() { return(enable_idle_local_hosts_cache); }; + inline bool do_auto_logout() { return(enable_auto_logout); }; inline char* get_cpu_affinity() { return(cpu_affinity); }; inline u_int get_http_port() { return(http_port); }; inline u_int get_https_port() { return(https_port); }; @@ -159,7 +159,8 @@ class Prefs { inline u_int get_redis_db_id() { return(redis_db_id); }; inline char* get_pid_path() { return(pid_path); }; inline char* get_packet_filter() { return(packet_filter); }; - inline u_int16_t get_host_max_idle(bool localHost) { return(localHost ? local_host_max_idle : non_local_host_max_idle); }; + inline u_int get_host_max_idle(bool localHost) { return(localHost ? local_host_max_idle : non_local_host_max_idle); }; + inline u_int get_local_host_cache_duration() { return(local_host_cache_duration); }; inline u_int16_t get_housekeeping_frequency() { return(housekeeping_frequency); }; inline u_int16_t get_flow_max_idle() { return(flow_max_idle); }; inline u_int32_t get_max_num_hosts() { return(max_num_hosts); }; diff --git a/include/ntop_defines.h b/include/ntop_defines.h index 02fdcde993..dde65edee7 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -137,6 +137,7 @@ #define DNS_CACHE_DURATION 3600 /* 1 h */ #define LOCAL_HOSTS_CACHE_DURATION 3600 /* 1 h */ #define HOST_LABEL_NAMES "ntopng.host_labels" +#define HOST_SERIALIZED_KEY "ntopng.serialized_hosts.ifid_%u__%s@%d" #define NTOP_HOSTS_SERIAL "ntopng.host_serial" #define DUMMY_IFACE_ID (MAX_NUM_INTERFACES-2) #define STDIN_IFACE_ID (MAX_NUM_INTERFACES-3) @@ -272,7 +273,8 @@ #define NTOPNG_NDPI_OS_PROTO_ID (NDPI_LAST_IMPLEMENTED_PROTOCOL+NDPI_MAX_NUM_CUSTOM_PROTOCOLS-2) #define CONST_DEFAULT_HOME_NET "192.168.1.0/24" #define CONST_DEFAULT_DATA_DIR "/var/tmp/ntopng" -#define CONST_DEFAULT_IS_AUTOLOGOUT_ENABLED 1 +#define CONST_DEFAULT_IS_AUTOLOGOUT_ENABLED 1 +#define CONST_DEFAULT_IS_IDLE_LOCAL_HOSTS_CACHE_ENABLED 1 #define CONST_DEFAULT_DOCS_DIR "httpdocs" #define CONST_DEFAULT_SCRIPTS_DIR "scripts" #define CONST_FLOWACTIVITY_SCRIPT "flowactivity.lua" @@ -293,7 +295,7 @@ #define CONST_LUA_ERROR 0 #define CONST_LUA_PARAM_ERROR -1 #define CONST_MAX_NUM_SYN_PER_SECOND 8192 -#define CONST_MAX_NUM_HOST_ACTIVE_FLOWS 65536 +#define CONST_MAX_NUM_HOST_ACTIVE_FLOWS 32768 #define CONST_MAX_NEW_FLOWS_SECOND 25 #define CONST_ALERT_GRACE_PERIOD 60 /* No more than 1 alert/min */ #define CONST_CONTACTED_BY "contacted_by" @@ -324,7 +326,6 @@ #define CONST_NBOX_USER "ntopng.prefs.nbox_user" #define CONST_NBOX_PASSWORD "ntopng.prefs.nbox_password" #define CONST_IFACE_ID_PREFS "ntopng.prefs.iface_id" -#define CONST_LOCAL_HOST_IDLE_PREFS "ntopng.prefs.local_host_max_idle" #define CONST_REMOTE_HOST_IDLE_PREFS "ntopng.prefs.non_local_host_max_idle" #define CONST_FLOW_MAX_IDLE_PREFS "ntopng.prefs.flow_max_idle" #define CONST_MAX_NEW_FLOWS_PREFS "ntopng.prefs.host_max_new_flows_sec_threshold" @@ -340,8 +341,11 @@ #define CONST_OTHER_RRD_1D_DAYS "ntopng.prefs.other_rrd_1d_days" #define CONST_PROFILES_PREFS "ntopng.prefs.profiles" +#define CONST_LOCAL_HOST_CACHE_DURATION_PREFS "ntopng.prefs.local_host_cache_duration" +#define CONST_LOCAL_HOST_IDLE_PREFS "ntopng.prefs.local_host_max_idle" #define CONST_RUNTIME_IS_AUTOLOGOUT_ENABLED "ntopng.prefs.is_autologon_enabled" +#define CONST_RUNTIME_IDLE_LOCAL_HOSTS_CACHE_ENABLED "ntopng.prefs.is_local_host_cache_enabled" #define CONST_RUNTIME_PREFS_HOUSEKEEPING_FREQUENCY "ntopng.prefs.housekeeping_frequency" #define CONST_RUNTIME_PREFS_HOST_RRD_CREATION "ntopng.prefs.host_rrd_creation" /* 0 / 1 */ #define CONST_RUNTIME_PREFS_HOST_NDPI_RRD_CREATION "ntopng.prefs.host_ndpi_rrd_creation" /* 0 / 1 */ @@ -489,10 +493,9 @@ // sqlite (StoreManager and subclasses) related fields #define STORE_MANAGER_MAX_QUERY 1024 #define STORE_MANAGER_MAX_KEY 20 -#define ALERTS_MANAGER_EXPERIMENTS "exp_" #define ALERTS_MANAGER_TABLE_NAME "closed_alerts" #define ALERTS_MANAGER_ENGAGED_TABLE_NAME "engaged_alerts" -#define ALERTS_MANAGER_STORE_NAME ALERTS_MANAGER_EXPERIMENTS "alerts.db" +#define ALERTS_MANAGER_STORE_NAME "alerts.db" #define ALERTS_MANAGER_QUEUE_NAME "ntopng.alerts.ifid_%i.queue" #define ALERTS_MANAGER_TYPE_FIELD "alert_type" #define ALERTS_MANAGER_SEVERITY_FIELD "alert_severity" diff --git a/include/ntop_typedefs.h b/include/ntop_typedefs.h index 1abe11da10..897d2b91d9 100644 --- a/include/ntop_typedefs.h +++ b/include/ntop_typedefs.h @@ -68,7 +68,8 @@ typedef enum { alert_entity_interface = 0, alert_entity_host, alert_entity_network, - alert_entity_snmp_device + alert_entity_snmp_device, + alert_entity_flow } AlertEntity; typedef enum { diff --git a/ntopng.8 b/ntopng.8 index 950fd9cab0..a0d8ece54d 100644 --- a/ntopng.8 +++ b/ntopng.8 @@ -339,7 +339,9 @@ lines in your proxy's configuration: ProxyPass /ntopng/ http://192.168.0.3:3000/ntopng/ ProxyPassReverse /ntopng/ http://192.168.0.3:3000/ntopng/ .br -you must use ntopng with \-Z "/ntopng/" +You must use ntopng with \-Z "/ntopng" +.br +Do not use trailing shashes in the HTTP prefix. .It \-\-shutdown\-when\-done Terminate ntopng when the input pcap file is over (debug only). diff --git a/packages/ntopng.spec.in b/packages/ntopng.spec.in index 984de3e120..2e628c3faa 100644 --- a/packages/ntopng.spec.in +++ b/packages/ntopng.spec.in @@ -9,7 +9,7 @@ Source: ntopng-%{version}.tgz Packager: Luca Deri # Temporary location where the RPM will be built BuildRoot: %{_tmppath}/%{name}-%{version}-root -Requires: pfring = @PFRING_VERSION@-@PFRING_GIT_RELEASE@, redis >= 2.4.0, GeoIP >= 1.4.8, rrdtool >= 1.3.8, numactl, libcurl, ntopng-data, logrotate, zeromq, openldap, openssl, libnetfilter_queue +Requires: pfring = @PFRING_VERSION@-@PFRING_GIT_RELEASE@, redis >= 2.4.0, GeoIP >= 1.4.8, rrdtool >= 1.3.8, numactl, libcurl, ntopng-data, logrotate, zeromq >= 4.0.0, openldap, openssl, hiredis, mysql, libnetfilter_queue # Disable shared libs dependency check (needed by FPGA libs) AutoReqProv: no diff --git a/scripts/callbacks/flowactivity.lua b/scripts/callbacks/flowactivity.lua index 1afc5153cc..0815c7f343 100644 --- a/scripts/callbacks/flowactivity.lua +++ b/scripts/callbacks/flowactivity.lua @@ -2,7 +2,9 @@ -- (C) 2016 - ntop.org -- +-- Enable tracings here local trace_hk = false + local profile_activity_match local default_activity_parameters = {filter.SMA} local media_activity_defaults = {filter.SMA, --[[min bytes]] 500, --[[min samples]]1, --[[bound time]]500, --[[sustain time]]4000} @@ -59,52 +61,29 @@ end -- ######################################################## +-- +-- This callback is called periodically for all active flows +-- Add here housekeeping of periodic activities you want to +-- perform in a flow +-- function flowUpdate() if(trace_hk) then print("flowUpdate()\n") end - -- print("=>"..flow.getNdpiProto().."@"..flow.getProfileId().."\n") - -- flow.setProfileId(os.time()) - - local proto = flow.getNdpiProto() - local master, sub = splitProto(proto) - - if master == "HTTP" then - local contentType = flow.getHTTPContentType() - if contentType then - if flow.getProfileId() ~= profile.Media then - local mDetected = false - - -- Try to detect a media type - for i=1, #media_activity_mime_types do - if contentType:starts(media_activity_mime_types[i]) then - flow.setActivityFilter(profile.Media, unpack(media_activity_defaults)) - mDetected = true - break - end - end - - -- Try to detect a web type - if not mDetected and flow.getActivityFilterId() ~= filter.All then - for i=1, #web_activity_mime_types do - if contentType:starts(web_activity_mime_types[i]) then - -- Be always active - flow.setActivityFilter(profile.Web, filter.All, true) - break - end - end - end - end - end - end end -- ######################################################## +-- +-- This callback is called once, when a new flow is created +-- function flowCreate() if(trace_hk) then print("flowCreate()\n") end end -- ######################################################## +-- +-- This callback is called once, when a new flow is deleted +-- function flowDelete() if(trace_hk) then print("flowDelete()\n") end end @@ -330,6 +309,10 @@ local profile_activity_match = { -- ######################################################## +-- +-- This callback is called once as soon as the flow application +-- protocol has been identified by the ntopng core +-- function flowProtocolDetected() local proto = flow.getNdpiProto() local master, sub = splitProto(proto) @@ -337,7 +320,35 @@ function flowProtocolDetected() local matched = nil if master ~= "DNS" then - + if master == "HTTP" then + local contentType = flow.getHTTPContentType() + if contentType then + if flow.getProfileId() ~= profile.Media then + local mDetected = false + + -- Try to detect a media type + for i=1, #media_activity_mime_types do + if contentType:starts(media_activity_mime_types[i]) then + flow.setActivityFilter(profile.Media, unpack(media_activity_defaults)) + mDetected = true + break + end + end + + -- Try to detect a web type + if not mDetected and flow.getActivityFilterId() ~= filter.All then + for i=1, #web_activity_mime_types do + if contentType:starts(web_activity_mime_types[i]) then + -- Be always active + flow.setActivityFilter(profile.Web, filter.All, true) + break + end + end + end + end + end + end + -- BEGIN Particular protocols if sub == "YouTube" and srv:ends("googlevideo.com") then matched = {["profile"]=profile.Media, ["config"]={filter.All, true}} @@ -380,7 +391,7 @@ function flowProtocolDetected() end if(trace_hk) then - f = flow.dump() + f = flow.dump() print("flowProtocolDetected(".. getFlowKey(f)..") = "..f["proto.ndpi"].."\n") end end diff --git a/scripts/callbacks/startup.lua b/scripts/callbacks/startup.lua index 73636da409..0c01dfea92 100644 --- a/scripts/callbacks/startup.lua +++ b/scripts/callbacks/startup.lua @@ -12,6 +12,29 @@ end require "lua_utils" require "alert_utils" +local prefs = ntop.getPrefs() + +-- restore sticky hosts +if prefs.sticky_hosts ~= nil then + -- if the sticky hosts are set, then we try and restore them out of redis + for _, ifname in pairs(interface.getIfNames()) do + interface.select(ifname) + local ifid = getInterfaceId(ifname) + -- an example key is ntopng.serialized_hosts.ifid_6__192.168.2.136@0 + local keys_pattern = "ntopng.serialized_hosts.ifid_"..ifid.."__*" + local dumped_hosts = ntop.getKeysCache("ntopng.serialized_hosts.ifid_"..ifid.."__*") + if dumped_hosts ~= nil then + for hostkey, _ in pairs(dumped_hosts) do + -- let's extract just the host name and vlan from the whole key; + -- restore host will do the rest ... + hostkey = string.split(hostkey, "__") -- the double-underscore separates host info + hostkey = hostkey[2] + interface.restoreHost(hostkey, true --[[ skip privileges checks: no web access --]]) + end + end + end +end + -- old host alerts were global and did not consider vlans -- this part of the script aims at converting old global alerts to per-interface, vlan aware alerts diff --git a/scripts/lua/admin/prefs.lua b/scripts/lua/admin/prefs.lua index afcb97a805..30a6e1ef28 100644 --- a/scripts/lua/admin/prefs.lua +++ b/scripts/lua/admin/prefs.lua @@ -39,7 +39,7 @@ users_active = "" logging_active = "" if (subpage_active == nil or subpage_active == "") then - subpage_active = "report" + subpage_active = "users" end if (subpage_active == "report") then @@ -344,10 +344,24 @@ function printInMemory() print('') print('') - prefsInputFieldPrefs("Local Host Idle Timeout", "Inactivity time after which a local host is considered idle (sec). Default: 300.", "ntopng.prefs.","local_host_max_idle", prefs.local_host_max_idle) + prefsInputFieldPrefs("Local Host Idle Timeout", "Inactivity time after which a local host is considered idle (sec). ".. + "Idle local hosts are dumped to a cache so their counters can be restored in case they become active again. ".. + "Counters include, but are not limited to, packets and bytes total and per Layer-7 application. ".. + "Default: 300.", "ntopng.prefs.","local_host_max_idle", prefs.local_host_max_idle) prefsInputFieldPrefs("Remote Host Idle Timeout", "Inactivity time after which a remote host is considered idle (sec). Default: 60.", "ntopng.prefs.", "non_local_host_max_idle", prefs.non_local_host_max_idle) prefsInputFieldPrefs("Flow Idle Timeout", "Inactivity time after which a flow is considered idle (sec). Default: 60.", "ntopng.prefs.", "flow_max_idle", prefs.flow_max_idle) + print('') + toggleTableButtonPrefs("Local Host Cache", + "Toggle the creation of cache entries for idle local hosts. ".. + "Cached local hosts counters are restored automatically to their previous values ".. + " upon detection of additional host traffic.", + "On", "1", "success", "Off", "0", "danger", + "toggle_local_host_cache_enabled", + "ntopng.prefs.is_local_host_cache_enabled", "1") + prefsInputFieldPrefs("Local Host Cache Duration", "Time after which an idle local host is deleted from the cache (sec). ".. + "Default: 3600.", "ntopng.prefs.","local_host_cache_duration", prefs.local_host_cache_duration) + print('') prefsInputFieldPrefs("Update frequency in seconds", "Some host statistics such as throughputs are updated periodically. ".. diff --git a/scripts/lua/get_alerts_data.lua b/scripts/lua/get_alerts_data.lua index 797773efef..7e0c5121a4 100644 --- a/scripts/lua/get_alerts_data.lua +++ b/scripts/lua/get_alerts_data.lua @@ -11,6 +11,8 @@ sendHTTPHeader('text/html; charset=iso-8859-1') currentPage = _GET["currentPage"] perPage = _GET["perPage"] +status = _GET["alert_status"] +alertsImpl = _GET["alerts_impl"] if(currentPage == nil) then currentPage = 1 @@ -24,24 +26,37 @@ else perPage = tonumber(perPage) end +engaged = false +if status == "engaged" then + engaged = true +end + initial_idx = (currentPage-1)*perPage interface.select(ifname) -alerts = interface.getQueuedAlerts(initial_idx, perPage) + +alerts = interface.getAlerts(initial_idx, perPage, engaged) +num_alerts = interface.getNumAlerts(engaged) print ("{ \"currentPage\" : " .. currentPage .. ",\n \"data\" : [\n") total = 0 if alerts == nil then alerts = {} end -for _key,_value in pairs(alerts) do - if(total > 0) then print(",\n") end - values = split(string.gsub(_value, "\n", ""), "|") - column_id = "" - column_date = os.date("%c", values[1]) - column_severity = alertSeverityLabel(tonumber(values[2])) - column_type = alertTypeLabel(tonumber(values[4])) - column_msg = values[5] +for _key,_value in ipairs(alerts) do + if(total > 0) then print(",\n") end + + alert_id = _value["rowid"] + column_date = os.date("%c", _value["alert_tstamp"]) + if tonumber(_value["alert_tstamp_end"]) ~= nil then + local duration = secondsToTime(tonumber(_value["alert_tstamp_end"]) - tonumber(_value["alert_tstamp"])) + column_date = duration.." ending on "..os.date("%c", _value["alert_tstamp_end"]) + end + column_severity = alertSeverityLabel(tonumber(_value["alert_severity"])) + column_type = alertTypeLabel(tonumber(_value["alert_type"])) + column_msg = _value["alert_json"] + + column_id = "" print('{ "column_key" : "'..column_id..'", "column_date" : "'..column_date..'", "column_severity" : "'..column_severity..'", "column_type" : "'..column_type..'", "column_msg" : "'..column_msg..'" }') @@ -51,4 +66,4 @@ end -- for print ("\n], \"perPage\" : " .. perPage .. ",\n") print ("\"sort\" : [ [ \"\", \"\" ] ],\n") -print ("\"totalRows\" : " .. interface.getNumQueuedAlerts() .. " \n}") +print ("\"totalRows\" : " ..num_alerts .. " \n}") diff --git a/scripts/lua/get_host_activity.lua b/scripts/lua/get_host_activity.lua index 7083d1cc93..529b65405e 100644 --- a/scripts/lua/get_host_activity.lua +++ b/scripts/lua/get_host_activity.lua @@ -14,12 +14,17 @@ sendHTTPHeader('text/html; charset=iso-8859-1') local res = {} local name_map = { + { "Chat", "Chat and Realtime Communications" }, { "RemoteControl", "Remote Access" }, { "MailSend", "Email Send" }, + { "Media", "Media and Streaming" }, { "MailSync", "Email Synchronization" }, { "FileTransfer", "File Transfer" }, { "FileSharing", "File Sharing" }, { "SocialNetwork", "Social Networks" }, + { "Web", "Web Browsing" }, + { "Game", "Online Gaming" }, + { "Other", "Other Traffic" } } function mapRRDname(name) diff --git a/scripts/lua/host_details.lua b/scripts/lua/host_details.lua index ab79f68769..28bb1c1aae 100644 --- a/scripts/lua/host_details.lua +++ b/scripts/lua/host_details.lua @@ -27,6 +27,15 @@ always_show_hist = _GET["always_show_hist"] ntopinfo = ntop.getInfo() active_page = "hosts" +interface.select(ifname) +ifstats = interface.getStats() + +ifId = ifstats.id + +is_packetdump_enabled = isLocalPacketdumpEnabled() +host = nil +family = nil + local hostkey = hostinfo2hostkey(host_info, nil, true --[[ force show vlan --]]) if((host_name == nil) or (host_ip == nil)) then @@ -64,14 +73,7 @@ end if(protocol_id == nil) then protocol_id = "" end -interface.select(ifname) -ifstats = interface.getStats() -ifId = ifstats.id - -is_packetdump_enabled = isLocalPacketdumpEnabled() -host = nil -family = nil -- print(">>>") print(host_info["host"]) print("<<<") if(debug_hosts) then traceError(TRACE_DEBUG,TRACE_CONSOLE, "Host:" .. host_info["host"] .. ", Vlan: "..host_vlan.."\n") end @@ -653,7 +655,7 @@ end end print("") - if(host["tcp.packets.seq_problems"]) then + if host["tcp.packets.seq_problems"] == true then print("\n") print("\n") print("\n") @@ -1158,8 +1160,9 @@ if host["localhost"] == true then setShowMode("updown");

- NOTE: The above map filters host application traffic by splitting it in real user reaffic (e.g. web page access) -
and background traffic (e.g. your email client periodically checks for email presence). Note that this work is still in progress. + NOTE:
The above map filters host application traffic by splitting it in real user reaffic (e.g. web page access) +
and background traffic (e.g. your email client periodically checks for email presence). Host traffic sent (upload)
+is marked as positive value in blue, traffic received (download) is marked as negative in green. ]] -- showHostActivityStats(hostbase, "", "1h") @@ -1567,7 +1570,6 @@ print [[

Idle Timeout Settings
Idle Local Hosts Cache Settings
Hosts Statistics Update Frequency
TCP Packets Sent AnalysisRetransmissions".. formatPackets(host["tcp.packets.retransmissions"]) .."
Out of Order".. formatPackets(host["tcp.packets.out_of_order"]) .."
Lost".. formatPackets(host["tcp.packets.lost"]) .."
]] elseif(page == "snmp") then - if(ntop.isPro()) then print_snmp_report(host_info["host"], true, ifId) end diff --git a/scripts/lua/if_stats.lua b/scripts/lua/if_stats.lua index ee9c528397..4de986d7ff 100644 --- a/scripts/lua/if_stats.lua +++ b/scripts/lua/if_stats.lua @@ -385,7 +385,7 @@ print("\n") if(ifstats.stats.drops > 0) then print('') end print("  \n") - if(prefs.is_dump_flows_enabled) and false --[[ temporarily disabled --]] then + if(prefs.is_dump_flows_enabled) then local dump_to = "MySQL" if prefs.is_dump_flows_to_es_enabled == true then dump_to = "ElasticSearch" @@ -394,7 +394,8 @@ print("\n") print("") print("Exported Flows") - print(""..formatValue(2048).." ["..formatValue(1024).." Flows/s]") + print(""..formatValue(ifstats.stats.flow_export_count).."") + print(" ["..formatValue(round(ifstats.stats.flow_export_rate, 2)).." Flows/s]") print("Dropped Flows") local span_danger = "" if(ifstats.stats.flow_export_drops > 0) then @@ -848,6 +849,7 @@ else --print(alerts) if(to_save) then + refresh_alert_configuration(ifname_clean, ifname, tab, alerts) if(alerts == "") then ntop.delHashCache(get_alerts_hash_name(tab, ifname), ifname_clean) else @@ -1425,6 +1427,8 @@ print [["; if(rsp.drops > 0) { drops = drops + ''; } $('#if_drops').html(drops); + $('#exported_flows').html(fint(rsp.flow_export_count)); + $('#exported_flows_rate').html(Math.round(rsp.flow_export_rate * 100) / 100); if(rsp.flow_export_drops > 0) { $('#exported_flows_drops') .addClass("label label-danger") diff --git a/scripts/lua/inc/footer.lua b/scripts/lua/inc/footer.lua index 06220003e1..92a1971114 100644 --- a/scripts/lua/inc/footer.lua +++ b/scripts/lua/inc/footer.lua @@ -305,24 +305,37 @@ print [[/lua/logout.lua"); }, */ var bps = Math.round((bytes_diff*8) / epoch_diff ); var bps_local2remote = Math.round((local_diff*8) / epoch_diff); var bps_remote2local = Math.round((remote_diff*8) / epoch_diff); - + + /* don't use the remote_{b,p}ps values to update the gauge if(rsp.remote_pps != 0) { pps = Math.max(rsp.remote_pps, 0); } if(rsp.remote_bps != 0) { bps = Math.max(rsp.remote_bps, 0); } + */ $('#gauge_text_allTraffic').html(bitsToSize(bps, 1000) + " [" + addCommas(pps) + " pps]"); $('#chart-local2remote-text').html(" "+bitsToSize(bps_local2remote, 1000)); $('#chart-remote2local-text').html(" "+bitsToSize(bps_remote2local, 1000)); var msg = "Uptime: "+rsp.uptime+"
"; - if(rsp.alerts > 0) { + if(rsp.alerts > 0 || rsp.engaged_alerts > 0) { msg += "  "+addCommas(rsp.alerts)+" Alert"; - if(rsp.alerts > 1) msg += "s"; +print [[/lua/show_alerts.lua>" - msg += " "; + if(rsp.engaged_alerts > 0) { + msg += " "+addCommas(rsp.engaged_alerts)+" Engaged Alert"; + if(rsp.engaged_alerts > 1) msg += "s"; + msg += ""; + } + + if(rsp.alerts > 0) { + msg += " "+addCommas(rsp.alerts)+" Alert"; + if(rsp.alerts > 1) msg += "s"; + msg += ""; } + msg += " " + } + var alarm_threshold_low = 60; /* 60% */ var alarm_threshold_high = 90; /* 90% */ var alert = 0; diff --git a/scripts/lua/inc/menu.lua b/scripts/lua/inc/menu.lua index 02a95b1286..49e4138d95 100644 --- a/scripts/lua/inc/menu.lua +++ b/scripts/lua/inc/menu.lua @@ -287,7 +287,7 @@ print [[/lua/logout.lua"> Logout ]] print(_COO ]] end -if(interface.getNumQueuedAlerts() > 0 and ntop.getPref("ntopng.prefs.disable_alerts_generation") ~= "1") then +if((interface.getNumAlerts(true) > 0 or interface.getNumAlerts(false) > 0) and ntop.getPref("ntopng.prefs.disable_alerts_generation") ~= "1") then print [[
  • Threshold "..t[1].."@"..key.." not crossed [value="..val.."]["..op.." "..t[3].."]

    \n") end - if ntop.isPro() and not is_alert_re_arming(key, mode, t[1], ifname) then - ntop.withdrawNagiosAlert(string.gsub(key, "@0", "") --[[ vlan 0 is implicit for hosts --]], - mode, t[1], "service OK") - if ntop.isEnterprise() then - interface.releaseHostAlert(key, alert_id, alert_type, alert_level, "released!") - end + if not is_alert_re_arming(key, mode, t[1], ifname) then + interface.releaseHostAlert(key, alert_id, alert_type, alert_level, "released!") + if ntop.isPro() then + ntop.withdrawNagiosAlert(string.gsub(key, "@0", "") --[[ vlan 0 is implicit for hosts --]], + mode, t[1], "service OK") + end end end end @@ -361,6 +364,10 @@ function check_network_alert(ifname, network_name, mode, key, old_table, new_tab tprint(old_table) end + local alert_level = 1 -- alert_level_warning + local alert_status = 1 -- alert_on + local alert_type = 2 -- alert_threshold_exceeded + deltas = {} local delta_names = {'ingress', 'egress', 'inner'} for i = 1, 3 do @@ -397,19 +404,17 @@ function check_network_alert(ifname, network_name, mode, key, old_table, new_tab local f = loadstring(what) local rc = f() + local alert_id = mode.."_"..t[1] -- the alert identifies is the concat. of time granularity and condition, e.g., min_bytes if(rc) then local alert_msg = "Threshold "..t[1].." crossed by network "..network_name.." [".. val .." ".. op .. " " .. t[3].."]" - local alert_level = 1 -- alert_level_warning - local alert_status = 1 -- alert_on - local alert_type = 2 -- alert_threshold_exceeded if not is_alert_re_arming(network_name, mode, t[1], ifname) then if verbose then io.write("queuing alert\n") end re_arm_alert(network_name, mode, t[1], ifname) - interface.queueAlert(alert_level, alert_status, alert_type, alert_msg) + interface.engageNetworkAlert(network_name, alert_id, alert_type, alert_level, alert_msg) if ntop.isPro() then -- possibly send the alert to nagios as well - ntop.sendNagiosAlert(network_name, mode, t[1], alert_msg) + ntop.sendNagiosAlert(network_name, mode, t[1], alert_msg) end else if verbose then io.write("alarm silenced, re-arm in progress\n") end @@ -417,8 +422,11 @@ function check_network_alert(ifname, network_name, mode, key, old_table, new_tab if(verbose) then print("".. alert_msg .."
    \n") end else if(verbose) then print("

    Network threshold "..t[1].."@"..network_name.." not crossed [value="..val.."]["..op.." "..t[3].."]

    \n") end - if ntop.isPro() and not is_alert_re_arming(network_name, mode, t[1], ifname) then - ntop.withdrawNagiosAlert(network_name, mode, t[1], "service OK") + if not is_alert_re_arming(network_name, mode, t[1], ifname) then + interface.releaseNetworkAlert(network_name, alert_id, alert_type, alert_level, "released!") + if ntop.isPro() then + ntop.withdrawNagiosAlert(network_name, mode, t[1], "service OK") + end end end end @@ -433,6 +441,10 @@ function check_interface_alert(ifname, mode, old_table, new_table) print("check_interface_alert("..ifname..", "..mode..")
    \n") end + local alert_level = 1 -- alert_level_warning + local alert_status = 1 -- alert_on + local alert_type = 2 -- alert_threshold_exceeded + -- Needed because Lua. loadstring() won't work otherwise. old = old_table new = new_table @@ -463,21 +475,19 @@ function check_interface_alert(ifname, mode, old_table, new_table) local what = "val = "..t[1].."(old, new); if(val ".. op .. " " .. t[3] .. ") then return(true) else return(false) end" local f = loadstring(what) local rc = f() + local alert_id = mode.."_"..t[1] -- the alert identifies is the concat. of time granularity and condition, e.g., min_bytes if(rc) then local alert_msg = "Threshold "..t[1].." crossed by interface "..ifname.." [".. val .." ".. op .. " " .. t[3].."]" - local alert_level = 1 -- alert_level_warning - local alert_status = 1 -- alert_on - local alert_type = 2 -- alert_threshold_exceeded if not is_alert_re_arming(ifname_clean, mode, t[1], ifname) then if verbose then io.write("queuing alert\n") end re_arm_alert(ifname_clean, mode, t[1], ifname) - interface.queueAlert(alert_level, alert_status, alert_type, alert_msg) + interface.engageInterfaceAlert(alert_id, alert_type, alert_level, alert_msg) if ntop.isPro() then -- possibly send the alert to nagios as well - ntop.sendNagiosAlert(ifname_clean, mode, t[1], alert_msg) + ntop.sendNagiosAlert(ifname_clean, mode, t[1], alert_msg) end else if verbose then io.write("alarm silenced, re-arm in progress\n") end @@ -486,8 +496,11 @@ function check_interface_alert(ifname, mode, old_table, new_table) if(verbose) then print("".. alert_msg .."
    \n") end else if(verbose) then print("

    Threshold "..t[1].."@"..ifname.." not crossed [value="..val.."]["..op.." "..t[3].."]

    \n") end - if ntop.isPro() and not is_alert_re_arming(ifname_clean, mode, t[1], ifname) then - ntop.withdrawNagiosAlert(ifname_clean, mode, t[1], "service OK") + if not is_alert_re_arming(ifname_clean, mode, t[1], ifname) then + interface.releaseInterfaceAlert(alert_id, alert_type, alert_level, "released!") + if ntop.isPro() then + ntop.withdrawNagiosAlert(ifname_clean, mode, t[1], "service OK") + end end end end diff --git a/scripts/lua/modules/graph_utils.lua b/scripts/lua/modules/graph_utils.lua index 40c0599bca..34527152b5 100644 --- a/scripts/lua/modules/graph_utils.lua +++ b/scripts/lua/modules/graph_utils.lua @@ -876,12 +876,13 @@ function createRRDcounter(path, step, verbose) if(not(ntop.exists(path))) then if(verbose) then print('Creating RRD ', path, '\n') end local prefs = ntop.getPrefs() + local hb = step * 2 -- Default hb = 2 minutes ntop.rrd_create( path, step, -- step - 'DS:sent:DERIVE:600:U:U', - 'DS:rcvd:DERIVE:600:U:U', - 'RRA:AVERAGE:0.5:1:'..tostring(prefs.other_rrd_raw_days*24*300), -- raw: 1 day = 1 * 24 = 24 * 300 sec = 7200 + 'DS:sent:DERIVE:'..hb..':U:U', + 'DS:rcvd:DERIVE:'..hb..':U:U', + 'RRA:AVERAGE:0.5:1:'..tostring(prefs.other_rrd_raw_days*24*(3600/step)), -- raw: 1 day = 1 * 24 = 24 * 12 = 288 'RRA:AVERAGE:0.5:12:'..tostring(prefs.other_rrd_1h_days*24), -- 1h resolution (12 points) 2400 hours = 100 days 'RRA:AVERAGE:0.5:288:'..tostring(prefs.other_rrd_1d_days) -- 1d resolution (288 points) 365 days --'RRA:HWPREDICT:1440:0.1:0.0035:20' @@ -895,14 +896,16 @@ function createSingleRRDcounter(path, step, verbose) if(not(ntop.exists(path))) then if(verbose) then print('Creating RRD ', path, '\n') end local prefs = ntop.getPrefs() + local hb = step * 2 -- Default hb = 2 minutes ntop.rrd_create( path, step, -- step - 'DS:num:DERIVE:600:U:U', - 'RRA:AVERAGE:0.5:1:'..tostring(prefs.other_rrd_raw_days*24*300), -- raw: 1 day = 1 * 24 = 24 * 300 sec = 7200 + 'DS:num:DERIVE:'..hb..':U:U', + 'RRA:AVERAGE:0.5:1:'..tostring(prefs.other_rrd_raw_days*24*(3600/step)), -- raw: 1 day = 1 * 24 = 24 * 12 = 288 'RRA:AVERAGE:0.5:12:'..tostring(prefs.other_rrd_1h_days*24), -- 1h resolution (12 points) 2400 hours = 100 days - 'RRA:AVERAGE:0.5:288:'..tostring(prefs.other_rrd_1d_days), -- 1d resolution (288 points) 365 days - 'RRA:HWPREDICT:1440:0.1:0.0035:20') + 'RRA:AVERAGE:0.5:288:'..tostring(prefs.other_rrd_1d_days) -- 1d resolution (288 points) 365 days + -- 'RRA:HWPREDICT:1440:0.1:0.0035:20' + ) end end @@ -912,13 +915,14 @@ function createTripleRRDcounter(path, step, verbose) if(not(ntop.exists(path))) then if(verbose) then io.write('Creating RRD '..path..'\n') end local prefs = ntop.getPrefs() + local hb = step * 2 -- Default hb = 2 minutes ntop.rrd_create( path, step, -- step - 'DS:ingress:DERIVE:600:U:U', - 'DS:egress:DERIVE:600:U:U', - 'DS:inner:DERIVE:600:U:U', - 'RRA:AVERAGE:0.5:1:'..tostring(prefs.other_rrd_raw_days*24*300), -- raw: 1 day = 1 * 24 = 24 * 300 sec = 7200 + 'DS:ingress:DERIVE:'..hb..':U:U', + 'DS:egress:DERIVE:'..hb..':U:U', + 'DS:inner:DERIVE:'..hb..':U:U', + 'RRA:AVERAGE:0.5:1:'..tostring(prefs.other_rrd_raw_days*24*(3600/step)), -- raw: 1 day = 1 * 24 = 24 * 12 = 288 'RRA:AVERAGE:0.5:12:'..tostring(prefs.other_rrd_1h_days*24), -- 1h resolution (12 points) 2400 hours = 100 days 'RRA:AVERAGE:0.5:288:'..tostring(prefs.other_rrd_1d_days) -- 1d resolution (288 points) 365 days --'RRA:HWPREDICT:1440:0.1:0.0035:20' @@ -932,15 +936,14 @@ function createActivityRRDCounter(path, step, verbose) if(not(ntop.exists(path))) then if(verbose) then io.write('Creating RRD '..path..'\n') end local prefs = ntop.getPrefs() - local hb = step * 2 + local hb = step * 2 -- Default hb = 2 minutes ntop.rrd_create( path, step, - 'DS:in:COUNTER:'..hb..':U:U', - 'DS:out:COUNTER:'..hb..':U:U', - 'DS:bg:COUNTER:'..hb..':U:U', - -- TODO create separate ntop prefs and decide - 'RRA:AVERAGE:0.5:1:'..tostring(prefs.other_rrd_raw_days*24*300), + 'DS:in:DERIVE:'..hb..':U:U', + 'DS:out:DERIVE:'..hb..':U:U', + 'DS:bg:DERIVE:'..hb..':U:U', + 'RRA:AVERAGE:0.5:1:'..tostring(prefs.other_rrd_raw_days*24*(3600/step)), 'RRA:AVERAGE:0.5:12:'..tostring(prefs.other_rrd_1h_days*24), 'RRA:AVERAGE:0.5:288:'..tostring(prefs.other_rrd_1d_days) ) diff --git a/scripts/lua/modules/lua_utils.lua b/scripts/lua/modules/lua_utils.lua index 099e110a8a..c00812f956 100644 --- a/scripts/lua/modules/lua_utils.lua +++ b/scripts/lua/modules/lua_utils.lua @@ -291,7 +291,8 @@ alert_entity_keys = { { "Interface", 0, "interface" }, { "Host", 1, "host" }, { "Network", 2, "network" }, - { "SNMP device", 3, "snmp_device" } + { "SNMP device", 3, "snmp_device" }, + { "Flow", 4, "flow" } } function alertSeverityLabel(v) diff --git a/scripts/lua/modules/top_scripts/top_asn.lua b/scripts/lua/modules/top_scripts/top_asn.lua index fdf9d94d75..44b7c27eff 100644 --- a/scripts/lua/modules/top_scripts/top_asn.lua +++ b/scripts/lua/modules/top_scripts/top_asn.lua @@ -6,7 +6,6 @@ dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "top_talkers" -require "json" local top_asn_intf = {} @@ -190,7 +189,7 @@ end local function getTopASFromJSON(content, add_vlan) if(content == nil) then return("[ ]\n") end - local table = parseJSON(content) + local table = json.decode(content, 1) local rsp = printTopASFromTable(table, add_vlan) if (rsp == nil or rsp == "") then return "[ ]\n" end return rsp diff --git a/scripts/lua/modules/top_scripts/top_countries.lua b/scripts/lua/modules/top_scripts/top_countries.lua index 7d1d7e1c34..f877e60af4 100644 --- a/scripts/lua/modules/top_scripts/top_countries.lua +++ b/scripts/lua/modules/top_scripts/top_countries.lua @@ -6,7 +6,6 @@ dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "top_talkers" -require "json" local top_countries_intf = {} @@ -203,7 +202,7 @@ end local function getTopCountriesFromJSON(content, add_vlan) if(content == nil) then return("[ ]\n") end - local table = parseJSON(content) + local table = json.decode(content, 1) local rsp = printTopCountriesFromTable(table, add_vlan) if (rsp == nil or rsp == "") then return "[ ]\n" end return rsp diff --git a/scripts/lua/modules/top_scripts/top_networks.lua b/scripts/lua/modules/top_scripts/top_networks.lua index bcc510049e..a9e8637b88 100644 --- a/scripts/lua/modules/top_scripts/top_networks.lua +++ b/scripts/lua/modules/top_scripts/top_networks.lua @@ -6,7 +6,6 @@ dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "top_talkers" -require "json" local top_networks_intf = {} @@ -193,7 +192,7 @@ end local function getTopNetworksFromJSON(content, add_vlan) if(content == nil) then return("[ ]\n") end - local table = parseJSON(content) + local table = json.decode(content, 1) local rsp = printTopNetworksFromTable(table, add_vlan) if (rsp == nil or rsp == "") then return "[ ]\n" end return rsp diff --git a/scripts/lua/modules/top_scripts/top_os.lua b/scripts/lua/modules/top_scripts/top_os.lua index c8701ada35..48ae692ff1 100644 --- a/scripts/lua/modules/top_scripts/top_os.lua +++ b/scripts/lua/modules/top_scripts/top_os.lua @@ -6,7 +6,6 @@ dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "top_talkers" -require "json" local top_os_intf = {} @@ -193,7 +192,7 @@ end local function getTopOSFromJSON(content, add_vlan) if(content == nil) then return("[ ]\n") end - local table = parseJSON(content) + local table = json.decode(content, 1) local rsp = printTopOSFromTable(table, add_vlan) if (rsp == nil or rsp == "") then return "[ ]\n" end return rsp diff --git a/scripts/lua/modules/top_scripts/top_os_local.lua b/scripts/lua/modules/top_scripts/top_os_local.lua index 3508efddbe..4c590be322 100644 --- a/scripts/lua/modules/top_scripts/top_os_local.lua +++ b/scripts/lua/modules/top_scripts/top_os_local.lua @@ -6,7 +6,6 @@ dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "top_talkers" -require "json" local top_os_local_intf = {} @@ -193,7 +192,7 @@ end local function getTopOSFromJSON(content, add_vlan) if(content == nil) then return("[ ]\n") end - local table = parseJSON(content) + local table = json.decode(content, 1) local rsp = printTopOSLocalFromTable(table, add_vlan) if (rsp == nil or rsp == "") then return "[ ]\n" end return rsp diff --git a/scripts/lua/modules/top_scripts/top_talkers.lua b/scripts/lua/modules/top_scripts/top_talkers.lua index daadf6fb1f..2a2fb784e4 100644 --- a/scripts/lua/modules/top_scripts/top_talkers.lua +++ b/scripts/lua/modules/top_scripts/top_talkers.lua @@ -6,7 +6,6 @@ dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "top_talkers" -require "json" local top_talkers_intf = {} @@ -187,7 +186,7 @@ end local function getTopTalkersFromJSON(content, add_vlan) if(content == nil) then return("[ ]\n") end - local table = parseJSON(content) + local table = json.decode(content, 1) local rsp = printTopTalkersFromTable(table, add_vlan) if(rsp == nil or rsp == "") then return "[ ]\n" end return rsp @@ -227,7 +226,7 @@ local function getHistoricalTopTalkersInInterval(ifid, ifname, epoch_start, epoc --]] local res = {["senders"] = {}, ["receivers"] = {}} for record,_ in pairs(query) do - record = parseJSON(record) + record = json.decode(record, 1) if not record or not next(record) or not record["vlan"] then goto next_record end -- tprint(record) for _, vlan in pairs(record["vlan"]) do diff --git a/scripts/lua/modules/top_scripts/top_vlan.lua b/scripts/lua/modules/top_scripts/top_vlan.lua index d37fb5d27a..eecc6b48e1 100644 --- a/scripts/lua/modules/top_scripts/top_vlan.lua +++ b/scripts/lua/modules/top_scripts/top_vlan.lua @@ -7,7 +7,6 @@ package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "top_talkers" require "top_structure" -require "json" local top_vlan_intf = {} @@ -43,7 +42,7 @@ end local function getTopVLANFromJSON(content, add_vlan) if(content == nil) then return("[ ]\n") end - local table = parseJSON(content) + local table = json.decode(content, 1) if (table == nil or table[top_vlan_intf.JSONkey] == nil) then return "[ ]\n" end local nr_elements = 0 diff --git a/scripts/lua/network_details.lua b/scripts/lua/network_details.lua index d97bb0250d..c9418a43b6 100644 --- a/scripts/lua/network_details.lua +++ b/scripts/lua/network_details.lua @@ -258,6 +258,7 @@ elseif(page == "alerts") then end end if(to_save) then + refresh_alert_configuration(network_name, ifname, tab, alerts) if(alerts == "") then ntop.delHashCache(get_alerts_hash_name(tab, ifname), network_name) else diff --git a/scripts/lua/network_load.lua b/scripts/lua/network_load.lua index 6305d402f9..c6d028845e 100644 --- a/scripts/lua/network_load.lua +++ b/scripts/lua/network_load.lua @@ -33,11 +33,14 @@ function dumpInterfaceStats(interface_name) if prefs.is_dump_flows_to_es_enabled == true or prefs.is_dump_flows_to_mysql_enabled == true then - res["flow_export_drops"] = ifstats.stats.flow_export_drops + res["flow_export_drops"] = ifstats.stats.flow_export_drops + res["flow_export_rate"] = ifstats.stats.flow_export_rate + res["flow_export_count"] = ifstats.stats.flow_export_count end if prefs.are_alerts_enabled == true then - res["alerts"] = interface.getNumQueuedAlerts() + res["alerts"] = interface.getNumAlerts(false--[[ not engaged --]]) + res["engaged_alerts"] = interface.getNumAlerts(true --[[ engaged --]]) end diff --git a/scripts/lua/show_alerts.lua b/scripts/lua/show_alerts.lua index 4283aeee59..32e4b7cbaf 100644 --- a/scripts/lua/show_alerts.lua +++ b/scripts/lua/show_alerts.lua @@ -15,10 +15,18 @@ ntop.dumpFile(dirs.installdir .. "/httpdocs/inc/header.inc") if(_GET["csrf"] ~= nil) then if(_GET["id_to_delete"] ~= nil) then if(_GET["id_to_delete"] == "__all__") then - interface.flushAllQueuedAlerts() + interface.deleteAlerts(true --[[ engaged --]]) + interface.deleteAlerts(false --[[ and not engaged --]]) print("") else - interface.deleteQueuedAlert(tonumber(_GET["id_to_delete"])) + local id_to_delete = tonumber(_GET["id_to_delete"]) + if id_to_delete ~= nil then + if _GET["engaged"] == "true" then + interface.deleteAlerts(true, id_to_delete) + else + interface.deleteAlerts(false, id_to_delete) + end + end end end end @@ -26,19 +34,33 @@ end active_page = "alerts" dofile(dirs.installdir .. "/scripts/lua/inc/menu.lua") +local num_alerts = interface.getNumAlerts(false --[[ NOT engaged --]]) +local num_engaged_alerts = interface.getNumAlerts(true --[[ engaged --]]) if ntop.getPrefs().are_alerts_enabled == false then print("

    Alerts are disabled. Please check the preferences page to enable them.
    ") --return +elseif num_alerts == 0 and num_engaged_alerts == 0 then + print("
    No recorded alerts so far for interface "..ifname.."
    ") else -print [[ -
    -
    +local alert_items = {} + +if num_engaged_alerts > 0 then + alert_items[#alert_items + 1] = {["label"] = "Currently Engaged Alerts", ["div-id"] = "table-engaged-alerts", ["status"] = "engaged", ["date"] = "First Seen"} +end + +if num_alerts > 0 then + alert_items[#alert_items +1] = {["label"] = "Alerts History", ["div-id"] = "table-alerts-history", ["status"] = "historical", ["date"] = "Time"} +end + +for k, t in ipairs(alert_items) do + print [[ +
    ]] +end -if(interface.getNumQueuedAlerts() > 0) then +if num_alerts > 0 or num_engaged_alerts > 0 then print [[ Purge All Alerts diff --git a/scripts/lua/top_generic.lua b/scripts/lua/top_generic.lua index 3666407dfe..f6cb4cdbeb 100644 --- a/scripts/lua/top_generic.lua +++ b/scripts/lua/top_generic.lua @@ -6,6 +6,7 @@ dirs = ntop.getDirs() package.path = dirs.installdir .. "/scripts/lua/modules/?.lua;" .. package.path require "lua_utils" require "top_talkers" +json = require("dkjson") sendHTTPHeader('text/html; charset=iso-8859-1') diff --git a/src/ActivityFilters.cpp b/src/ActivityFilters.cpp index 1d901460e2..c1a7f1ea08 100644 --- a/src/ActivityFilters.cpp +++ b/src/ActivityFilters.cpp @@ -286,7 +286,7 @@ static bool activity_filter_fun_ratio(const activity_filter_config * config, if (! d) status->ratio.detected = true; else - status->ratio.detected = n + d >= config->ratio.minbytes && r >= abs(config->ratio.clisrv_ratio); + status->ratio.detected = n + d >= config->ratio.minbytes && r >= fabsf(config->ratio.clisrv_ratio); char buf[32]; ntop->getTrace()->traceEvent(TRACE_DEBUG, "%c Ratio filter[%s] url/cert='%s%s' cli=%lu srv=%lu : %.3f\n", diff --git a/src/AlertsManager.cpp b/src/AlertsManager.cpp index 5e2c2d4076..27e9f2348c 100644 --- a/src/AlertsManager.cpp +++ b/src/AlertsManager.cpp @@ -66,6 +66,7 @@ int AlertsManager::openStore() { snprintf(create_query, sizeof(create_query), "CREATE TABLE IF NOT EXISTS %s (" "alert_tstamp INTEGER NOT NULL, " + "alert_tstamp_end INTEGER DEFAULT NULL, " "alert_type INTEGER NOT NULL, " "alert_severity INTEGER NOT NULL, " "alert_entity INTEGER NOT NULL, " @@ -73,11 +74,12 @@ int AlertsManager::openStore() { "alert_json TEXT DEFAULT NULL " "); " // no need to create a primary key, sqlite has the rowid "CREATE INDEX IF NOT EXISTS t1i_tstamp ON %s(alert_tstamp); " + "CREATE INDEX IF NOT EXISTS t1i_tstamp_e ON %s(alert_tstamp_end); " "CREATE INDEX IF NOT EXISTS t1i_type ON %s(alert_type); " "CREATE INDEX IF NOT EXISTS t1i_severity ON %s(alert_severity); " "CREATE INDEX IF NOT EXISTS t1i_entity ON %s(alert_entity, alert_entity_val); ", ALERTS_MANAGER_TABLE_NAME, ALERTS_MANAGER_TABLE_NAME, ALERTS_MANAGER_TABLE_NAME, - ALERTS_MANAGER_TABLE_NAME, ALERTS_MANAGER_TABLE_NAME); + ALERTS_MANAGER_TABLE_NAME, ALERTS_MANAGER_TABLE_NAME, ALERTS_MANAGER_TABLE_NAME); m.lock(__FILE__, __LINE__); rc = exec_query(create_query, NULL, NULL); m.unlock(__FILE__, __LINE__); @@ -106,7 +108,7 @@ int AlertsManager::openStore() { } /* **************************************************** */ - +#ifdef NOTUSED int AlertsManager::storeAlert(AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { char query[STORE_MANAGER_MAX_QUERY]; sqlite3_stmt *stmt = NULL; @@ -242,7 +244,7 @@ int AlertsManager::storeAlert(lua_State *L, int index) { return retval; }; - +#endif /* **************************************************** */ bool AlertsManager::isAlertEngaged(AlertEntity alert_entity, const char *alert_entity_value, const char *engaged_alert_id) { @@ -358,9 +360,9 @@ int AlertsManager::releaseAlert(AlertEntity alert_entity, const char *alert_enti /* move the alert from engaged to closed */ snprintf(query, sizeof(query), "INSERT INTO %s " - "(alert_tstamp, alert_type, alert_severity, alert_entity, alert_entity_val, alert_json) " + "(alert_tstamp, alert_tstamp_end, alert_type, alert_severity, alert_entity, alert_entity_val, alert_json) " "SELECT " - "alert_tstamp, alert_type, alert_severity, alert_entity, alert_entity_val, alert_json " + "alert_tstamp, strftime('%%s','now'), alert_type, alert_severity, alert_entity, alert_entity_val, alert_json " "FROM %s " "WHERE alert_entity = ? AND alert_entity_val = ? AND alert_id = ? " "LIMIT 1;" /* limit not even needed as the where clause yields unique tuples */, @@ -421,6 +423,54 @@ int AlertsManager::releaseAlert(AlertEntity alert_entity, const char *alert_enti return rc; } +/* **************************************************** */ + +int AlertsManager::storeAlert(AlertEntity alert_entity, const char *alert_entity_value, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { + char query[STORE_MANAGER_MAX_QUERY]; + sqlite3_stmt *stmt = NULL; + int rc = 0; + + if(!store_initialized || !store_opened) + return -1; + + + /* This alert is being engaged */ + snprintf(query, sizeof(query), + "INSERT INTO %s " + "(alert_tstamp, alert_type, alert_severity, alert_entity, alert_entity_val, alert_json) " + "VALUES (?, ?, ?, ?, ?, ?); ", + ALERTS_MANAGER_TABLE_NAME); + + m.lock(__FILE__, __LINE__); + + if(sqlite3_prepare(db, query, -1, &stmt, 0) + || sqlite3_bind_int64(stmt, 1, static_cast(time(NULL))) + || sqlite3_bind_int(stmt, 2, static_cast(alert_type)) + || sqlite3_bind_int(stmt, 3, static_cast(alert_severity)) + || sqlite3_bind_int(stmt, 4, static_cast(alert_entity)) + || sqlite3_bind_text(stmt, 5, alert_entity_value, strlen(alert_entity_value), SQLITE_TRANSIENT) + || sqlite3_bind_text(stmt, 6, alert_json, strlen(alert_json), SQLITE_TRANSIENT)) { + rc = 1; + goto out; + } + + while((rc = sqlite3_step(stmt)) != SQLITE_DONE) { + if(rc == SQLITE_ERROR) { + ntop->getTrace()->traceEvent(TRACE_INFO, "SQL Error: step"); + rc = 1; + goto out; + } + } + + rc = 0; + out: + if (stmt) sqlite3_finalize(stmt); + m.unlock(__FILE__, __LINE__); + + return rc; +} + /* ******************************************* */ int AlertsManager::engageReleaseHostAlert(Host *h, @@ -442,6 +492,204 @@ int AlertsManager::engageReleaseHostAlert(Host *h, /* ******************************************* */ +int AlertsManager::engageReleaseNetworkAlert(const char *cidr, + const char *engaged_alert_id, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json, + bool engage) { + struct in_addr addr4; + struct in6_addr addr6; + char ip_buf[256]; + char *slash; + + if(!cidr) return -1; + + strncpy(ip_buf, cidr, sizeof(ip_buf)); + if ((slash = strchr(ip_buf, '/')) == NULL) return -2; + slash[0] = '\0'; + + if(inet_pton(AF_INET, ip_buf, &addr4) != 1 && inet_pton(AF_INET6, ip_buf, &addr6) != 1) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Error parsing network %s\n", cidr); + return -2; /* not a valid network */ + } + + if (engage) + return engageAlert(alert_entity_network, cidr, + engaged_alert_id, alert_type, alert_severity, alert_json); + else + return releaseAlert(alert_entity_network, cidr, + engaged_alert_id, alert_type, alert_severity, alert_json); +}; + +/* ******************************************* */ + +int AlertsManager::engageReleaseInterfaceAlert(NetworkInterface *n, + const char *engaged_alert_id, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json, + bool engage) { + char id_buf[8]; + if(!n) return -1; + + snprintf(id_buf, sizeof(id_buf), "%u", n -> get_id()); + + if (engage) + return engageAlert(alert_entity_interface, id_buf, + engaged_alert_id, alert_type, alert_severity, alert_json); + else + return releaseAlert(alert_entity_interface, id_buf, + engaged_alert_id, alert_type, alert_severity, alert_json); +}; + +/* ******************************************* */ + +int AlertsManager::storeHostAlert(Host *h, + AlertType alert_type, AlertLevel alert_severity, const char *alert_json) { + if (!h) return -1; + char ipbuf[256], ipbuf_id[256]; + IpAddress *ip = h->get_ip(); + if(!ip) return -2; + snprintf(ipbuf_id, sizeof(ipbuf_id), "%s@%i", ip->print(ipbuf, sizeof(ipbuf)), h->get_vlan_id()); + return storeAlert(alert_entity_host, ipbuf_id, alert_type, alert_severity, alert_json); +}; + +/* ******************************************* */ +struct alertsRetriever { + lua_State *vm; + u_int32_t current_offset; +}; + +static int getAlertsCallback(void *data, int argc, char **argv, char **azColName){ + alertsRetriever *ar = (alertsRetriever*)data; + lua_State *vm = ar->vm; + + lua_newtable(vm); + + for(int i = 0; i < argc; i++){ + lua_push_str_table_entry(vm, azColName[i], argv[i]); + } + + lua_pushnumber(vm, ++ar->current_offset); + lua_insert(vm, -2); + lua_settable(vm, -3); + + return 0; +} + +/* ******************************************* */ + +int AlertsManager::getAlerts(lua_State* vm, patricia_tree_t *allowed_hosts, + u_int32_t start_offset, u_int32_t end_offset, + bool engaged) { + alertsRetriever ar; + char query[STORE_MANAGER_MAX_QUERY]; + char *zErrMsg = 0; + int rc = 0; + + if(!store_initialized || !store_opened) + return -1; + + snprintf(query, sizeof(query), + "SELECT rowid, alert_tstamp, alert_type, alert_severity, alert_entity, alert_entity_val, alert_json %s " + "FROM %s ORDER BY alert_tstamp DESC LIMIT %u,%u", + engaged ? "" : ", alert_tstamp_end ", + engaged ? ALERTS_MANAGER_ENGAGED_TABLE_NAME : ALERTS_MANAGER_TABLE_NAME, + start_offset, end_offset - start_offset + 1); + + m.lock(__FILE__, __LINE__); + + lua_newtable(vm); + + ar.vm = vm, ar.current_offset = 0; + rc = sqlite3_exec(db, query, getAlertsCallback, (void*)&ar, &zErrMsg); + + if( rc != SQLITE_OK ){ + rc = 1; + ntop->getTrace()->traceEvent(TRACE_ERROR, "SQL Error: %s\n%s", zErrMsg, query); + sqlite3_free(zErrMsg); + goto out; + } + + rc = 0; + out: + m.unlock(__FILE__, __LINE__); + + return rc; +} + +/* **************************************************** */ + +int AlertsManager::getNumAlerts(bool engaged) { + char query[STORE_MANAGER_MAX_QUERY]; + sqlite3_stmt *stmt = NULL; + int rc; + int num = -1; + + snprintf(query, sizeof(query), + "SELECT count(*) FROM %s ", + engaged ? ALERTS_MANAGER_ENGAGED_TABLE_NAME : ALERTS_MANAGER_TABLE_NAME); + + m.lock(__FILE__, __LINE__); + if(sqlite3_prepare(db, query, -1, &stmt, 0)) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to prepare statement for query %s.", query); + goto out; + } + + while((rc = sqlite3_step(stmt)) != SQLITE_DONE) { + if (rc == SQLITE_ROW) { + num = sqlite3_column_int(stmt, 0); + // ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s\n", sqlite3_column_text(stmt, 0)); + } else if(rc == SQLITE_ERROR) { + ntop->getTrace()->traceEvent(TRACE_INFO, "SQL Error: step"); + goto out; + } + } + + out: + if (stmt) sqlite3_finalize(stmt); + m.unlock(__FILE__, __LINE__); + + return num; +} + +/* **************************************************** */ + +int AlertsManager::deleteAlerts(bool engaged, const int *rowid) { + char query[STORE_MANAGER_MAX_QUERY]; + sqlite3_stmt *stmt = NULL; + int rc; + + snprintf(query, sizeof(query), + "DELETE FROM %s %s ", + engaged ? ALERTS_MANAGER_ENGAGED_TABLE_NAME : ALERTS_MANAGER_TABLE_NAME, + rowid ? "WHERE rowid = ?" : ""); + + m.lock(__FILE__, __LINE__); + if(sqlite3_prepare(db, query, -1, &stmt, 0)) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to prepare statement for query %s.", query); + rc = -1; + goto out; + } else if(rowid && sqlite3_bind_int(stmt, 1, *rowid)) { + ntop->getTrace()->traceEvent(TRACE_ERROR, "Unable to bind values to prepared statement for query %s.", query); + rc = -2; + goto out; + } + + while((rc = sqlite3_step(stmt)) != SQLITE_DONE) { + if(rc == SQLITE_ERROR) { + ntop->getTrace()->traceEvent(TRACE_INFO, "SQL Error: step"); + goto out; + } + } + + rc = 0; + out: + if (stmt) sqlite3_finalize(stmt); + m.unlock(__FILE__, __LINE__); + + return rc; +} + +/* ******************************************* */ +#ifdef NOTUSED int AlertsManager::queueAlert(AlertLevel level, AlertStatus s, AlertType t, char *msg) { char what[1024]; @@ -484,7 +732,7 @@ int AlertsManager::getQueuedAlerts(lua_State* vm, patricia_tree_t *allowed_hosts for(int i = 0; i < rc; i++) { lua_pushstring(vm, l_elements[i] ? l_elements[i] : ""); - lua_rawseti(vm, -2, i); + lua_rawseti(vm, -2, i+1); if(l_elements[i]) free(l_elements[i]); } free(l_elements); @@ -519,3 +767,4 @@ int AlertsManager::flushAllQueuedAlerts() { return redis->delKey(queue_name); } +#endif diff --git a/src/ElasticSearch.cpp b/src/ElasticSearch.cpp index 4bdbe1767b..9e687bb337 100644 --- a/src/ElasticSearch.cpp +++ b/src/ElasticSearch.cpp @@ -38,6 +38,9 @@ ElasticSearch::ElasticSearch() { tail = NULL; reportDrops = false; elkDroppedFlowsQueueTooLong = 0; + elkExportedFlows = 0, elkLastExportedFlows = 0; + elkExportRate = 0; + lastUpdateTime.tv_sec = 0, lastUpdateTime.tv_usec = 0; } /* **************************************** */ @@ -46,6 +49,33 @@ ElasticSearch::~ElasticSearch() { } +/* ******************************************* */ + +void ElasticSearch::updateStats(const struct timeval *tv) { + if(tv == NULL) return; + + if(lastUpdateTime.tv_sec > 0) { + float tdiffMsec = ((float)(tv->tv_sec-lastUpdateTime.tv_sec)*1000)+((tv->tv_usec-lastUpdateTime.tv_usec)/(float)1000); + if(tdiffMsec >= 1000) { /* al least one second */ + u_int64_t diffFlows = elkExportedFlows - elkLastExportedFlows; + elkLastExportedFlows = elkExportedFlows; + + elkExportRate = ((float)(diffFlows * 1000)) / tdiffMsec; + if (elkExportRate < 0) elkExportRate = 0; + } + } + + memcpy(&lastUpdateTime, tv, sizeof(struct timeval)); +} + +/* ******************************************* */ + +void ElasticSearch::lua(lua_State *vm) const { + lua_push_int_table_entry(vm, "flow_export_count", elkExportedFlows); + lua_push_int32_table_entry(vm, "flow_export_drops", elkDroppedFlowsQueueTooLong); + lua_push_float_table_entry(vm, "flow_export_rate", elkExportRate); +} + /* **************************************** */ int ElasticSearch::sendToES(char* msg) { @@ -150,8 +180,10 @@ void ElasticSearch::indexESdata() { postbuf)) { /* Post failure */ sleep(1); - } else + } else { ntop->getTrace()->traceEvent(TRACE_INFO, "Sent %u flow(s) to ES", num_flows); + elkExportedFlows += num_flows; + } } else sleep(1); } /* while */ diff --git a/src/Flow.cpp b/src/Flow.cpp index 5a4c664d77..51358b61a4 100644 --- a/src/Flow.cpp +++ b/src/Flow.cpp @@ -232,8 +232,8 @@ Flow::~Flow() { s, iface->get_name(), srv_host->get_name() ? srv_host->get_name() : s, print(fbuf, sizeof(fbuf))); - iface->getAlertsManager()->queueAlert(alert_level_warning, alert_permanent, - alert_suspicious_activity, alert_msg); + iface->getAlertsManager()->storeFlowAlert(this, alert_suspicious_activity, + alert_level_warning, alert_msg); break; default: @@ -301,7 +301,8 @@ void Flow::checkBlacklistedFlow() { s, iface->get_name(), srv_host->get_name() ? srv_host->get_name() : s, print(fbuf, sizeof(fbuf))); - iface->getAlertsManager()->queueAlert(alert_level_warning, alert_permanent, alert_dangerous_host, alert_msg); + iface->getAlertsManager()->storeFlowAlert(this, alert_dangerous_host, + alert_level_warning, alert_msg); } blacklist_alarm_emitted = true; @@ -499,10 +500,11 @@ void Flow::guessProtocol() { if((protocol == IPPROTO_TCP) || (protocol == IPPROTO_UDP)) { if(cli_host && srv_host) { /* We can guess the protocol */ + IpAddress *cli_ip = cli_host->get_ip(), *srv_ip = srv_host->get_ip(); ndpiDetectedProtocol = ndpi_guess_undetected_protocol(iface->get_ndpi_struct(), protocol, - ntohl(cli_host->get_ip()->get_ipv4()), + ntohl(cli_ip ? cli_ip->get_ipv4() : 0), ntohs(cli_port), - ntohl(srv_host->get_ip()->get_ipv4()), + ntohl(srv_ip ? srv_ip->get_ipv4() : 0), ntohs(srv_port)); } @@ -2309,7 +2311,6 @@ void Flow::dissectHTTP(bool src2dst_direction, char *payload, u_int16_t payload_ if (protos.http.last_content_type) free(protos.http.last_content_type); protos.http.last_content_type = strdup(ct); - iface->luaEvalFlow(this, callback_flow_update); break; } } @@ -2370,7 +2371,8 @@ void Flow::checkFlowCategory() { s, iface->get_name(), s, srv_port, host_server_name, host_server_name); - iface->getAlertsManager()->queueAlert(alert_level_warning, alert_malware_detection, alert_msg); + iface->getAlertsManager()->storeFlowAlert(this, alert_malware_detection, + alert_level_warning, alert_msg); badFlow = true, setDropVerdict(); } #endif diff --git a/src/GenericHost.cpp b/src/GenericHost.cpp index 24f80b03d3..f2124e977b 100644 --- a/src/GenericHost.cpp +++ b/src/GenericHost.cpp @@ -56,8 +56,8 @@ void GenericHost::readStats() { host_key = get_string_key(buf, sizeof(buf)); strftime(daybuf, sizeof(daybuf), "%y/%m/%d", localtime(&when)); - snprintf(dump_path, sizeof(dump_path), "%s/%d/activities/%s/%s", - ntop->get_working_dir(), iface->get_id(), daybuf, host_key); + snprintf(dump_path, sizeof(dump_path), "%s/%d/activities/%s/%s@%u", + ntop->get_working_dir(), iface->get_id(), daybuf, host_key, vlan_id); ntop->fixPath(dump_path); if(activityStats.readDump(dump_path)) @@ -84,8 +84,8 @@ void GenericHost::dumpStats(bool forceDump) { ntop->fixPath(dump_path); Utils::mkdir_tree(dump_path); - snprintf(dump_path, sizeof(dump_path), "%s/%d/activities/%s/%s", - ntop->get_working_dir(), iface->get_id(), daybuf, host_key); + snprintf(dump_path, sizeof(dump_path), "%s/%d/activities/%s/%s@%u", + ntop->get_working_dir(), iface->get_id(), daybuf, host_key, vlan_id); ntop->fixPath(dump_path); activityStats.writeDump(dump_path); ntop->getTrace()->traceEvent(TRACE_INFO, "Dumping %s", dump_path); diff --git a/src/Host.cpp b/src/Host.cpp index af88814297..8bc06a53e3 100644 --- a/src/Host.cpp +++ b/src/Host.cpp @@ -30,9 +30,10 @@ Host::Host(NetworkInterface *_iface) : GenericHost(_iface) { /* *************************************** */ -Host::Host(NetworkInterface *_iface, char *ipAddress) : GenericHost(_iface) { +Host::Host(NetworkInterface *_iface, char *ipAddress, u_int16_t _vlanId) : GenericHost(_iface) { ip = new IpAddress(ipAddress); - initialize(NULL, 0, false); + if (ip && ip->isEmpty()) free(ip), ip = NULL; + initialize(NULL, _vlanId, true); } /* *************************************** */ @@ -69,13 +70,14 @@ Host::~Host() { // ntop->getTrace()->traceEvent(TRACE_NORMAL, "Deleting %s (%s)", k, localHost ? "local": "remote"); if((localHost || systemHost) - && ntop->getPrefs()->is_host_persistency_enabled()) { + && ntop->getPrefs()->is_idle_local_host_cache_enabled() + && ip && !ip->isEmpty()) { char *json = serialize(); char host_key[128], key[128]; char *k = get_string_key(host_key, sizeof(host_key)); - snprintf(key, sizeof(key), "%s.%d.json", k, vlan_id); - ntop->getRedis()->set(key, json, LOCAL_HOSTS_CACHE_DURATION); + snprintf(key, sizeof(key), HOST_SERIALIZED_KEY, iface->get_id(), k, vlan_id); + ntop->getRedis()->set(key, json, ntop->getPrefs()->get_local_host_cache_duration()); ntop->getTrace()->traceEvent(TRACE_INFO, "Dumping serialization %s", k); //ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s => %s", k, json); free(json); @@ -149,6 +151,7 @@ void Host::initialize(u_int8_t mac[6], u_int16_t _vlanId, bool init_all) { networkStats = NULL, local_network_id = -1, nextResolveAttempt = 0; syn_flood_attacker_alert = new AlertCounter(max_num_syn_sec_threshold, CONST_MAX_THRESHOLD_CROSS_DURATION); syn_flood_victim_alert = new AlertCounter(max_num_syn_sec_threshold, CONST_MAX_THRESHOLD_CROSS_DURATION); + flow_flood_attacker_alert = flow_flood_victim_alert = false; os[0] = '\0', trafficCategory[0] = '\0', blacklisted_host = false; num_uses = 0, symbolic_name = NULL, vlan_id = _vlanId, ingress_shaper_id = egress_shaper_id = DEFAULT_SHAPER_ID, @@ -164,7 +167,7 @@ void Host::initialize(u_int8_t mac[6], u_int16_t _vlanId, bool init_all) { asn = 0, asname = NULL, country = NULL, city = NULL; longitude = 0, latitude = 0, host_quota_mb = 0; k = get_string_key(key, sizeof(key)); - snprintf(redis_key, sizeof(redis_key), "%s.%d.json", k, vlan_id); + snprintf(redis_key, sizeof(redis_key), HOST_SERIALIZED_KEY, iface->get_id(), k, vlan_id); dns = NULL, http = NULL, categoryStats = NULL, topSitesKey = NULL; #ifdef NTOPNG_PRO @@ -199,7 +202,7 @@ void Host::initialize(u_int8_t mac[6], u_int16_t _vlanId, bool init_all) { } if((localHost || systemHost) - && ntop->getPrefs()->is_host_persistency_enabled()){ + && ntop->getPrefs()->is_idle_local_host_cache_enabled()){ char *json; if((json = (char*)malloc(HOST_MAX_SERIALIZED_LEN * sizeof(char))) == NULL) ntop->getTrace()->traceEvent(TRACE_ERROR, @@ -261,10 +264,10 @@ void Host::initialize(u_int8_t mac[6], u_int16_t _vlanId, bool init_all) { #endif readStats(); } - - readAlertPrefs(); } + loadAlertPrefs(); + readAlertPrefs(); if(!host_serial) computeHostSerial(); updateHostL7Policy(); } @@ -541,8 +544,8 @@ void Host::lua(lua_State* vm, patricia_tree_t *ptree, lua_push_bool_table_entry(vm, "tcp.packets.seq_problems", (tcpPacketStats.pktRetr - | tcpPacketStats.pktOOO - | tcpPacketStats.pktLost) ? true : false); + || tcpPacketStats.pktOOO + || tcpPacketStats.pktLost) ? true : false); lua_push_int_table_entry(vm, "tcp.packets.retransmissions", tcpPacketStats.pktRetr); lua_push_int_table_entry(vm, "tcp.packets.out_of_order", tcpPacketStats.pktOOO); lua_push_int_table_entry(vm, "tcp.packets.lost", tcpPacketStats.pktLost); @@ -871,19 +874,26 @@ json_object* Host::getJSONObject() { if((my_object = json_object_new_object()) == NULL) return(NULL); json_object_object_add(my_object, "mac_address", json_object_new_string(get_mac(buf, sizeof(buf), mac_address))); - json_object_object_add(my_object, "seen.last", json_object_new_int(last_seen)); + json_object_object_add(my_object, "seen.first", json_object_new_int64(first_seen)); + json_object_object_add(my_object, "seen.last", json_object_new_int64(last_seen)); json_object_object_add(my_object, "asn", json_object_new_int(asn)); if(symbolic_name) json_object_object_add(my_object, "symbolic_name", json_object_new_string(symbolic_name)); if(country) json_object_object_add(my_object, "country", json_object_new_string(country)); if(city) json_object_object_add(my_object, "city", json_object_new_string(city)); if(asname) json_object_object_add(my_object, "asname", json_object_new_string(asname)); + if(strlen(os)) json_object_object_add(my_object, "os", json_object_new_string(os)); if(trafficCategory[0] != '\0') json_object_object_add(my_object, "trafficCategory", json_object_new_string(trafficCategory)); if(vlan_id != 0) json_object_object_add(my_object, "vlan_id", json_object_new_int(vlan_id)); if(latitude) json_object_object_add(my_object, "latitude", json_object_new_double(latitude)); if(longitude) json_object_object_add(my_object, "longitude", json_object_new_double(longitude)); if(ip) json_object_object_add(my_object, "ip", ip->getJSONObject()); + if(deviceIfIdx) json_object_object_add(my_object, "device_if_idx", json_object_new_int(deviceIfIdx)); + if(deviceIP) json_object_object_add(my_object, "device_ip", json_object_new_int(deviceIP)); json_object_object_add(my_object, "localHost", json_object_new_boolean(localHost)); json_object_object_add(my_object, "systemHost", json_object_new_boolean(systemHost)); + json_object_object_add(my_object, "is_blacklisted", json_object_new_boolean(blacklisted_host)); + json_object_object_add(my_object, "flow_flood_attacker_alert", json_object_new_boolean(flow_flood_attacker_alert)); + json_object_object_add(my_object, "flow_flood_victim_alert", json_object_new_boolean(flow_flood_victim_alert)); json_object_object_add(my_object, "tcp_sent", tcp_sent.getJSONObject()); json_object_object_add(my_object, "tcp_rcvd", tcp_rcvd.getJSONObject()); json_object_object_add(my_object, "udp_sent", udp_sent.getJSONObject()); @@ -892,8 +902,23 @@ json_object* Host::getJSONObject() { json_object_object_add(my_object, "icmp_rcvd", icmp_rcvd.getJSONObject()); json_object_object_add(my_object, "other_ip_sent", other_ip_sent.getJSONObject()); json_object_object_add(my_object, "other_ip_rcvd", other_ip_rcvd.getJSONObject()); + + /* packet stats */ json_object_object_add(my_object, "pktStats.sent", sent_stats.getJSONObject()); json_object_object_add(my_object, "pktStats.recv", recv_stats.getJSONObject()); + + /* TCP packet stats (serialize only anomalies) */ + if(tcpPacketStats.pktRetr) json_object_object_add(my_object, + "tcpPacketStats.pktRetr", + json_object_new_int(tcpPacketStats.pktRetr)); + if(tcpPacketStats.pktOOO) json_object_object_add(my_object, + "tcpPacketStats.pktOOO", + json_object_new_int(tcpPacketStats.pktOOO)); + if(tcpPacketStats.pktLost) json_object_object_add(my_object, + "tcpPacketStats.pktLost", + json_object_new_int(tcpPacketStats.pktLost)); + + /* throughput stats */ json_object_object_add(my_object, "throughput_bps", json_object_new_double(bytes_thpt)); json_object_object_add(my_object, "throughput_trend_bps", json_object_new_string(Utils::trend2str(bytes_thpt_trend))); json_object_object_add(my_object, "throughput_pps", json_object_new_double(pkts_thpt)); @@ -962,6 +987,9 @@ bool Host::deserialize(char *json_str, char *key) { return(false); } + if(json_object_object_get_ex(o, "seen.first", &obj)) first_seen = json_object_get_int64(obj); + if(json_object_object_get_ex(o, "seen.last", &obj)) last_seen = json_object_get_int64(obj); + if(json_object_object_get_ex(o, "mac_address", &obj)) set_mac((char*)json_object_get_string(obj)); if(json_object_object_get_ex(o, "asn", &obj)) asn = json_object_get_int(obj); if(json_object_object_get_ex(o, "source_id", &obj)) source_id = json_object_get_int(obj); @@ -970,8 +998,11 @@ bool Host::deserialize(char *json_str, char *key) { if(json_object_object_get_ex(o, "country", &obj)) { if(country) free(country); country = strdup(json_object_get_string(obj)); } if(json_object_object_get_ex(o, "city", &obj)) { if(city) free(city); city = strdup(json_object_get_string(obj)); } if(json_object_object_get_ex(o, "asname", &obj)) { if(asname) free(asname); asname = strdup(json_object_get_string(obj)); } - if(json_object_object_get_ex(o, "trafficCategory", &obj)) { snprintf(trafficCategory, sizeof(trafficCategory), "%s", json_object_get_string(obj)); } - if(json_object_object_get_ex(o, "vlan_id", &obj)) vlan_id = json_object_get_int(obj); + if(json_object_object_get_ex(o, "os", &obj)) { snprintf(os, sizeof(os), "%s", json_object_get_string(obj)); } + if(json_object_object_get_ex(o, "trafficCategory", &obj)){ snprintf(trafficCategory, sizeof(trafficCategory), "%s", json_object_get_string(obj)); } + if(json_object_object_get_ex(o, "vlan_id", &obj)) vlan_id = json_object_get_int(obj); + if(json_object_object_get_ex(o, "device_if_idx", &obj)) deviceIfIdx = json_object_get_int(obj); + if(json_object_object_get_ex(o, "device_ip", &obj)) deviceIP = json_object_get_int(obj); if(json_object_object_get_ex(o, "latitude", &obj)) latitude = (float)json_object_get_double(obj); if(json_object_object_get_ex(o, "longitude", &obj)) longitude = (float)json_object_get_double(obj); if(json_object_object_get_ex(o, "ip", &obj)) { if(ip == NULL) ip = new IpAddress(); if(ip) ip->deserialize(obj); } @@ -985,11 +1016,25 @@ bool Host::deserialize(char *json_str, char *key) { if(json_object_object_get_ex(o, "icmp_rcvd", &obj)) icmp_rcvd.deserialize(obj); if(json_object_object_get_ex(o, "other_ip_sent", &obj)) other_ip_sent.deserialize(obj); if(json_object_object_get_ex(o, "other_ip_rcvd", &obj)) other_ip_rcvd.deserialize(obj); + + /* packet stats */ + if(json_object_object_get_ex(o, "pktStats.sent", &obj)) sent_stats.deserialize(obj); + if(json_object_object_get_ex(o, "pktStats.recv", &obj)) recv_stats.deserialize(obj); + + /* TCP packet stats */ + if(json_object_object_get_ex(o, "tcpPacketStats.pktRetr", &obj)) tcpPacketStats.pktRetr = json_object_get_int(obj); + if(json_object_object_get_ex(o, "tcpPacketStats.pktOOO", &obj)) tcpPacketStats.pktOOO = json_object_get_int(obj); + if(json_object_object_get_ex(o, "tcpPacketStats.pktLost", &obj)) tcpPacketStats.pktLost = json_object_get_int(obj); + if(json_object_object_get_ex(o, "flows.as_client", &obj)) total_num_flows_as_client = json_object_get_int(obj); if(json_object_object_get_ex(o, "flows.as_server", &obj)) total_num_flows_as_server = json_object_get_int(obj); if(json_object_object_get_ex(o, "userActivities", &obj)) user_activities.deserialize(obj); - if(json_object_object_get_ex(o, "num_alerts", &obj)) num_alerts_detected = json_object_get_int(obj); + if(json_object_object_get_ex(o, "flow_flood_attacker_alert", &obj)) flow_flood_attacker_alert = json_object_get_boolean(obj); + if(json_object_object_get_ex(o, "flow_flood_victim_alert", &obj)) flow_flood_victim_alert = json_object_get_boolean(obj); + if(json_object_object_get_ex(o, "is_blacklisted", &obj)) blacklisted_host = json_object_get_boolean(obj); + if(json_object_object_get_ex(o, "num_alerts", &obj)) num_alerts_detected = json_object_get_boolean(obj); + if(json_object_object_get_ex(o, "sent", &obj)) sent.deserialize(obj); if(json_object_object_get_ex(o, "rcvd", &obj)) rcvd.deserialize(obj); @@ -1093,7 +1138,7 @@ void Host::updateSynFlags(time_t when, u_int8_t flags, Flow *f, bool syn_sent) { } ntop->getTrace()->traceEvent(TRACE_INFO, "SYN Flood: %s", msg); - iface->getAlertsManager()->queueAlert(alert_level_error, alert_on, alert_syn_flood, msg); + iface->getAlertsManager()->storeHostAlert(this, alert_syn_flood, alert_level_error, msg); incNumAlerts(); } } @@ -1104,36 +1149,42 @@ void Host::incNumFlows(bool as_client) { if(as_client) { total_num_flows_as_client++, num_active_flows_as_client++; - if(num_active_flows_as_client == max_num_active_flows && localHost && triggerAlerts()) { - const char* error_msg = "Host %s is a possible scanner [%u active flows]"; + if(num_active_flows_as_client >= max_num_active_flows && localHost && triggerAlerts() && !flow_flood_attacker_alert) { + const char* error_msg = "Host %s is a possible scanner [%u active flows exceeded]"; char ip_buf[48], *h, msg[512]; h = ip->print(ip_buf, sizeof(ip_buf)); snprintf(msg, sizeof(msg), error_msg, ntop->getPrefs()->get_http_prefix(), - h, iface->get_name(), h, num_active_flows_as_client); + h, iface->get_name(), h, max_num_active_flows); ntop->getTrace()->traceEvent(TRACE_INFO, "Begin scan attack: %s", msg); - iface->getAlertsManager()->queueAlert(alert_level_error, alert_on, alert_flow_flood, msg); + iface->getAlertsManager()->engageHostAlert(this, + (char*)"scan_attacker", + alert_flow_flood, alert_level_error, msg); incNumAlerts(); + flow_flood_attacker_alert = true; } } else { total_num_flows_as_server++, num_active_flows_as_server++; - if(num_active_flows_as_server == max_num_active_flows && localHost && triggerAlerts()) { - const char* error_msg = "Host %s is possibly under scan attack [%u active flows]"; + if(num_active_flows_as_server >= max_num_active_flows && localHost && triggerAlerts() && !flow_flood_victim_alert) { + const char* error_msg = "Host %s is possibly under scan attack [%u active flows exceeded]"; char ip_buf[48], *h, msg[512]; h = ip->print(ip_buf, sizeof(ip_buf)); snprintf(msg, sizeof(msg), error_msg, ntop->getPrefs()->get_http_prefix(), - h, iface->get_name(), h, num_active_flows_as_server); + h, iface->get_name(), h, max_num_active_flows); ntop->getTrace()->traceEvent(TRACE_INFO, "Begin scan attack: %s", msg); - iface->getAlertsManager()->queueAlert(alert_level_error, alert_on, alert_flow_flood, msg); + iface->getAlertsManager()->engageHostAlert(this, + (char*)"scan_victim", + alert_flow_flood, alert_level_error, msg); incNumAlerts(); + flow_flood_victim_alert = true; } } } @@ -1145,19 +1196,22 @@ void Host::decNumFlows(bool as_client) { if(num_active_flows_as_client) { num_active_flows_as_client--; - if(num_active_flows_as_client == max_num_active_flows && localHost && triggerAlerts()) { - const char* error_msg = "Host %s is no longer a possible scanner [%u active flows]"; + if(num_active_flows_as_client <= max_num_active_flows && localHost && triggerAlerts() && flow_flood_attacker_alert) { + const char* error_msg = "Host %s is no longer a possible scanner [less than %u active flows]"; char ip_buf[48], *h, msg[512]; h = ip->print(ip_buf, sizeof(ip_buf)); snprintf(msg, sizeof(msg), error_msg, ntop->getPrefs()->get_http_prefix(), - h, iface->get_name(), h, num_active_flows_as_client); + h, iface->get_name(), h, max_num_active_flows); ntop->getTrace()->traceEvent(TRACE_INFO, "End scan attack: %s", msg); - iface->getAlertsManager()->queueAlert(alert_level_info, alert_off, alert_flow_flood, msg); + iface->getAlertsManager()->releaseHostAlert(this, + (char*)"scan_attacker", + alert_flow_flood, alert_level_error, msg); incNumAlerts(); + flow_flood_attacker_alert = false; } } else ntop->getTrace()->traceEvent(TRACE_WARNING, "Internal error: invalid counter value"); @@ -1165,19 +1219,22 @@ void Host::decNumFlows(bool as_client) { if(num_active_flows_as_server) { num_active_flows_as_server--; - if(num_active_flows_as_server == max_num_active_flows && localHost && triggerAlerts()) { - const char* error_msg = "Host %s is no longer under scan attack [%u active flows]"; + if(num_active_flows_as_server <= max_num_active_flows && localHost && triggerAlerts() && flow_flood_victim_alert) { + const char* error_msg = "Host %s is no longer under scan attack [less than %u active flows]"; char ip_buf[48], *h, msg[512]; h = ip->print(ip_buf, sizeof(ip_buf)); snprintf(msg, sizeof(msg), error_msg, ntop->getPrefs()->get_http_prefix(), - h, iface->get_name(), h, num_active_flows_as_server); + h, iface->get_name(), h, max_num_active_flows); - ntop->getTrace()->traceEvent(TRACE_INFO, "End scan attack: %s", msg); - iface->getAlertsManager()->queueAlert(alert_level_info, alert_off, alert_flow_flood, msg); + ntop->getTrace()->traceEvent(TRACE_INFO, "End scan attack: %s", msg); // TODO: remove + iface->getAlertsManager()->releaseHostAlert(this, + (char*)"scan_victim", + alert_flow_flood, alert_level_error, msg); incNumAlerts(); + flow_flood_victim_alert = false; } } else ntop->getTrace()->traceEvent(TRACE_WARNING, "Internal error: invalid counter value"); @@ -1234,7 +1291,7 @@ void Host::updateStats(struct timeval *tv) { snprintf(msg, sizeof(msg), error_msg, ntop->getPrefs()->get_http_prefix(), h, iface->get_name(), h, host_quota_mb); - iface->getAlertsManager()->queueAlert(alert_level_warning, alert_permanent, alert_quota, msg); + iface->getAlertsManager()->storeHostAlert(this, alert_quota, alert_level_warning, msg); } } @@ -1279,7 +1336,7 @@ void Host::loadSynAlertPrefs() { } void Host::loadFlowsAlertPrefs() { - int retval = CONST_MAX_NUM_HOST_ACTIVE_FLOWS; + u_int32_t retval = CONST_MAX_NUM_HOST_ACTIVE_FLOWS; if(ip != NULL) { char rkey[128], rsp[16]; @@ -1288,7 +1345,7 @@ void Host::loadFlowsAlertPrefs() { snprintf(rkey, sizeof(rkey), "ntopng.prefs.%s:%d.flows_alert_threshold", ip->print(ip_buf, sizeof(ip_buf)), vlan_id); if(ntop->getRedis()->get(rkey, rsp, sizeof(rsp)) == 0) - retval = atoi(rsp); + retval = (u_int32_t)strtoul(rsp, NULL, 10); } max_num_active_flows = retval; @@ -1402,7 +1459,10 @@ void Host::incLowGoodputFlows(bool asClient) { c, iface->get_name(), get_name() ? get_name() : c, HOST_LOW_GOODPUT_THRESHOLD, asClient ? "client" : "server"); - iface->getAlertsManager()->queueAlert(alert_level_error, alert_on, asClient ? alert_host_under_attack : alert_host_attacker, alert_msg); + iface->getAlertsManager()->engageHostAlert(this, + asClient ? (char*)"low_goodput_victim", (char*)"low_goodput_attacker", + asClient ? alert_host_under_attack : alert_host_attacker, + alert_level_error, msg); #endif good_low_flow_detected = true; } @@ -1420,7 +1480,12 @@ void Host::decLowGoodputFlows(bool asClient) { } if(alert && good_low_flow_detected) { - /* TODO: send end of alert */ + /* TODO: send end of alert + iface->getAlertsManager()->releaseHostAlert(this, + asClient ? (char*)"low_goodput_victim", (char*)"low_goodput_attacker", + asClient ? alert_host_under_attack : alert_host_attacker, + alert_level_error, msg); + */ good_low_flow_detected = false; } } diff --git a/src/Lua.cpp b/src/Lua.cpp index c807878435..745507f66a 100644 --- a/src/Lua.cpp +++ b/src/Lua.cpp @@ -1516,16 +1516,22 @@ static int ntop_restore_interface_host(lua_State* vm) { NetworkInterface *ntop_interface = getCurrentInterface(vm); char *host_ip; u_int16_t vlan_id = 0; + bool skip_privileges_check = false; char buf[64]; ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); - if(!Utils::isUserAdministrator(vm)) return(CONST_LUA_ERROR); + if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING)) return(CONST_LUA_ERROR); get_host_vlan_info((char*)lua_tostring(vm, 1), &host_ip, &vlan_id, buf, sizeof(buf)); - if((!ntop_interface) || !ntop_interface->restoreHost(host_ip)) + /* make sure skip privileges check cannot be set from the web interface */ + if(lua_type(vm, 2) == LUA_TBOOLEAN) skip_privileges_check = lua_toboolean(vm, 2); + + if(!skip_privileges_check && !Utils::isUserAdministrator(vm)) return(CONST_LUA_ERROR); + + if((!ntop_interface) || !ntop_interface->restoreHost(host_ip, vlan_id)) return(CONST_LUA_ERROR); else return(CONST_LUA_OK); @@ -4102,74 +4108,34 @@ static int ntop_redis_get_id_to_host(lua_State* vm) { /* ****************************************** */ -static int ntop_interface_get_num_queued_alerts(lua_State* vm) { +static int ntop_interface_get_alerts(lua_State* vm) { NetworkInterface *iface = getCurrentInterface(vm); - - ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); - lua_pushinteger(vm, iface->getAlertsManager()->getNumQueuedAlerts()); - - return(CONST_LUA_OK); -} - -/* ****************************************** */ - -static int ntop_interface_delete_queued_alert(lua_State* vm) { - NetworkInterface *iface = getCurrentInterface(vm); - - ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); - - if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TNUMBER)) return(CONST_LUA_ERROR); - iface->getAlertsManager()->deleteQueuedAlert((u_int32_t)lua_tonumber(vm, 1)); - lua_pushnil(vm); /* Always return a value to Lua */ - return(CONST_LUA_OK); -} - -/* ****************************************** */ - -static int ntop_interface_flush_all_queued_alerts(lua_State* vm) { - NetworkInterface *iface = getCurrentInterface(vm); - - if(!Utils::isUserAdministrator(vm)) - return(CONST_LUA_ERROR); // alerts are only allowed for physical interfaces - - ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); - - iface->getAlertsManager()->flushAllQueuedAlerts(); - lua_pushnil(vm); /* Always return a value to Lua */ - return(CONST_LUA_OK); -} - -/* ****************************************** */ - -static int ntop_interface_queue_alert(lua_State* vm) { - NetworkInterface *iface = getCurrentInterface(vm); - int alert_level; - int alert_status; - int alert_type; - char *alert_msg; + u_int32_t num, start_idx = 0; + bool engaged = false; + AlertsManager *am; ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TNUMBER)) return(CONST_LUA_ERROR); - alert_level = (int)lua_tonumber(vm, 1); + start_idx = (u_int32_t)lua_tonumber(vm, 1); if(ntop_lua_check(vm, __FUNCTION__, 2, LUA_TNUMBER)) return(CONST_LUA_ERROR); - alert_status = (int)lua_tonumber(vm, 2); + num = (u_int32_t)lua_tonumber(vm, 2); - if(ntop_lua_check(vm, __FUNCTION__, 3, LUA_TNUMBER)) return(CONST_LUA_ERROR); - alert_type = (int)lua_tonumber(vm, 3); + if(lua_type(vm, 3) == LUA_TBOOLEAN) + engaged = lua_toboolean(vm, 3); - if(ntop_lua_check(vm, __FUNCTION__, 4, LUA_TSTRING)) return(CONST_LUA_ERROR); - alert_msg = (char*)lua_tostring(vm, 4); + if(num < 1) num = 1; + else if(num > CONST_MAX_NUM_READ_ALERTS) num = CONST_MAX_NUM_READ_ALERTS; - if(iface->getAlertsManager()->queueAlert((AlertLevel)alert_level, (AlertStatus)alert_status, - (AlertType)alert_type, alert_msg)) - return(CONST_LUA_ERROR); - return(CONST_LUA_OK); + if(!iface || !(am = iface->getAlertsManager())) + return (CONST_LUA_ERROR); + + return am->getAlerts(vm, get_allowed_nets(vm), start_idx, start_idx + num - 1, engaged) ? CONST_LUA_ERROR : CONST_LUA_OK; } /* ****************************************** */ - +#ifdef NOTUSED static int ntop_interface_store_alert(lua_State* vm) { int ifid; NetworkInterface* iface; @@ -4190,7 +4156,7 @@ static int ntop_interface_store_alert(lua_State* vm) { return am->storeAlert(vm, 2) ? CONST_LUA_ERROR : CONST_LUA_OK; } - +#endif /* ****************************************** */ static int ntop_interface_engage_release_host_alert(lua_State* vm, bool engage) { @@ -4239,6 +4205,86 @@ static int ntop_interface_engage_release_host_alert(lua_State* vm, bool engage) /* ****************************************** */ +static int ntop_interface_engage_release_network_alert(lua_State* vm, bool engage) { + NetworkInterface *ntop_interface = getCurrentInterface(vm); + char *cidr; + int alert_severity; + int alert_type; + char *alert_json, *engaged_alert_id; + AlertsManager *am; + int ret; + + ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); + + if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING)) return(CONST_LUA_ERROR); + cidr = (char*)lua_tostring(vm, 1); + + if(ntop_lua_check(vm, __FUNCTION__, 2, LUA_TSTRING)) return(CONST_LUA_ERROR); + engaged_alert_id = (char*)lua_tostring(vm, 2); + + if(ntop_lua_check(vm, __FUNCTION__, 3, LUA_TNUMBER)) return(CONST_LUA_ERROR); + alert_type = (int)lua_tonumber(vm, 3); + + if(ntop_lua_check(vm, __FUNCTION__, 4, LUA_TNUMBER)) return(CONST_LUA_ERROR); + alert_severity = (int)lua_tonumber(vm, 4); + + if(ntop_lua_check(vm, __FUNCTION__, 5, LUA_TSTRING)) return(CONST_LUA_ERROR); + alert_json = (char*)lua_tostring(vm, 5); + + if((!ntop_interface) + || ((am = ntop_interface->getAlertsManager()) == NULL)) + return(CONST_LUA_ERROR); + + if(engage) + ret = am->engageNetworkAlert(cidr, engaged_alert_id, + (AlertType)alert_type, (AlertLevel)alert_severity, alert_json); + else + ret = am->releaseNetworkAlert(cidr, engaged_alert_id, + (AlertType)alert_type, (AlertLevel)alert_severity, alert_json); + + return !ret ? CONST_LUA_OK : CONST_LUA_ERROR; +} + +/* ****************************************** */ + +static int ntop_interface_engage_release_interface_alert(lua_State* vm, bool engage) { + NetworkInterface *ntop_interface = getCurrentInterface(vm); + int alert_severity; + int alert_type; + char *alert_json, *engaged_alert_id; + AlertsManager *am; + int ret; + + ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); + + if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING)) return(CONST_LUA_ERROR); + engaged_alert_id = (char*)lua_tostring(vm, 1); + + if(ntop_lua_check(vm, __FUNCTION__, 2, LUA_TNUMBER)) return(CONST_LUA_ERROR); + alert_type = (int)lua_tonumber(vm, 2); + + if(ntop_lua_check(vm, __FUNCTION__, 3, LUA_TNUMBER)) return(CONST_LUA_ERROR); + alert_severity = (int)lua_tonumber(vm, 3); + + if(ntop_lua_check(vm, __FUNCTION__, 4, LUA_TSTRING)) return(CONST_LUA_ERROR); + alert_json = (char*)lua_tostring(vm, 4); + + if((!ntop_interface) + || ((am = ntop_interface->getAlertsManager()) == NULL)) + return(CONST_LUA_ERROR); + + if(engage) + ret = am->engageInterfaceAlert(ntop_interface, engaged_alert_id, + (AlertType)alert_type, (AlertLevel)alert_severity, alert_json); + else + ret = am->releaseInterfaceAlert(ntop_interface, engaged_alert_id, + (AlertType)alert_type, (AlertLevel)alert_severity, alert_json); + + return !ret ? CONST_LUA_OK : CONST_LUA_ERROR; +} + +/* ****************************************** */ + static int ntop_interface_engage_host_alert(lua_State* vm) { return ntop_interface_engage_release_host_alert(vm, true /* engage */); } @@ -4251,6 +4297,68 @@ static int ntop_interface_release_host_alert(lua_State* vm) { /* ****************************************** */ +static int ntop_interface_engage_network_alert(lua_State* vm) { + return ntop_interface_engage_release_network_alert(vm, true /* engage */); +} + +/* ****************************************** */ + +static int ntop_interface_release_network_alert(lua_State* vm) { + return ntop_interface_engage_release_network_alert(vm, false /* release */); +} + +/* ****************************************** */ + +static int ntop_interface_engage_interface_alert(lua_State* vm) { + return ntop_interface_engage_release_interface_alert(vm, true /* engage */); +} + +/* ****************************************** */ + +static int ntop_interface_release_interface_alert(lua_State* vm) { + return ntop_interface_engage_release_interface_alert(vm, false /* release */); +} + +/* ****************************************** */ + +static int ntop_interface_get_num_alerts(lua_State* vm) { + NetworkInterface *iface = getCurrentInterface(vm); + bool engaged = false; + + ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); + + if(lua_type(vm, 1) == LUA_TBOOLEAN) + engaged = lua_toboolean(vm, 1); + + lua_pushinteger(vm, iface->getAlertsManager()->getNumAlerts(engaged)); + + return(CONST_LUA_OK); +} + +/* ****************************************** */ + +static int ntop_interface_delete_alerts(lua_State* vm) { + NetworkInterface *iface = getCurrentInterface(vm); + bool engaged = false; + int rowid, *rowid_ptr = NULL; + + ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); + + if(lua_type(vm, 1) == LUA_TBOOLEAN) + engaged = lua_toboolean(vm, 1); + + if(lua_type(vm, 2) == LUA_TNUMBER){ + rowid = lua_tonumber(vm, 2); + rowid_ptr = &rowid; + } + + lua_pushinteger(vm, iface->getAlertsManager()->deleteAlerts(engaged, rowid_ptr)); + + return(CONST_LUA_OK); +} + +/* ****************************************** */ + #if NTOPNG_PRO static int ntop_nagios_reload_config(lua_State* vm) { @@ -4368,28 +4476,6 @@ static int ntop_reload_traffic_profiles(lua_State* vm) { /* ****************************************** */ -static int ntop_interface_get_queued_alerts(lua_State* vm) { - NetworkInterface *iface = getCurrentInterface(vm); - u_int32_t num, start_idx = 0; - - ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__); - - if(ntop_lua_check(vm, __FUNCTION__, 1, LUA_TNUMBER)) return(CONST_LUA_ERROR); - start_idx = (u_int32_t)lua_tonumber(vm, 1); - - if(ntop_lua_check(vm, __FUNCTION__, 2, LUA_TNUMBER)) return(CONST_LUA_ERROR); - num = (u_int32_t)lua_tonumber(vm, 2); - - if(num < 1) num = 1; - else if(num > CONST_MAX_NUM_READ_ALERTS) num = CONST_MAX_NUM_READ_ALERTS; - - iface->getAlertsManager()->getQueuedAlerts(vm, get_allowed_nets(vm), start_idx, start_idx + num); - - return(CONST_LUA_OK); -} - -/* ****************************************** */ - static int ntop_set_redis(lua_State* vm) { char *key, *value; u_int expire_secs = 0; // default 0 = no expiration @@ -4752,19 +4838,18 @@ static const luaL_Reg ntop_interface_reg[] = { { "getFlowDevices", ntop_getflowdevices }, { "getFlowDeviceInfo", ntop_getflowdeviceinfo }, - /* Alerts */ - { "getNumQueuedAlerts", ntop_interface_get_num_queued_alerts }, - { "getQueuedAlerts", ntop_interface_get_queued_alerts }, - { "queueAlert", ntop_interface_queue_alert }, - { "deleteQueuedAlert", ntop_interface_delete_queued_alert }, - { "flushAllQueuedAlerts", ntop_interface_flush_all_queued_alerts }, - { "enableHostAlerts", ntop_interface_host_enable_alerts }, - { "disableHostAlerts", ntop_interface_host_disable_alerts }, - /* New generation alerts */ - { "storeAlert", ntop_interface_store_alert }, + { "getAlerts" , ntop_interface_get_alerts }, + { "getNumAlerts", ntop_interface_get_num_alerts }, + { "deleteAlerts", ntop_interface_delete_alerts }, { "engageHostAlert", ntop_interface_engage_host_alert }, { "releaseHostAlert", ntop_interface_release_host_alert }, + { "engageNetworkAlert", ntop_interface_engage_network_alert }, + { "releaseNetworkAlert", ntop_interface_release_network_alert }, + { "engageInterfaceAlert", ntop_interface_engage_interface_alert }, + { "releaseInterfaceAlert",ntop_interface_release_interface_alert }, + { "enableHostAlerts", ntop_interface_host_enable_alerts }, + { "disableHostAlerts", ntop_interface_host_disable_alerts }, { NULL, NULL } }; diff --git a/src/MySQLDB.cpp b/src/MySQLDB.cpp index 64c78e4f7a..57711e53cd 100644 --- a/src/MySQLDB.cpp +++ b/src/MySQLDB.cpp @@ -389,7 +389,9 @@ bool MySQLDB::createDBSchema() { MySQLDB::MySQLDB(NetworkInterface *_iface) : DB(_iface) { mysqlDroppedFlowsQueueTooLong = 0; - mysqlExportedFlows = 0; + mysqlExportedFlows = 0, mysqlLastExportedFlows = 0; + mysqlExportRate = 0; + lastUpdateTime.tv_sec = 0, lastUpdateTime.tv_usec = 0; connectToDB(&mysql, false); } @@ -405,6 +407,34 @@ void MySQLDB::startDBLoop() { pthread_create(&queryThreadLoop, NULL, ::queryLoop, (void*)this); } + +/* ******************************************* */ + +void MySQLDB::updateStats(const struct timeval *tv) { + if(tv == NULL) return; + + if(lastUpdateTime.tv_sec > 0) { + float tdiffMsec = ((float)(tv->tv_sec-lastUpdateTime.tv_sec)*1000)+((tv->tv_usec-lastUpdateTime.tv_usec)/(float)1000); + if(tdiffMsec >= 1000) { /* al least one second */ + u_int64_t diffFlows = mysqlExportedFlows - mysqlLastExportedFlows; + mysqlLastExportedFlows = mysqlExportedFlows; + + mysqlExportRate = ((float)(diffFlows * 1000)) / tdiffMsec; + if (mysqlExportRate < 0) mysqlExportRate = 0; + } + } + + memcpy(&lastUpdateTime, tv, sizeof(struct timeval)); +} + +/* ******************************************* */ + +void MySQLDB::lua(lua_State *vm) const { + lua_push_int_table_entry(vm, "flow_export_count", mysqlExportedFlows); + lua_push_int32_table_entry(vm, "flow_export_drops", mysqlDroppedFlowsQueueTooLong); + lua_push_float_table_entry(vm, "flow_export_rate", mysqlExportRate); +} + /* ******************************************* */ #ifdef NOTUSED diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 409286675b..8b3069e6a5 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -637,7 +637,9 @@ void NetworkInterface::triggerTooManyFlowsAlert() { ntop->getPrefs()->get_http_prefix(), id, get_name()); - alertsManager->queueAlert(alert_level_error, alert_on, alert_app_misconfiguration, alert_msg); + alertsManager->engageInterfaceAlert(this, + (char*)"app_misconfiguration", + alert_app_misconfiguration, alert_level_error, alert_msg); tooManyFlowsAlertTriggered = true; } } @@ -653,7 +655,9 @@ void NetworkInterface::triggerTooManyHostsAlert() { ntop->getPrefs()->get_http_prefix(), id, get_name()); - alertsManager->queueAlert(alert_level_error, alert_on, alert_app_misconfiguration, alert_msg); + alertsManager->releaseInterfaceAlert(this, + (char*)"app_misconfiguration", + alert_app_misconfiguration, alert_level_error, alert_msg); tooManyHostsAlertTriggered = true; } } @@ -1013,7 +1017,7 @@ bool NetworkInterface::processPacket(const struct bpf_timeval *when, e.g., during three-way-handshake, or when acknowledging. Make sure only non-zero-payload segments are processed. */ - if(payload_len > 0 && payload) { + if((payload_len > 0) && payload) { /* DNS-over-TCP has a 2-bytes field with DNS payload length at the beginning. See RFC1035 section 4.2.2. TCP usage. @@ -1067,7 +1071,7 @@ bool NetworkInterface::processPacket(const struct bpf_timeval *when, break; default: - if (flow->isSSLProto()) + if(flow->isSSLProto()) flow->dissectSSL(payload, payload_len, when, src2dst_direction); } @@ -1107,12 +1111,12 @@ bool NetworkInterface::processPacket(const struct bpf_timeval *when, // Detect user activities UserActivityID activity; u_int64_t up=0, down=0, backgr=0, bytes=payload_len; - if (flow->getActivityId(&activity) && (!flow->isSSLProto() || flow->isSSLData())) { + if(flow->getActivityId(&activity) && (!flow->isSSLProto() || flow->isSSLData())) { Host *cli = flow->get_cli_host(); Host *srv = flow->get_srv_host(); - if (flow->invokeActivityFilter(when, src2dst_direction, payload_len)) { - if (src2dst_direction) + if(flow->invokeActivityFilter(when, src2dst_direction, payload_len)) { + if(src2dst_direction) up = bytes; else down = bytes; @@ -1120,9 +1124,9 @@ bool NetworkInterface::processPacket(const struct bpf_timeval *when, backgr = bytes; } - if (cli->isLocalHost()) + if(cli->isLocalHost()) cli->incActivityBytes(activity, up, down, backgr); - if (srv->isLocalHost()) + if(srv->isLocalHost()) srv->incActivityBytes(activity, down, up, backgr); } @@ -1659,6 +1663,10 @@ void NetworkInterface::periodicStatsUpdate() { flows_hash->walk(flow_update_hosts_stats, (void*)&tv); hosts_hash->walk(update_hosts_stats, (void*)&tv); + + if(ntop->getPrefs()->do_dump_flows_on_mysql()){ + static_cast(db)->updateStats(&tv); + } } /* **************************************************** */ @@ -1745,8 +1753,8 @@ static bool find_host_by_name(GenericHashEntry *h, void *user_data) { /* **************************************************** */ -bool NetworkInterface::restoreHost(char *host_ip) { - Host *h = new Host(this, host_ip); +bool NetworkInterface::restoreHost(char *host_ip, u_int16_t vlan_id) { + Host *h = new Host(this, host_ip, vlan_id); if(!h) return(false); @@ -2588,7 +2596,7 @@ void NetworkInterface::getNetworksStats(lua_State* vm) { u_int8_t num_local_networks = ntop->getNumLocalNetworks(); lua_newtable(vm); - for (u_int8_t network_id = 0; network_id < num_local_networks; network_id++) { + for(u_int8_t network_id = 0; network_id < num_local_networks; network_id++) { network_stats = getNetworkStats(network_id); // do not add stats of networks that have not generated any traffic if(!network_stats || !network_stats->trafficSeen()) @@ -2658,7 +2666,7 @@ static bool host_activity_walker(GenericHashEntry *he, void *user_data) { return (false); /* false = keep on walking */ r->found = true; - for (i=0; icounters[i] = *h->getActivityBytes((UserActivityID) i); return true; /* found, stop walking */ } @@ -2675,7 +2683,7 @@ void NetworkInterface::getLocalHostActivity(lua_State* vm, const char *host) { if(retriever.found) { lua_newtable(vm); - for (i=0; igetPrefs()->do_dump_flows_on_es()) - lua_push_int_table_entry(vm, "flow_export_drops", ntop->getElasticSearch()->numDroppedFlows()); - else if (ntop->getPrefs()->do_dump_flows_on_mysql()) - lua_push_int_table_entry(vm, "flow_export_drops", static_cast(db)->numDroppedFlows()); + if(ntop->getPrefs()->do_dump_flows_on_es()) { + ntop->getElasticSearch()->lua(vm); + } else if(ntop->getPrefs()->do_dump_flows_on_mysql()) { + db->lua(vm); + } lua_pushstring(vm, "stats"); lua_insert(vm, -2); @@ -3558,7 +3567,14 @@ void NetworkInterface::setRemoteStats(char *name, char *address, u_int32_t speed ntop->getTrace()->traceEvent(TRACE_INFO, "[%s][bytes=%u/%u (%d)][pkts=%u/%u (%d)]", ifname, remBytes, ethStats.getNumBytes(), remBytes-ethStats.getNumBytes(), remPkts, ethStats.getNumPackets(), remPkts-ethStats.getNumPackets()); + /* + * Don't override ethStats here, these stats are properly updated + * inside NetworkInterface::processFlow for ZMQ interfaces. + * Overriding values here may cause glitches and non-strictly-increasing counters + * yielding negative rates. ethStats.setNumBytes(remBytes), ethStats.setNumPackets(remPkts); + * + */ } } @@ -3779,7 +3795,7 @@ static int lua_flow_set_activity_filter(lua_State* vm) { if(ntop_lua_check(vm, __FUNCTION__, params+1, LUA_TNUMBER)) return(CONST_LUA_ERROR); activityID = (UserActivityID)lua_tonumber(vm, ++params); - if (activityID >= UserActivitiesN) return(CONST_LUA_ERROR); + if(activityID >= UserActivitiesN) return(CONST_LUA_ERROR); if(lua_type(vm, params+1) == LUA_TNUMBER) filterID = (ActivityFilterID)lua_tonumber(vm, ++params); @@ -3800,16 +3816,16 @@ static int lua_flow_set_activity_filter(lua_State* vm) { if(lua_type(vm, params+1) == LUA_TNUMBER) { config.web.numsamples = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) { + if(lua_type(vm, params+1) == LUA_TNUMBER) { config.web.minbytes = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) { + if(lua_type(vm, params+1) == LUA_TNUMBER) { config.web.maxinterval = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TBOOLEAN) { + if(lua_type(vm, params+1) == LUA_TBOOLEAN) { config.web.forceWebProfile = lua_toboolean(vm, ++params); - if (lua_type(vm, params+1) == LUA_TBOOLEAN) + if(lua_type(vm, params+1) == LUA_TBOOLEAN) config.web.serverdominant = lua_toboolean(vm, ++params); } } @@ -3828,10 +3844,10 @@ static int lua_flow_set_activity_filter(lua_State* vm) { if(lua_type(vm, params+1) == LUA_TNUMBER) { config.ratio.numsamples = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) { + if(lua_type(vm, params+1) == LUA_TNUMBER) { config.ratio.minbytes = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) + if(lua_type(vm, params+1) == LUA_TNUMBER) config.ratio.clisrv_ratio = lua_tonumber(vm, ++params); } } @@ -3846,7 +3862,7 @@ static int lua_flow_set_activity_filter(lua_State* vm) { if(lua_type(vm, params+1) == LUA_TNUMBER) { config.interflow.minflows = min((int)lua_tonumber(vm, ++params), INTER_FLOW_ACTIVITY_SLOTS); - if (lua_type(vm, params+1) == LUA_TNUMBER) { + if(lua_type(vm, params+1) == LUA_TNUMBER) { config.interflow.minpkts = lua_tonumber(vm, ++params); if (lua_type(vm, params+1) == LUA_TNUMBER) { @@ -3871,13 +3887,13 @@ static int lua_flow_set_activity_filter(lua_State* vm) { if(lua_type(vm, params+1) == LUA_TNUMBER) { config.sma.edge = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) { + if(lua_type(vm, params+1) == LUA_TNUMBER) { config.sma.minsamples = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) { + if(lua_type(vm, params+1) == LUA_TNUMBER) { config.sma.timebound = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) + if(lua_type(vm, params+1) == LUA_TNUMBER) config.sma.sustain = lua_tonumber(vm, ++params); } } @@ -3894,13 +3910,13 @@ static int lua_flow_set_activity_filter(lua_State* vm) { if(lua_type(vm, params+1) == LUA_TNUMBER) { config.wma.edge = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) { + if(lua_type(vm, params+1) == LUA_TNUMBER) { config.wma.minsamples = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) { + if(lua_type(vm, params+1) == LUA_TNUMBER) { config.wma.timescale = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) + if(lua_type(vm, params+1) == LUA_TNUMBER) config.wma.aggrsecs = lua_tonumber(vm, ++params); } } @@ -3920,7 +3936,7 @@ static int lua_flow_set_activity_filter(lua_State* vm) { if(lua_type(vm, params+1) == LUA_TNUMBER) { config.command_sequence.minbytes = lua_tonumber(vm, ++params); - if (lua_type(vm, params+1) == LUA_TNUMBER) { + if(lua_type(vm, params+1) == LUA_TNUMBER) { config.command_sequence.maxinterval = lua_tonumber(vm, ++params); if (lua_type(vm, params+1) == LUA_TNUMBER) { @@ -4022,7 +4038,7 @@ lua_State* NetworkInterface::initLuaInterpreter(const char *lua_file) { // Activity profiles - see ntop_typedefs.h lua_newtable(L); - for (int i=0; irunHousekeepingTasks(); + + /* ES stats are updated once as the present implementation is not per-interface */ + if (ntop->getPrefs()->do_dump_flows_on_es()) { + struct timeval tv; + gettimeofday(&tv, NULL); + ntop->getElasticSearch()->updateStats(&tv); + } } /* ******************************************* */ diff --git a/src/Prefs.cpp b/src/Prefs.cpp index d5596faa54..7badba4fd0 100755 --- a/src/Prefs.cpp +++ b/src/Prefs.cpp @@ -61,7 +61,7 @@ Prefs::Prefs(Ntop *_ntop) { disable_alerts = false; pid_path = strdup(DEFAULT_PID_PATH); packet_filter = NULL; - disable_host_persistency = false; + enable_idle_local_hosts_cache = true; num_interfaces = 0, enable_auto_logout = true; dump_flows_on_es = dump_flows_on_mysql = false; enable_taps = false; @@ -75,9 +75,10 @@ Prefs::Prefs(Ntop *_ntop) { enable_ixia_timestamps = enable_vss_apcon_timestamps = false; /* Defaults */ - non_local_host_max_idle = MAX_REMOTE_HOST_IDLE /* sec */; - local_host_max_idle = MAX_LOCAL_HOST_IDLE /* sec */; - flow_max_idle = MAX_FLOW_IDLE /* sec */; + local_host_cache_duration = LOCAL_HOSTS_CACHE_DURATION /* sec */; + non_local_host_max_idle = MAX_REMOTE_HOST_IDLE /* sec */; + local_host_max_idle = MAX_LOCAL_HOST_IDLE /* sec */; + flow_max_idle = MAX_FLOW_IDLE /* sec */; intf_rrd_raw_days = INTF_RRD_RAW_DAYS; intf_rrd_1min_days = INTF_RRD_1MIN_DAYS; @@ -207,7 +208,6 @@ void usage() { " | (e.g. -m \"192.168.0.0/24,172.16.0.0/16\")\n" "[--ndpi-protocols|-p] .protos | Specify a nDPI protocol file\n" " | (eg. protos.txt)\n" - "[--disable-host-persistency|-P] | Disable host persistency in the Redis cache\n" "[--redis|-r] | Redis connection. is [h[:port[:pwd]]][@db-id]\n" " | db-id is the identifier of the redis database (default 0).\n" " | h is the host that is running the Redis server (default\n" @@ -381,9 +381,10 @@ void Prefs::getDefaultStringPrefsValue(const char *pref_key, char **buffer, cons void Prefs::reloadPrefsFromRedis() { /* attempt to load preferences set from the web ui and apply default values in not found */ - local_host_max_idle = getDefaultPrefsValue(CONST_LOCAL_HOST_IDLE_PREFS, MAX_LOCAL_HOST_IDLE); - non_local_host_max_idle = getDefaultPrefsValue(CONST_REMOTE_HOST_IDLE_PREFS, MAX_REMOTE_HOST_IDLE); - flow_max_idle = getDefaultPrefsValue(CONST_FLOW_MAX_IDLE_PREFS, MAX_FLOW_IDLE); + local_host_cache_duration = getDefaultPrefsValue(CONST_LOCAL_HOST_CACHE_DURATION_PREFS, LOCAL_HOSTS_CACHE_DURATION); + local_host_max_idle = getDefaultPrefsValue(CONST_LOCAL_HOST_IDLE_PREFS, MAX_LOCAL_HOST_IDLE); + non_local_host_max_idle = getDefaultPrefsValue(CONST_REMOTE_HOST_IDLE_PREFS, MAX_REMOTE_HOST_IDLE); + flow_max_idle = getDefaultPrefsValue(CONST_FLOW_MAX_IDLE_PREFS, MAX_FLOW_IDLE); intf_rrd_raw_days = getDefaultPrefsValue(CONST_INTF_RRD_RAW_DAYS, INTF_RRD_RAW_DAYS); intf_rrd_1min_days = getDefaultPrefsValue(CONST_INTF_RRD_1MIN_DAYS, INTF_RRD_1MIN_DAYS); @@ -397,7 +398,10 @@ void Prefs::reloadPrefsFromRedis() { HOUSEKEEPING_FREQUENCY); // sets to the default value in redis if no key is found - getDefaultPrefsValue(CONST_RUNTIME_IS_AUTOLOGOUT_ENABLED, CONST_DEFAULT_IS_AUTOLOGOUT_ENABLED); + getDefaultPrefsValue(CONST_RUNTIME_IS_AUTOLOGOUT_ENABLED, + CONST_DEFAULT_IS_AUTOLOGOUT_ENABLED); + enable_idle_local_hosts_cache = getDefaultPrefsValue(CONST_RUNTIME_IDLE_LOCAL_HOSTS_CACHE_ENABLED, + CONST_DEFAULT_IS_IDLE_LOCAL_HOSTS_CACHE_ENABLED); setTraceLevelFromRedis(); setAlertsEnabledFromRedis(); @@ -451,7 +455,6 @@ static const struct option long_options[] = { #endif { "disable-alerts", no_argument, NULL, 'H' }, { "export-flows", required_argument, NULL, 'I' }, - { "disable-host-persistency", no_argument, NULL, 'P' }, { "capture-direction", required_argument, NULL, 'Q' }, { "sticky-hosts", required_argument, NULL, 'S' }, { "enable-taps", no_argument, NULL, 'T' }, @@ -574,9 +577,9 @@ int Prefs::setOption(int optkey, char *optarg) { case 'S': if(!strcmp(optarg, "all")) sticky_hosts = location_all; - else if(!strcmp(optarg, "local")) sticky_hosts = location_local_only; + else if(!strcmp(optarg, "local")) sticky_hosts = location_local_only; else if(!strcmp(optarg, "remote")) sticky_hosts = location_remote_only; - else if(!strcmp(optarg, "none")) sticky_hosts = location_none; + else if(!strcmp(optarg, "none")) sticky_hosts = location_none; else ntop->getTrace()->traceEvent(TRACE_ERROR, "Unknown value %s for -S", optarg); break; @@ -627,10 +630,6 @@ int Prefs::setOption(int optkey, char *optarg) { } break; - case 'P': - disable_host_persistency = true; - break; - case 'T': enable_taps = true; break; @@ -1134,6 +1133,7 @@ void Prefs::lua(lua_State* vm) { lua_push_bool_table_entry(vm, "are_alerts_enabled", !disable_alerts); lua_push_int_table_entry(vm, "http_port", http_port); lua_push_int_table_entry(vm, "local_host_max_idle", local_host_max_idle); + lua_push_int_table_entry(vm, "local_host_cache_duration", local_host_cache_duration); lua_push_int_table_entry(vm, "non_local_host_max_idle", non_local_host_max_idle); lua_push_int_table_entry(vm, "flow_max_idle", flow_max_idle); lua_push_int_table_entry(vm, "max_num_hosts", max_num_hosts); @@ -1156,6 +1156,15 @@ void Prefs::lua(lua_State* vm) { lua_push_str_table_entry(vm, "instance_name", instance_name ? instance_name : (char*)""); + /* Sticky hosts preferences */ + if (sticky_hosts != location_none) { + char *location_string = NULL; + if(sticky_hosts == location_all) location_string = (char*)"all"; + else if(sticky_hosts == location_local_only) location_string = (char*)"local"; + else if(sticky_hosts == location_remote_only) location_string = (char*)"remote"; + if(location_string) lua_push_str_table_entry(vm, "sticky_hosts", location_string); + } + /* Command line options */ lua_push_bool_table_entry(vm, "has_cmdl_trace_lvl", has_cmdl_trace_lvl); lua_push_bool_table_entry(vm, "has_cmdl_disable_alerts", has_cmdl_disable_alerts); @@ -1180,6 +1189,10 @@ int Prefs::refresh(const char *pref_name, const char *pref_value) { (char*)CONST_RUNTIME_PREFS_HOUSEKEEPING_FREQUENCY, strlen((char*)CONST_RUNTIME_PREFS_HOUSEKEEPING_FREQUENCY))) housekeeping_frequency = atoi(pref_value); + else if (!strncmp(pref_name, + (char*)CONST_LOCAL_HOST_CACHE_DURATION_PREFS, + strlen((char*)CONST_LOCAL_HOST_CACHE_DURATION_PREFS))) + local_host_cache_duration = atoi(pref_value); else if (!strncmp(pref_name, (char*)CONST_LOCAL_HOST_IDLE_PREFS, strlen((char*)CONST_LOCAL_HOST_IDLE_PREFS))) @@ -1228,6 +1241,10 @@ int Prefs::refresh(const char *pref_name, const char *pref_value) { (char*)CONST_ALERT_DISABLED_PREFS, strlen((char*)CONST_ALERT_DISABLED_PREFS))) disable_alerts = pref_value[0] == '1' ? true : false; + else if (!strncmp(pref_name, + (char*)CONST_RUNTIME_IDLE_LOCAL_HOSTS_CACHE_ENABLED, + strlen((char*)CONST_RUNTIME_IDLE_LOCAL_HOSTS_CACHE_ENABLED))) + enable_idle_local_hosts_cache = pref_value[0] == '1' ? true : false; return 0; } diff --git a/third-party/mongoose/mongoose.c b/third-party/mongoose/mongoose.c index 59b0714d7c..26798f2e11 100644 --- a/third-party/mongoose/mongoose.c +++ b/third-party/mongoose/mongoose.c @@ -384,6 +384,11 @@ struct ssl_func { #define SSL_pending (* (int (*)(SSL *)) ssl_sw[18].ptr) #define SSL_CTX_set_verify (* (void (*)(SSL_CTX *, int, int)) ssl_sw[19].ptr) +/* ntop */ +#define SSL_CTX_set_cipher_list (* (int (*)(SSL_CTX *ctx, const char *str)) ssl_sw[20].ptr) +#define SSL_CTX_set_options (* (long (*)(SSL_CTX *ctx, long options)) ssl_sw[21].ptr) +#define SSL_CTX_get_options (* (long (*)(SSL_CTX *ctx)) ssl_sw[22].ptr) +#define SSL_CTX_set_min_proto_version (* (void (*)(SSL_CTX *ctx, int version)) ssl_sw[23].ptr) #define CRYPTO_num_locks (* (int (*)(void)) crypto_sw[0].ptr) #define CRYPTO_set_locking_callback \ (* (void (*)(void (*)(int, int, const char *, int))) crypto_sw[1].ptr) @@ -417,6 +422,14 @@ static struct ssl_func ssl_sw[] = { {"SSLv23_client_method", NULL}, {"SSL_pending", NULL}, {"SSL_CTX_set_verify", NULL}, + {"SSL_CTX_set_cipher_list", NULL}, +#ifndef __APPLE__ + {"SSL_CTX_set_options", NULL}, + {"SSL_CTX_get_options", NULL}, +#ifdef MODERN_OPENSSL + {"SSL_CTX_set_min_proto_version", NULL }, /* ntop */ +#endif +#endif {NULL, NULL} }; @@ -4721,11 +4734,36 @@ static int set_ssl_option(struct mg_context *ctx) { SSL_library_init(); SSL_load_error_strings(); + if ((ctx->ssl_ctx = SSL_CTX_new(SSLv23_server_method())) == NULL) { cry(fc(ctx), "SSL_CTX_new (server) error: %s", ssl_error()); return 0; } + /* ntop */ + SSL_CTX_set_cipher_list(ctx->ssl_ctx, "HIGH:!aNULL:!MD5:!RC4"); + +#ifndef __APPLE__ /* Brew comes with an old OpenSSL version */ +#ifdef MODERN_OPENSSL +#ifndef TLS1_1_VERSION +#define TLS1_1_VERSION 0x0302 +#endif + SSL_CTX_set_min_proto_version(ctx->ssl_ctx, TLS1_1_VERSION); +#else + { +#ifndef SSL_OP_NO_TLSv1 +#define SSL_OP_NO_TLSv1 0x04000000L +#endif + + long opts = SSL_CTX_get_options(ctx->ssl_ctx); + + opts |= SSL_OP_NO_TLSv1; + SSL_CTX_set_options(ctx->ssl_ctx, opts); + } +#endif +#endif + /* end ntop */ + // If user callback returned non-NULL, that means that user callback has // set up certificate itself. In this case, skip sertificate setting. if ((ctx->callbacks.init_ssl == NULL ||