Implements Local Host behaviour analysis and it's alert

Alert in case the host has an unexpected behaviour

include/HostStats.h

@@ -110,7 +110,7 @@ class HostStats: public GenericTrafficElement {
   virtual void luaAnomalies(lua_State* vm, time_t when) {};
   virtual void luaPeers(lua_State *vm) 			{};
   virtual void lua(lua_State* vm, bool mask_host, DetailsLevel details_level);
-
+  virtual void luaHostBehaviour(lua_State* vm)     { };
 #ifdef NTOPNG_PRO
   inline void incQuotaEnforcementStats(time_t when, u_int16_t ndpi_proto,
 				       u_int64_t sent_packets, u_int64_t sent_bytes,

include/LocalHost.h

@@ -112,6 +112,7 @@ class LocalHost : public Host, public SerializableElement {
   void custom_periodic_stats_update(const struct timeval *tv) {
   }
 
+  virtual void luaHostBehaviour(lua_State* vm)    { if(stats) stats->luaHostBehaviour(vm); }
   virtual void incDohDoTUses(Host *srv_host);
 
   virtual void incNTPContactCardinality(Host *h)  { stats->incNTPContactCardinality(h);  }

include/LocalHostStats.h

@@ -36,12 +36,12 @@ class LocalHostStats: public HostStats {
   /* Estimate of the number of critical servers used by this host */
   Cardinality num_dns_servers, num_smtp_servers, num_ntp_servers;
 
-  /* Estimate the number of visited pages using HyperLogLog */
-  struct ndpi_hll visited_pages_hll;
-  double last_hll_visited_pages_value;
-  /* Holt-Winters structure, used to have a feedback regarding the visited pages */
-  BehaviouralCounter *visited_pages_hw;
-  bool hw_visited_pages_report;
+  /* Estimate the number of contacted hosts using HyperLogLog */
+  struct ndpi_hll hll_contacted_hosts;
+  double last_hll_contacted_hosts_value;
+  /* Holt-Winters structure, used to have a feedback regarding the contacted hosts */
+  BehaviouralCounter *hw_contacted_hosts;
+  bool hw_contacted_hosts_report;
   u_int16_t hw_learning_values;
   u_int8_t hw_init_count;
   u_int32_t prediction, lower_bound, upper_bound;
@@ -66,7 +66,7 @@ class LocalHostStats: public HostStats {
   void getCurrentTime(struct tm *t_now);
   void serializeDeserialize(char *host_buf, struct tm *t_now, bool do_serialize);
   void deserializeTopSites(char* redis_key_current);
-  void updateVisitedPagesHll();
+  void updateContactedHostsBehaviour();
   
  public:
   LocalHostStats(Host *_host);
@@ -93,6 +93,7 @@ class LocalHostStats: public HostStats {
   virtual void luaPeers(lua_State *vm);
   virtual void incrVisitedWebSite(char *hostname);
   virtual void lua_get_timeseries(lua_State* vm);
+  virtual void luaHostBehaviour(lua_State* vm);
   virtual bool hasAnomalies(time_t when);
   virtual void luaAnomalies(lua_State* vm, time_t when);
   virtual HTTPstats* getHTTPstats() { return(http); };

scripts/locales/en.lua

@@ -544,7 +544,10 @@ local lang = {
   },
   ["alerts_dashboard"] = {
     ["active_flows_anomaly"] = "Active Flows Anomaly",
-    ["alert"] = "Alert",
+    ["unexpected_host_behaviour_title"] = "Unexpected Host Behaviour",
+    ["sites_behaviour_description"] = "Triggers an alert when an host contacts too many or too few sites in contrast with its expected behaviour",
+    ["unexpected_host_behavior_description"] = "Unexpected behaviour from host %{host}. %{value} %{type_of_behaviour} while the expected value is %{prediction} with a lower bound of %{lower_bound} and upper bound of %{upper_bound}",
+    ["alert"] = "Alert",      		    
     ["alert_counts"] = "Counts",
     ["alert_duration"] = "Duration",
     ["alert_periodicity_update"] = "Flow Periodicity Changed",
@@ -2338,6 +2341,7 @@ local lang = {
     ["http_stats"] = "HTTP Stats",
     ["influxdb_not_responding"] = "Query has been aborted as InfluxDB is not responding. Query timeout can be configured from the <a href=\"%{url}\">%{flask_icon} Preferences</a> .",
     ["last_ms"] = "Last ms",
+    ["host_contacts_behaviour"] = "Contacts Behaviour",
     ["max"] = "Max",
     ["max_ms"] = "Max ms",
     ["max_rtt"] = "Max RTT Time",

scripts/lua/host_details.lua

@@ -2205,6 +2205,7 @@ graph_utils.drawGraphs(ifId, schema, tags, _GET["zoom"], url, selected_epoch, {
       {schema="host:alerted_flows",          label=i18n("graphs.total_alerted_flows")},
       {schema="host:unreachable_flows",      label=i18n("graphs.total_unreachable_flows")},
       {schema="host:contacts",               label=i18n("graphs.active_host_contacts")},
+      {schema="host:contacts_behaviour",     label=i18n("graphs.host_contacts_behaviour")},
       {schema="host:total_alerts",           label=i18n("details.alerts")},
       {schema="host:engaged_alerts",         label=i18n("show_alerts.engaged_alerts")},
       {schema="host:host_unreachable_flows", label=i18n("graphs.host_unreachable_flows")},

scripts/lua/modules/alert_definitions/alert_unexpected_behaviour.lua

@@ -0,0 +1,65 @@
+--
+-- (C) 2019-21 - ntop.org
+--
+
+-- ##############################################
+
+local alert_keys = require "alert_keys"
+local classes = require "classes"
+local alert = require "alert"
+
+-- ##############################################
+
+local alert_unexpected_behaviour = classes.class(alert)
+
+-- ##############################################
+
+alert_unexpected_behaviour.meta = {
+   alert_key = alert_keys.ntopng.alert_unexpected_behaviour,
+   i18n_title = "alerts_dashboard.unexpected_host_behaviour_title",
+   icon = "fas fa-exclamation",
+}
+
+-- ##############################################
+
+-- @brief Prepare an alert table used to generate the alert
+-- @param value       The value got from the measurement
+-- @param prediction  The value instead predicted
+-- @param lower_bound The lower bound of the measurement
+-- @param upper_bound The upper bound of the measurement
+-- @return A table with the alert built
+function alert_unexpected_behaviour:init(type_of_behaviour, value, prediction, upper_bound, lower_bound)
+   -- Call the parent constructor
+   self.super:init()
+
+   self.alert_type_params = {
+      type_of_behaviour = type_of_behaviour,
+      value = value,
+      prediction = prediction,
+      upper_bound = upper_bound,
+      lower_bound = lower_bound,
+   }
+end
+
+-- #######################################################
+
+-- @brief Format an alert into a human-readable string
+-- @param ifid The integer interface id of the generated alert
+-- @param alert The alert description table, including alert data such as the generating entity, timestamp, granularity, type
+-- @param alert_type_params Table `alert_type_params` as built in the `:init` method
+-- @return A human-readable string
+function alert_unexpected_behaviour.format(ifid, alert, alert_type_params)
+    return(i18n("alerts_dashboard.unexpected_host_behavior_description",
+		{
+		   host = firstToUpper(alert_consts.formatAlertEntity(ifid, alert_consts.alertEntityRaw(alert["alert_entity"]), alert["alert_entity_val"])),
+		   type_of_behaviour = alert_type_params.type_of_behaviour,
+		   value = alert_type_params.value,
+		   prediction = alert_type_params.prediction,
+		   upper_bound = alert_type_params.upper_bound,
+		   lower_bound = alert_type_params.lower_bound,
+		}))
+end
+
+-- #######################################################
+
+return alert_unexpected_behaviour

scripts/lua/modules/alert_keys.lua

@@ -113,7 +113,8 @@ local alert_keys = {
       alert_tcp_syn_scan_victim            = {NO_PEN, 98},
       alert_remote_to_local_insecure_proto = {NO_PEN, 99},
       alert_contacted_peers                = {NO_PEN, 100},
-      
+      alert_unexpected_behaviour           = {NO_PEN, 101},
+
       -- Add here additional keys for alerts generated
       -- by ntopng plugins
       -- WARNING: make sure integers do NOT OVERLAP with

scripts/lua/modules/timeseries/schemas/ts_5min.lua

@@ -417,6 +417,16 @@ schema:addMetric("num_as_server")
 
 -- ##############################################
 
+schema = ts_utils.newSchema("host:contacts_behaviour", {step=300, metrics_type=ts_utils.metrics.gauge})
+schema:addTag("ifid")
+schema:addTag("host")
+schema:addMetric("hll_value")
+schema:addMetric("hw_prediction")
+schema:addMetric("hw_upper_bound")
+schema:addMetric("hw_lower_bound")
+
+-- ##############################################
+
 schema = ts_utils.newSchema("host:l4protos", {step=300})
 schema:addTag("ifid")
 schema:addTag("host")

scripts/lua/modules/timeseries/ts_utils_core.lua

@@ -826,6 +826,7 @@ function ts_utils.getPossiblyChangedSchemas()
       "iface:alerted_flows",
 
       "host:contacts", -- split in "as_client" and "as_server"
+      "host:contacts_behaviour",
       "host:ndpi_categories", --split in "bytes_sent" and "bytes_rcvd"
 
       -- Added missing ifid tag

scripts/lua/modules/ts_5min_dump_utils.lua

@@ -355,6 +355,13 @@ function ts_dump.host_update_stats_rrds(when, hostname, host, ifstats, verbose)
             num_as_client=host["contacts.as_client"], num_as_server=host["contacts.as_server"]}, when)
   end
 
+  -- Contacted Hosts Behaviour
+  if host["contacted_hosts_behaviour"] and host["contacted_hosts_behaviour.hw_value"] then
+    ts_utils.append("host:contacts_behaviour", {ifid=ifstats.id, host=hostname,
+            hll_value=host["contacted_hosts_behaviour.hll_value"], hw_prediction=host["contacted_hosts_behaviour.prediction"], hw_lower_bound=host["contacted_hosts_behaviour.hw_lower_bound"], hw_upper_bound=host["contacted_hosts_behaviour.upper_bound"]}, when)
+  end
+
+
   -- L4 Protocols
   for id, _ in pairs(l4_keys) do
     k = l4_keys[id][2]

scripts/plugins/alerts/security/unexpected_host_behaviour/manifest.lua

@@ -0,0 +1,10 @@
+--
+-- (C) 2019-21 - ntop.org
+--
+
+return {
+  title = "Unexpected Host Behaviour",
+  description = "Detects if an host contacts an unexpected number of domains",
+  author = "ntop",
+  dependencies = {},
+}

scripts/plugins/alerts/security/unexpected_host_behaviour/user_scripts/host/unexpected_host_behaviour.lua

@@ -0,0 +1,70 @@
+--
+-- (C) 2019-21 - ntop.org
+--
+
+local user_scripts = require("user_scripts")
+local alert_severities = require "alert_severities"
+local alerts_api = require "alerts_api"
+local alert_consts = require("alert_consts")
+
+-- #################################################################
+
+local script = {
+   local_only = true,
+
+   default_enabled = true,
+   
+   -- Script category
+   category = user_scripts.script_categories.security, 
+
+   -- This script is only for alerts generation
+   is_alert = true,
+   
+   default_value = {
+      severity = alert_severities.warning,
+   },
+
+   -- NOTE: hooks defined below
+   hooks = {},
+
+   gui = {
+      i18n_title = "alerts_dashboard.unexpected_host_behaviour_title",
+      i18n_description = "alerts_dashboard.sites_behaviour_description",
+   }
+}
+
+-- #################################################################
+
+function script.hooks.min(params)
+   local stats = host.getBehaviourInfo() or nil
+
+   if stats then
+      if stats["contacted_hosts_behavior.hw_value"] ~= nil then 
+      	 local value = stats["contacted_hosts_behavior.hw_value"]	
+      	 local prediction = stats["contacted_hosts_behavior.hw_prediction"]
+      	 local estimated_value = stats["contacted_hosts_behavior.last_hll_estimate"]
+      	 local upper_bound = stats["contacted_hosts_behavior.hw_upper_bound"]
+      	 local lower_bound = stats["contacted_hosts_behavior.hw_lower_bound"]
+
+      	 local alert = alert_consts.alert_types.alert_unexpected_behaviour.new(
+            "Domain visited", -- Type of unexpected behaviour
+            estimated_value,
+            prediction,
+	    upper_bound,
+	    lower_bound
+         )
+
+         alert:set_severity(conf.severity)
+
+         if value == true then
+            alert:trigger(params.alert_entity, nil, params.cur_alerts)
+         else
+	    alert:release(params.alert_entity, nil, params.cur_alerts)
+         end
+      end
+   end
+end
+
+-- #################################################################
+
+return script

src/LocalHostStats.cpp

@@ -45,14 +45,14 @@ LocalHostStats::LocalHostStats(Host *_host) : HostStats(_host) {
   contacts_as_srv.init(4);                     /* 16 bytes  */
 
   /* hll init, 8 bits -> 256 bytes per LocalHost */
-  if(ndpi_hll_init(&visited_pages_hll, 8) != 0)
+  if(ndpi_hll_init(&hll_contacted_hosts, 8) != 0)
     throw "Failed HLL initialization";
   
-  last_hll_visited_pages_value = 0;
+  last_hll_contacted_hosts_value = 0;
   
   /* hw init */
-  visited_pages_hw = new HWCounter(12);
-  hw_visited_pages_report = false;
+  hw_contacted_hosts = new HWCounter(12);
+  hw_contacted_hosts_report = false;
   hw_init_count = 0;
   prediction = lower_bound = upper_bound = 0;
   
@@ -74,16 +74,15 @@ LocalHostStats::LocalHostStats(LocalHostStats &s) : HostStats(s) {
   num_contacts_as_cli = num_contacts_as_srv = 0;
 
   /* hll init, 8 bits -> 256 bytes per LocalHost */
-  if(ndpi_hll_init(&visited_pages_hll, 8))
+  if(ndpi_hll_init(&hll_contacted_hosts, 8))
     throw "Failed HLL initialization";
   
-  last_hll_visited_pages_value = 0;
+  last_hll_contacted_hosts_value = 0;
   hw_init_count = 0;
   
   /* hw init */
-  hw_learning_values = 12;
-  visited_pages_hw = (BehaviouralCounter *) new HWCounter(hw_learning_values);
-  hw_visited_pages_report = false;
+  hw_contacted_hosts = new HWCounter(12);
+  hw_contacted_hosts_report = false;
   prediction = lower_bound = upper_bound = 0;
   
   num_dns_servers.init(5);
@@ -100,9 +99,9 @@ LocalHostStats::~LocalHostStats() {
   if(http)               delete http;
   if(icmp)               delete icmp;
   if(peers)              delete(peers);
-  if(visited_pages_hw)   delete(visited_pages_hw);
+  if(hw_contacted_hosts) delete(hw_contacted_hosts);
 
-  ndpi_hll_destroy(&visited_pages_hll);
+  ndpi_hll_destroy(&hll_contacted_hosts);
 }
 
 /* *************************************** */
@@ -116,7 +115,7 @@ void LocalHostStats::incrVisitedWebSite(char *hostname) {
      ) {
 
     /* HyperLogLog update regarding visited sites */
-    ndpi_hll_add(&visited_pages_hll, hostname, sizeof(*hostname));
+    ndpi_hll_add(&hll_contacted_hosts, hostname, sizeof(*hostname));
 
     /* Top Sites update, done only if the preference is enabled */
     if(top_sites
@@ -151,9 +150,9 @@ void LocalHostStats::updateStats(const struct timeval *tv) {
     nextContactsUpdate = tv->tv_sec+HOST_CONTACTS_REFRESH;
   }
   
-  if(nextPeriodicUpdate > 0 && (tv->tv_sec >= nextPeriodicUpdate)) {
+  if(tv->tv_sec >= nextPeriodicUpdate) {
     /* hll visited sites update */
-    updateVisitedPagesHll();
+    updateContactedHostsBehaviour();
     
     /* Top Sites update */
     if(top_sites && ntop->getPrefs()->are_top_talkers_enabled()) {
@@ -198,6 +197,30 @@ void LocalHostStats::getJSONObject(json_object *my_object, DetailsLevel details_
 
 /* *************************************** */
 
+void LocalHostStats::luaHostBehaviour(lua_State* vm) {
+  lua_newtable(vm);
+    
+  lua_push_float_table_entry(vm, "hll_value",
+			     last_hll_contacted_hosts_value);
+
+  if(hw_contacted_hosts) {
+    lua_push_bool_table_entry(vm, "hw_value",
+			      hw_contacted_hosts_report);
+    lua_push_int32_table_entry(vm, "hw_prediction",
+			       prediction); 
+    lua_push_int32_table_entry(vm, "hw_lower_bound",
+			       lower_bound);
+    lua_push_int32_table_entry(vm, "hw_upper_bound",
+			       upper_bound);
+  }
+
+  lua_pushstring(vm, "contacted_hosts_behavior");
+  lua_insert(vm, -2);
+  lua_settable(vm, -3);
+}
+
+/* *************************************** */
+
 void LocalHostStats::lua(lua_State* vm, bool mask_host, DetailsLevel details_level) {
   HostStats::lua(vm, mask_host, details_level);
 
@@ -211,12 +234,8 @@ void LocalHostStats::lua(lua_State* vm, bool mask_host, DetailsLevel details_lev
       lua_push_str_table_entry(vm, "sites.old", old_sites ? old_sites : (char*)"{}");
   }
 
-  lua_push_float_table_entry(vm, "last_hll_visited_pages_estimate",
-			     last_hll_visited_pages_value);
+  luaHostBehaviour(vm);
   
-  lua_push_bool_table_entry(vm, "last_hw_visited_pages_report",
-			    hw_visited_pages_report);
-    
   if(details_level >= details_high) {
     luaICMP(vm,host->get_ip()->isIPv4(),true);
     luaDNS(vm, true);
@@ -585,14 +604,11 @@ void LocalHostStats::resetTopSitesData() {
 
 /* *************************************** */
 
-void LocalHostStats::updateVisitedPagesHll() {
-  last_hll_visited_pages_value = ndpi_hll_count(&visited_pages_hll);
+void LocalHostStats::updateContactedHostsBehaviour() {
+  last_hll_contacted_hosts_value = ndpi_hll_count(&hll_contacted_hosts);
 
-  ndpi_hll_reset(&visited_pages_hll);
+  ndpi_hll_reset(&hll_contacted_hosts);
 
-  if(visited_pages_hw) 
-    hw_visited_pages_report = visited_pages_hw->addObservation((u_int32_t) last_hll_visited_pages_value, &prediction, &lower_bound, &upper_bound);
-
-  if(hw_init_count < hw_learning_values)
-    hw_init_count++;
+  if(hw_contacted_hosts) 
+    hw_contacted_hosts_report = hw_contacted_hosts->addObservation((u_int32_t) last_hll_contacted_hosts_value, &prediction, &lower_bound, &upper_bound);
 }

src/LuaEngineHost.cpp

@@ -415,6 +415,8 @@ static int ntop_host_get_ts_key(lua_State* vm) {
 static int ntop_host_get_behaviour_info(lua_State* vm) {
   Host *h = ntop_host_get_context_host(vm);
 
+  lua_newtable(vm);
+
   if(h)
     h->luaHostBehaviour(vm);