Add Use Host Pools for Local Host Detection preference to mark as local all hosts in pools

include/Prefs.h

@@ -231,6 +231,7 @@ class Prefs {
   bool print_version, print_version_json;
   bool snmp_polling;
   bool active_monitoring, network_discovery, starttls;
+  bool use_host_pools_for_local;
   bool dump_pcap_to_clickhouse; /* Dump pcap-interface flows to ClickHouse */
   bool query_performance_log;
 
@@ -870,6 +871,7 @@ class Prefs {
   ASNConfiguration* getRemoteASN() { return (remote_asn); }
 
   inline bool useMacAddressInFlowKey() { return (use_mac_in_flow_key); }
+  inline bool useHostPoolsForLocal() const { return use_host_pools_for_local; }
   inline bool areFingerprintStatsEnabled() { return (fingerprint_stats); }
   inline bool doReforgeTimestamps() { return (do_reforge_timestamps); }
   inline void enableVLANCloudToExporters() {

include/ntop_defines.h

@@ -866,6 +866,8 @@
 
 #define CONST_PREFS_USE_MAC_IN_FLOW_KEY \
   NTOPNG_PREFS_PREFIX ".use_mac_in_flow_key"
+#define CONST_PREFS_USE_HOST_POOLS_FOR_LOCAL \
+  NTOPNG_PREFS_PREFIX ".use_host_pools_for_local"
 #define CONST_PREFS_FINGERPRINT_STATS NTOPNG_PREFS_PREFIX ".fingerprint_stats"
 
 #define CONST_PREFS_NETWORK_DISCOVERY_DEBUG \

scripts/locales/en.lua

@@ -8284,6 +8284,8 @@ local lang = {
     ["toggle_split_ts_direction_description"] = "Toggle between displaying a single total traffic timeseries (RX + TX) and separate timeseries for RX and TX. When the 'Total' option is selected (default), a combined timeseries is generated. Otherwise, separate timeseries are created for each direction (RX and TX).",
     ["toggle_use_mac_in_flow_key_description"] = "Use the MAC Address to generate the flow key. This is suggested in case the same IP can be seen using different MAC Addresses (e.g. a load balancer).",
     ["toggle_use_mac_in_flow_key_title"] = "Use MAC Address in Flow Key",
+    ["toggle_use_host_pools_for_local_description"] = "Mark as local hosts all hosts belonging to any Host Pool other than the Default pool, regardless of the configured Local Networks. An ntopng restart is required to refresh all active hosts.",
+    ["toggle_use_host_pools_for_local_title"] = "Use Host Pools for Local Host Detection",
     ["toggle_users_rrds_description"] = "Toggle the creation of bytes and applications timeseries for defined users.",
     ["toggle_users_rrds_title"] = "Users",
     ["toggle_vlan_rrds_description"] = "Toggle the creation of bytes and applications timeseries for VLANs.",

scripts/lua/admin/prefs.lua

@@ -953,6 +953,12 @@ if auth.has_capability(auth.capabilities.preferences) then
                 default = "0",
                 pref = "use_mac_in_flow_key"
             })
+
+            prefsToggleButton(subpage_active, {
+                field = "toggle_use_host_pools_for_local",
+                default = "0",
+                pref = "use_host_pools_for_local"
+            })
         end
 
         print('<thead class="table-primary"><tr><th colspan=2 class="info">' .. i18n("prefs.flow_table") ..

scripts/lua/modules/http_lint.lua

@@ -2435,6 +2435,7 @@ local known_parameters = {
    ["toggle_src_with_post_nat_src"] = validateBool,
    ["toggle_behaviour_analysis"] = validateBool,
    ["toggle_use_mac_in_flow_key"] = validateBool,
+   ["toggle_use_host_pools_for_local"] = validateBool,
    ["toggle_fingerprint_stats"] = validateBool,
    ["toggle_starttls"] = validateBool,
    ["toggle_dump_pcap_to_clickhouse"] = validateBool,

scripts/lua/modules/prefs_menu.lua

@@ -459,6 +459,10 @@ local menu_subpages = {{
             title = i18n("prefs.toggle_use_mac_in_flow_key_title"),
             description = i18n("prefs.toggle_use_mac_in_flow_key_description")
         },
+        toggle_use_host_pools_for_local = {
+            title = i18n("prefs.toggle_use_host_pools_for_local_title"),
+            description = i18n("prefs.toggle_use_host_pools_for_local_description")
+        },
         topk_heuristic_precision = {
             title = i18n("prefs.topk_heuristic_precision_title"),
             description = i18n("prefs.topk_heuristic_precision_description")

src/NetworkInterface.cpp

@@ -4396,9 +4396,22 @@ void NetworkInterface::findFlowHosts(int32_t iface_idx, u_int16_t vlan_id,
       return;
     }
 
-    if (_src_ip &&
-        (_src_ip->isLocalHost() || _src_ip->isLocalInterfaceAddress() ||
-         ntop->isInLocalASN(_src_ip))) {
+    bool src_is_local =
+        _src_ip && (_src_ip->isLocalHost() ||
+                    _src_ip->isLocalInterfaceAddress() ||
+                    ntop->isInLocalASN(_src_ip));
+
+    if (!src_is_local && _src_ip &&
+        ntop->getPrefs()->useHostPoolsForLocal() && host_pools) {
+      u_int16_t src_pool_id;
+      ndpi_patricia_node_t* src_node;
+      if (host_pools->findIpPool(_src_ip, vlan_id, &src_pool_id, &src_node) &&
+          src_pool_id != NO_HOST_POOL_ID &&
+          src_pool_id != DROP_HOST_POOL_ID)
+        src_is_local = true;
+    }
+
+    if (src_is_local) {
       INTERFACE_PROFILING_SECTION_ENTER(
           "NetworkInterface::findFlowHosts: new LocalHost", 4);
       (*src) = new (std::nothrow) LocalHost(this, iface_idx, src_mac, vlan_id,
@@ -4447,9 +4460,22 @@ void NetworkInterface::findFlowHosts(int32_t iface_idx, u_int16_t vlan_id,
       return;
     }
 
-    if (_dst_ip &&
-        (_dst_ip->isLocalHost() || _dst_ip->isLocalInterfaceAddress() ||
-         ntop->isInLocalASN(_dst_ip))) {
+    bool dst_is_local =
+        _dst_ip && (_dst_ip->isLocalHost() ||
+                    _dst_ip->isLocalInterfaceAddress() ||
+                    ntop->isInLocalASN(_dst_ip));
+
+    if (!dst_is_local && _dst_ip &&
+        ntop->getPrefs()->useHostPoolsForLocal() && host_pools) {
+      u_int16_t dst_pool_id;
+      ndpi_patricia_node_t* dst_node;
+      if (host_pools->findIpPool(_dst_ip, vlan_id, &dst_pool_id, &dst_node) &&
+          dst_pool_id != NO_HOST_POOL_ID &&
+          dst_pool_id != DROP_HOST_POOL_ID)
+        dst_is_local = true;
+    }
+
+    if (dst_is_local) {
       INTERFACE_PROFILING_SECTION_ENTER(
           "NetworkInterface::findFlowHosts: new LocalHost", 4);
       (*dst) = new (std::nothrow) LocalHost(this, iface_idx, dst_mac, vlan_id,

src/Prefs.cpp

@@ -93,6 +93,7 @@ Prefs::Prefs(Ntop* _ntop) {
   auto_assigned_pool_id = NO_HOST_POOL_ID;
   default_l7policy = PASS_ALL_SHAPER_ID;
   use_mac_in_flow_key = false;
+  use_host_pools_for_local = false;
   fingerprint_stats = false;
   ciphers_list = NULL;
   device_protocol_policies_enabled = false, enable_vlan_trunk_bridge = false;
@@ -983,6 +984,8 @@ void Prefs::reloadPrefsFromRedis() {
   enable_sql_log = getDefaultBoolPrefsValue(CONST_PREFS_ENABLE_SQL_LOG, false);
   use_mac_in_flow_key =
       getDefaultPrefsValue(CONST_PREFS_USE_MAC_IN_FLOW_KEY, false);
+  use_host_pools_for_local =
+      getDefaultBoolPrefsValue(CONST_PREFS_USE_HOST_POOLS_FOR_LOCAL, false);
   fingerprint_stats =
       getDefaultPrefsValue(CONST_PREFS_FINGERPRINT_STATS, false);
   // vulnerability scan preferences
@@ -3099,6 +3102,8 @@ void Prefs::lua(lua_State* vm) {
                             enable_flow_device_port_rrd_creation);
 
   lua_push_bool_table_entry(vm, "are_alerts_enabled", !disable_alerts);
+  lua_push_bool_table_entry(vm, "use_host_pools_for_local",
+                            use_host_pools_for_local);
   lua_push_bool_table_entry(vm, "is_arp_matrix_generation_enabled",
                             is_arp_matrix_generation_enabled());
   lua_push_bool_table_entry(vm, "is_users_login_enabled", enable_users_login);

third-party/clickhouse-cpp

@@ -1 +1 @@
-Subproject commit e22153cda89585e1a74409dc04e85b0b754308ca
+Subproject commit 1024efe8bbdbe199fdc89bbf7aba7fde92f7a53b