Rework host labels: serialize per interface, keep a LabelsConfiguration per NetworkInterface

httpdocs/dist

@@ -1 +1 @@
-Subproject commit e1d15c350189037f7603168e5d11d6a92688cced
+Subproject commit cb041357570d6813119c1509a837c23701766f06

include/NetworkInterface.h

@@ -207,6 +207,7 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity {
 
   string ip_addresses;
   AddressTree interface_networks;
+  LabelsConfiguration host_labels;
   int id;
   bool bridge_interface;
 
@@ -487,6 +488,11 @@ class NetworkInterface : public NetworkInterfaceAlertableEntity {
     ip_reassignment_alerts_enabled = status;
   };
   inline AddressTree* getInterfaceNetworks() { return (&interface_networks); };
+  u_int64_t getHostLabels(Host* host);
+  void setHostLabels(Host* host, u_int64_t bitmap);
+  /* Key-based overload used when Host is not available (e.g. inactive hosts from Lua) */
+  inline u_int64_t getHostLabels(const char* key) { return host_labels.getLabels(key); };
+  inline void setHostLabels(const char* key, u_int64_t bitmap) { host_labels.setLabels(key, bitmap); };
   virtual void startPacketPolling();
   virtual void startFlowDumping();
   virtual bool isLoading() { return false; };

include/Ntop.h

@@ -137,9 +137,6 @@ class Ntop {
   char* local_network_aliases[CONST_MAX_NUM_NETWORKS];
   AddressTree local_network_tree, cloud_local_network_tree;
 
-  /* Host label bitmaps preloaded from Redis at startup */
-  AddressTree host_labels_tree;
-
   /* Threads info */
   struct ThreadInfo {
     std::string name;
@@ -184,7 +181,6 @@ class Ntop {
   bool addLocalNetwork(char* _net);
 
   void loadLocalInterfaceAddress();
-  void loadHostLabels();
   void initAllowedProtocolPresets();
 
   void reloadASNPrefsOnInterfaces();
@@ -510,10 +506,6 @@ class Ntop {
   inline Redis* getRedis() { return (redis); };
   inline TimelineExtract* getTimelineExtract() { return (extract); };
 
-  u_int64_t getHostLabels(const char* ip_str);
-  u_int64_t getHostLabels(const IpAddress* ip);
-  void setHostLabels(const char* ip_str, u_int64_t bitmap);
-
 #ifdef HAVE_ZMQ
 #ifndef HAVE_NEDGE
   inline ExportInterface* get_export_interface() { return (export_interface); };

include/ntop_includes.h

@@ -225,6 +225,7 @@ using namespace std;
 #include "AddressTree.h"
 #include "VLANAddressTree.h"
 #include "ServerConfiguration.h"
+#include "LabelsConfiguration.h"
 #include "ASNConfiguration.h"
 #include "BroadcastDomains.h"
 #include "Cardinality.h"

scripts/lua/host_details.lua

@@ -731,7 +731,7 @@ else
          print("</td></tr>")
 
          -- Labels row
-         local host_labels_bitmap = ntop.getHostLabels(host_ip)
+         local host_labels_bitmap = interface.getHostLabels(host_ip)
          if host_labels_bitmap and host_labels_bitmap ~= 0 then
             local all_labels = label_badge_utils.getLabels()
             local badges_html = ""

scripts/lua/rest/v2/get/host/config.lua

@@ -30,7 +30,7 @@ local rsp = {
     host_pool_id = host["host_pool_id"],
     has_traffic_policies = ifstats.inline and (host.localhost or host.systemhost),
     drop_traffic = ntop.getHashCache("ntopng.prefs.drop_host_traffic", host_key),
-    labels = ntop.getHostLabels(host_info["host"])
+    labels = interface.getHostLabels(host_info["host"])
 }
 
 local rc = rest_utils.consts.success.ok

scripts/lua/rest/v2/set/host/config.lua

@@ -69,7 +69,7 @@ end
 
 if (_POST["host_labels_bitmap"] ~= nil) and isAdministrator() then
     local labels_bitmap = tonumber(_POST["host_labels_bitmap"]) or 0
-    ntop.setHostLabels(host_info["host"], labels_bitmap)
+    interface.setHostLabels(host_info["host"], labels_bitmap)
 end
 
 local rc = rest_utils.consts.success.ok

src/Host.cpp

@@ -279,7 +279,7 @@ void Host::initialize(Mac* _mac, int32_t _iface_idx, u_int16_t _vlanId,
   name_reset_requested = 0, prefs_loaded = 0;
   host_services_bitmap = 0, disabled_alerts_tstamp = 0, num_remote_access = 0,
   num_incomplete_flows = 0, deferred_init = 0;
-  labels_bitmap = ntop->getHostLabels(&ip);
+  labels_bitmap = 0;
 
   num_resolve_attempts = 0, nextResolveAttempt = 0,
   num_active_flows_as_client = 0, num_active_flows_as_server = 0,
@@ -1160,10 +1160,8 @@ char* Host::get_host_label(char* const buf, ssize_t buf_len) {
 /* ***************************************** */
 
 void Host::setLabels(u_int64_t bitmap) {
-  char ip_buf[64];
-
   labels_bitmap = bitmap;
-  ntop->setHostLabels(ip.print(ip_buf, sizeof(ip_buf)), bitmap);
+  iface->setHostLabels(this, bitmap);
 }
 
 /* ***************************************** */

src/LocalHost.cpp

@@ -115,6 +115,7 @@ void LocalHost::set_hash_entry_state_idle() {
  * initialization */
 void LocalHost::initialize() {
   char buf[64], host[96], rsp[256];
+
   BroadcastDomains* bd = iface->getBroadcastDomains();
 
   stats = allocateStats();
@@ -203,7 +204,12 @@ void LocalHost::initialize() {
 
 /* *************************************** */
 
-void LocalHost::deferredInitialization() { Host::deferredInitialization(); }
+void LocalHost::deferredInitialization() {
+
+  Host::deferredInitialization(); /* this also sets is_in_broadcast_domain */
+
+  labels_bitmap = iface->getHostLabels(this); 
+}
 
 /* *************************************** */
 

src/LuaEngineInterface.cpp

@@ -5403,6 +5403,79 @@ static int ntop_interface_release_engaged_alerts(lua_State* vm) {
 
 /* ****************************************** */
 
+static int ntop_interface_get_host_labels(lua_State* vm) {
+  NetworkInterface* iface = getCurrentInterface(vm);
+  Host *host;
+  char *host_ip, buf[64];
+  u_int16_t vlan_id = 0;
+  u_int64_t bitmap = 0;
+
+  ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__);
+
+  if (!iface)
+    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
+  if (ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING) != CONST_LUA_OK)
+    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
+
+  get_host_vlan_info((char*)lua_tostring(vm, 1), &host_ip, &vlan_id, buf,
+                     sizeof(buf));
+
+  host = iface->findHostByIP(get_allowed_nets(vm), host_ip, vlan_id,
+                                   getLuaVMUservalue(vm, observationPointId));
+
+  if (host) { /* Host online - read from host */
+    bitmap = host->getLabels();
+  } else { /* Host offline - lookup on redis */
+    char key_buf[CONST_MAX_LEN_REDIS_KEY];
+    snprintf(key_buf, sizeof(key_buf), HOST_SERIALIZED_SHORT_KEY,
+             iface->get_id(), host_ip, vlan_id);
+    bitmap = iface->getHostLabels(key_buf);
+  }
+
+  lua_pushinteger(vm, (lua_Integer)bitmap);
+  return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_ONE_RETURN_VALUE));
+}
+
+/* ****************************************** */
+
+static int ntop_interface_set_host_labels(lua_State* vm) {
+  NetworkInterface* iface = getCurrentInterface(vm);
+  Host *host;
+  char *host_ip, buf[64];
+  u_int16_t vlan_id = 0;
+  u_int64_t bitmap;
+
+  ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__);
+
+  if (!iface)
+    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
+  if (ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING) != CONST_LUA_OK)
+    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
+  if (ntop_lua_check(vm, __FUNCTION__, 2, LUA_TNUMBER) != CONST_LUA_OK)
+    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
+
+  get_host_vlan_info((char*)lua_tostring(vm, 1), &host_ip, &vlan_id, buf,
+                     sizeof(buf));
+  bitmap = (u_int64_t)lua_tonumber(vm, 2);
+
+  host = iface->findHostByIP(get_allowed_nets(vm), host_ip, vlan_id,
+                                   getLuaVMUservalue(vm, observationPointId));
+
+  if (host) { /* Host online - set to the host (host will update redis) */
+    host->setLabels(bitmap);
+  } else { /* Host offline - set on redis */
+    char key_buf[CONST_MAX_LEN_REDIS_KEY];
+    snprintf(key_buf, sizeof(key_buf), HOST_SERIALIZED_SHORT_KEY,
+             iface->get_id(), host_ip, vlan_id);
+    iface->setHostLabels(key_buf, bitmap);
+  }
+
+  lua_pushnil(vm);
+  return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_ONE_RETURN_VALUE));
+}
+
+/* ****************************************** */
+
 /* @brief Returns classification attributes for a host (device type, OS, category).  Lua: interface.getHostAttributes(host, vlan) → table */
 static int ntop_interface_get_host_attributes(lua_State* vm) {
   NetworkInterface* iface = getCurrentInterface(vm);
@@ -6705,6 +6778,8 @@ static luaL_Reg _ntop_interface_reg[] = {
     {"updateIPReassignment", ntop_interface_update_ip_reassignment},
     {"triggerTrafficAlert", ntop_interface_trigger_traffic_alert},
     {"getHostAttributes", ntop_interface_get_host_attributes},
+    {"getHostLabels", ntop_interface_get_host_labels},
+    {"setHostLabels", ntop_interface_set_host_labels},
 
     {"addDataToLocalHostAssets", ntop_add_data_to_assets},
     {"removeDataFromLocalHostAssets", ntop_remove_data_from_assets},

src/LuaEngineNtop.cpp

@@ -6588,61 +6588,6 @@ static int ntop_set_hash_redis(lua_State* vm) {
 /* ****************************************** */
 
 /* @brief Returns the label bitmap for the host identified by the given IP. */
-static int ntop_get_host_labels(lua_State* vm) {
-  char *host_ip, buf[64];
-  u_int16_t vlan_id = 0;
-
-  ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__);
-
-  if (ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING) != CONST_LUA_OK)
-    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
-  if ((host_ip = (char*)lua_tostring(vm, 1)) == NULL)
-    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
-
-  get_host_vlan_info(host_ip, &host_ip, &vlan_id, buf, sizeof(buf));
-
-  lua_pushinteger(vm, (lua_Integer)ntop->getHostLabels(host_ip));
-  return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_ONE_RETURN_VALUE));
-}
-
-/* ****************************************** */
-
-/* @brief Sets the label bitmap for the host identified by the given IP.
- *        Updates the radix tree, Redis, and any active host. */
-static int ntop_set_host_labels(lua_State* vm) {
-  char *host_ip, buf[64];
-  u_int16_t vlan_id = 0;
-  u_int64_t bitmap;
-
-  ntop->getTrace()->traceEvent(TRACE_DEBUG, "%s() called", __FUNCTION__);
-
-  if (ntop_lua_check(vm, __FUNCTION__, 1, LUA_TSTRING) != CONST_LUA_OK)
-    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
-  if ((host_ip = (char*)lua_tostring(vm, 1)) == NULL)
-    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
-  if (ntop_lua_check(vm, __FUNCTION__, 2, LUA_TNUMBER) != CONST_LUA_OK)
-    return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_NO_RETURN_VALUE));
-  bitmap = (u_int64_t)lua_tonumber(vm, 2);
-
-  get_host_vlan_info(host_ip, &host_ip, &vlan_id, buf, sizeof(buf));
-
-  /* Update radix tree and Redis */
-  ntop->setHostLabels(host_ip, bitmap);
-
-  /* Update active hosts in memory */
-  for (int i = 0; i < ntop->get_num_interfaces(); i++) {
-    NetworkInterface* iface = ntop->getInterface(i);
-    if (iface) {
-      Host* host = iface->findHostByIP(get_allowed_nets(vm), host_ip, vlan_id,
-                                       getLuaVMUservalue(vm, observationPointId));
-      if (host) host->setLabels(bitmap);
-    }
-  }
-
-  lua_pushnil(vm);
-  return (ntop_lua_return_value(vm, __FUNCTION__, CONST_LUA_ONE_RETURN_VALUE));
-}
-
 /* ****************************************** */
 
 static void ntop_reset_host_name(lua_State* vm, char* address) {
@@ -9322,8 +9267,6 @@ static luaL_Reg _ntop_reg[] = {
     {"getKeysCache", ntop_get_keys_redis},
 
     /* Host Labels */
-    {"getHostLabels", ntop_get_host_labels},
-    {"setHostLabels", ntop_set_host_labels},
     {"hasDumpCache", ntop_redis_has_dump},
     {"dumpCache", ntop_redis_dump},
     {"restoreCache", ntop_redis_restore},

src/NetworkInterface.cpp

@@ -9707,6 +9707,28 @@ void NetworkInterface::allocateStructures(bool disable_dump) {
   setEntityValue(buf);
   reloadGwMacs();
   removeRedisSitesKey();
+
+  if (id >= 0)
+    host_labels.loadFromRedis(id);
+}
+
+/* **************************************** */
+
+u_int64_t NetworkInterface::getHostLabels(Host* host) {
+  Mac* mac = host->getMac();
+  if (mac) {
+    u_int64_t v = host_labels.getLabels(mac->get_mac());
+    if (v) return v;
+  }
+  return host_labels.getLabels(host->get_ip(), host->get_vlan_id());
+}
+
+/* **************************************** */
+
+void NetworkInterface::setHostLabels(Host* host, u_int64_t bitmap) {
+  char key_buf[CONST_MAX_LEN_REDIS_KEY];
+  char* key = host->getSerializationKey(key_buf, sizeof(key_buf), true);
+  host_labels.setLabels(key, bitmap);
 }
 
 /* **************************************** */

src/Ntop.cpp

@@ -620,7 +620,6 @@ void Ntop::start() {
   FlowRiskAlerts::checkUndefinedRisks();
 
   loadLocalInterfaceAddress();
-  loadHostLabels();
 
   address->startResolveAddressLoop();
 
@@ -892,83 +891,6 @@ char* Ntop::getIfName(int if_id, char* name, u_int name_len) {
 
 /* ******************************************* */
 
-/*
- * Read from Redis for all labels and build host_labels_tree
- * so that Host::initialize() can initialize labels without a per-host Redis request.
- */
-void Ntop::loadHostLabels() {
-  char pattern[128];
-  char **keys = NULL;
-  int nkeys;
-
-  snprintf(pattern, sizeof(pattern), "%s*", HOST_LABELS_BITMAP_PREFIX);
-  nkeys = redis->keys(pattern, &keys);
-
-  for (int i = 0; i < nkeys; i++) {
-    char val_buf[32];
-
-    if (redis->get(keys[i], val_buf, sizeof(val_buf)) == 0 && val_buf[0] != '\0') {
-      u_int64_t bitmap = (u_int64_t)strtoull(val_buf, NULL, 10);
-
-      if (bitmap != 0) {
-        const char *ip_str = keys[i] + strlen(HOST_LABELS_BITMAP_PREFIX);
-        host_labels_tree.addAddress(ip_str, (int64_t)bitmap);
-      }
-    }
-
-    if (keys[i]) free(keys[i]);
-  }
-
-  if (keys) free(keys);
-
-  ntop->getTrace()->traceEvent(TRACE_NORMAL, "Loaded %d host label bitmap(s) from Redis", nkeys);
-}
-
-/* ******************************************* */
-
-u_int64_t Ntop::getHostLabels(const char* ip_str) {
-  int64_t val = host_labels_tree.find(ip_str);
-  /* Note: find() returns -1 when the address is not in the tree */
-  return (val == -1) ? 0 : (u_int64_t)val;
-}
-
-/* ******************************************* */
-
-u_int64_t Ntop::getHostLabels(const IpAddress* ip) {
-  int64_t val;
-
-  if (ip->isIPv4()) {
-    u_int32_t addr = ip->get_ipv4(); /* network byte order, as stored in tree */
-    val = host_labels_tree.findAddress(AF_INET, &addr, NULL);
-  } else if (ip->isIPv6()) {
-    val = host_labels_tree.findAddress(AF_INET6,
-                                       (void*)ip->get_ipv6(), NULL);
-  } else {
-    return 0;
-  }
-
-  return (val == -1) ? 0 : (u_int64_t)val;
-}
-
-/* ******************************************* */
-
-void Ntop::setHostLabels(const char* ip_str, u_int64_t bitmap) {
-  char redis_key[CONST_MAX_LEN_REDIS_KEY];
-  char val_buf[32];
-
-  /* Update radix tree (in memory) */
-  host_labels_tree.addAddress(ip_str, (int64_t)bitmap);
-
-  /* Update Redis (persistent) */
-  snprintf(redis_key, sizeof(redis_key), HOST_LABELS_BITMAP_KEY, ip_str);
-  if (bitmap == 0)
-    redis->del(redis_key);
-  else {
-    snprintf(val_buf, sizeof(val_buf), "%llu", (unsigned long long)bitmap);
-    redis->set(redis_key, val_buf);
-  }
-}
-
 /* ******************************************* */
 
 void Ntop::loadLocalInterfaceAddress() {

src/RemoteHost.cpp

@@ -85,4 +85,6 @@ void RemoteHost::initialize() {
 
   iface->incNumHosts(
       this, true /* Initialization: bytes are 0, considered RX only */);
+
+  labels_bitmap = iface->getHostLabels(this);
 }

third-party/clickhouse-cpp

@@ -1 +1 @@
-Subproject commit e22153cda89585e1a74409dc04e85b0b754308ca
+Subproject commit c441ec8a68013157d18ed77b4ee5ae40060936af