diff --git a/src/FrequentNumericItems.cpp b/attic/FrequentNumericItems.cpp similarity index 100% rename from src/FrequentNumericItems.cpp rename to attic/FrequentNumericItems.cpp diff --git a/include/FrequentNumericItems.h b/attic/FrequentNumericItems.h similarity index 100% rename from include/FrequentNumericItems.h rename to attic/FrequentNumericItems.h diff --git a/src/FrequentTrafficItems.cpp b/attic/FrequentTrafficItems.cpp similarity index 100% rename from src/FrequentTrafficItems.cpp rename to attic/FrequentTrafficItems.cpp diff --git a/include/FrequentTrafficItems.h b/attic/FrequentTrafficItems.h similarity index 100% rename from include/FrequentTrafficItems.h rename to attic/FrequentTrafficItems.h diff --git a/include/FrequentStringItems.h b/include/FrequentStringItems.h index edb67f6dbc..8658c0faf8 100644 --- a/include/FrequentStringItems.h +++ b/include/FrequentStringItems.h @@ -25,30 +25,21 @@ /* https://resources.sei.cmu.edu/asset_files/Presentation/2010_017_001_49763.pdf */ -typedef struct { - char *key; - u_int32_t value; - UT_hash_handle hh; /* makes this structure hashable */ -} FrequentStringKey_t; - /* *************************************** */ class FrequentStringItems { private: u_int32_t max_items, max_items_threshold; - FrequentStringKey_t *q; + std::map q; Mutex m; bool thread_safe; - void cleanup(); void prune(); public: - FrequentStringItems(u_int32_t _max_items, bool _thread_safe = true) { max_items =_max_items, max_items_threshold = 2*_max_items, q = NULL, thread_safe = _thread_safe; } - ~FrequentStringItems(); + FrequentStringItems(u_int32_t _max_items, bool _thread_safe = true) { max_items =_max_items, max_items_threshold = 2*_max_items, thread_safe = _thread_safe; } void add(char *key, u_int32_t value); - void print(); char* json(); }; diff --git a/include/NetworkInterface.h b/include/NetworkInterface.h index 8b8ad39e66..b5871cb195 100644 --- a/include/NetworkInterface.h +++ b/include/NetworkInterface.h @@ -52,12 +52,6 @@ class TrafficShaper; class NIndexFlowDB; #endif -typedef struct { - u_int32_t criteria; /* IP address, interface... */ - NetworkInterface *iface; - UT_hash_handle hh; /* makes this structure hashable */ -} FlowHashing; - /** @class NetworkInterface * @brief Main class of network interface of ntopng. * @details ....... @@ -100,9 +94,9 @@ class NetworkInterface : public AlertableEntity { /* Disaggregations */ u_int16_t numSubInterfaces; - set flowHashingIgnoredInterfaces; + std::set flowHashingIgnoredInterfaces; FlowHashingEnum flowHashingMode; - FlowHashing *flowHashing; + std::map flowHashing; /* Network Discovery */ NetworkDiscovery *discovery; @@ -207,7 +201,7 @@ class NetworkInterface : public AlertableEntity { void init(); void deleteDataStructures(); - NetworkInterface* getDynInterface(u_int32_t criteria, bool parser_interface); + NetworkInterface* getDynInterface(u_int64_t criteria, bool parser_interface); Flow* getFlow(Mac *srcMac, Mac *dstMac, u_int16_t vlan_id, u_int32_t deviceIP, u_int16_t inIndex, u_int16_t outIndex, const ICMPinfo * const icmp_info, @@ -370,7 +364,7 @@ class NetworkInterface : public AlertableEntity { inline void incLostPkts(u_int32_t num) { tcpPacketStats.incLost(num); }; inline void incKeepAlivePkts(u_int32_t num) { tcpPacketStats.incKeepAlive(num); }; virtual void checkPointCounters(bool drops_only); - bool registerSubInterface(NetworkInterface *sub_iface, u_int32_t criteria); + bool registerSubInterface(NetworkInterface *sub_iface, u_int64_t criteria); u_int32_t checkDroppedAlerts(); /* Overridden in ViewInterface.cpp */ diff --git a/include/ntop_defines.h b/include/ntop_defines.h index af406016e7..bfb0134152 100644 --- a/include/ntop_defines.h +++ b/include/ntop_defines.h @@ -638,6 +638,7 @@ #define DISAGGREGATION_PROBE_IP "probe_ip" #define DISAGGREGATION_IFACE_ID "iface_idx" #define DISAGGREGATION_INGRESS_IFACE_ID "ingress_iface_idx" +#define DISAGGREGATION_INGRESS_PROBE_IP_AND_IFACE_ID "probe_ip_and_ingress_iface_idx" #define DISAGGREGATION_INGRESS_VRF_ID "ingress_vrf_id" #define DISAGGREGATION_VLAN "vlan" #define DISAGGREGATION_NONE "none" diff --git a/include/ntop_includes.h b/include/ntop_includes.h index 313bbaf0f5..6d2597c6d2 100644 --- a/include/ntop_includes.h +++ b/include/ntop_includes.h @@ -258,9 +258,6 @@ using namespace std; #include "LdapAuthenticator.h" #endif #endif -#include "FrequentStringItems.h" -#include "FrequentNumericItems.h" -#include "FrequentTrafficItems.h" #include "HostPoolStats.h" #include "HostPools.h" #include "Fingerprint.h" @@ -351,8 +348,6 @@ using namespace std; #endif #include "LuaHandler.h" #include "FrequentStringItems.h" -#include "FrequentNumericItems.h" -#include "FrequentTrafficItems.h" #ifdef HAVE_NEDGE #include "NetfilterInterface.h" #endif diff --git a/include/ntop_typedefs.h b/include/ntop_typedefs.h index d6f952c9a6..5b83b3f940 100644 --- a/include/ntop_typedefs.h +++ b/include/ntop_typedefs.h @@ -265,7 +265,7 @@ typedef struct { typedef struct zmq_remote_stats { char remote_ifname[32], remote_ifaddress[64]; char remote_probe_address[64], remote_probe_public_address[64]; - char remote_probe_version[16], remote_probe_os[24]; + char remote_probe_version[64], remote_probe_os[64]; u_int8_t source_id, num_exporters; u_int64_t remote_bytes, remote_pkts, num_flow_exports; u_int32_t remote_ifspeed, remote_time, local_time, avg_bps, avg_pps; @@ -429,7 +429,8 @@ typedef enum { flowhashing_iface_idx, flowhashing_ingress_iface_idx, flowhashing_vlan, - flowhashing_vrfid /* VRF Id */ + flowhashing_vrfid, /* VRF Id */ + flowhashing_probe_ip_and_ingress_iface_idx, } FlowHashingEnum; typedef enum { diff --git a/scripts/locales/en.lua b/scripts/locales/en.lua index 741de8f62a..11ec956169 100644 --- a/scripts/locales/en.lua +++ b/scripts/locales/en.lua @@ -3371,6 +3371,7 @@ local lang = { ["preferences"] = "Preferences", ["primary_dns"] = "Primary DNS", ["probe_ip_address"] = "Probe IP", + ["probe_ip_and_ingress_iface_idx"] = "Probe IP + Ingress Interface", ["protocols"] = "Applications", ["radius_admin_group_description"] = "The user is authenticated as admin when the value for the Filter-Id Attribute-Value pair returned by RADIUS matches the one specified in this field. When there is no match, the user is authenticated as unprivileged.", ["radius_admin_group_title"] = "RADIUS Admin Filter-Id", diff --git a/scripts/lua/if_stats.lua b/scripts/lua/if_stats.lua index ab3d67e9ec..fb24232447 100644 --- a/scripts/lua/if_stats.lua +++ b/scripts/lua/if_stats.lua @@ -1726,7 +1726,8 @@ elseif(page == "config") then i18n("prefs.probe_ip_address"), i18n("prefs.flow_interface"), i18n("prefs.ingress_flow_interface"), - i18n("prefs.ingress_vrf_id") + i18n("prefs.ingress_vrf_id"), + i18n("prefs.probe_ip_and_ingress_iface_idx") } local values = {} @@ -1742,7 +1743,8 @@ elseif(page == "config") then "probe_ip", "iface_idx", "ingress_iface_idx", - "ingress_vrf_id" + "ingress_vrf_id", + "probe_ip_and_ingress_iface_idx", } end diff --git a/src/FrequentStringItems.cpp b/src/FrequentStringItems.cpp index cead1f4fdd..d93a35a2d0 100644 --- a/src/FrequentStringItems.cpp +++ b/src/FrequentStringItems.cpp @@ -22,46 +22,20 @@ /* ******************************************************** */ -FrequentStringItems::~FrequentStringItems() { - cleanup(); -} - -/* ******************************************************** */ - -void FrequentStringItems::cleanup() { - FrequentStringKey_t *current, *tmp; - - m.lock(__FILE__, __LINE__); - - HASH_ITER(hh, q, current, tmp) { - HASH_DEL(q, current); /* delete it */ - free(current->key); - free(current); /* free it */ - } - - m.unlock(__FILE__, __LINE__); -} - -/* ******************************************************** */ - void FrequentStringItems::add(char *key, u_int32_t value) { - FrequentStringKey_t *s = NULL; + std::map::iterator it; m.lock(__FILE__, __LINE__); - - HASH_FIND_STR(q, key, s); - if(s) - s->value += value; + it = q.find(std::string(key)); + + if(it != q.end()) + it->second += value; else { - if(HASH_COUNT(q) > max_items_threshold) + if(q.size() > max_items_threshold) prune(); - if((s = (FrequentStringKey_t*)malloc(sizeof(FrequentStringKey_t))) != NULL) { - s->key = strdup(key), s->value = value; - - HASH_ADD_STR(q, key, s); - } + q[std::string(key)] = value; } m.unlock(__FILE__, __LINE__); @@ -69,63 +43,49 @@ void FrequentStringItems::add(char *key, u_int32_t value) { /* ******************************************************** */ -static int value_sort(FrequentStringKey_t *a, FrequentStringKey_t *b) { - return(b->value - a->value); /* desc sort */ -} - -/* ******************************************************** */ +static bool sortByVal(const pair &a, + const pair &b) { + return (a.first < b.first); +} void FrequentStringItems::prune() { - FrequentStringKey_t *curr, *tmp; - u_int32_t num = 0; - /* No lock here */ - + u_int32_t num = 0; + std::vector< std::pair > vec; + /* Sort the hash items by value and remove those who exceeded the threshold of max_items_threshold */ - HASH_SORT(q, value_sort); + for(std::map::iterator it1 = q.begin(); it1 != q.end(); ++it1) + vec.push_back(std::make_pair(it1->second, it1->first)); - HASH_ITER(hh, q, curr, tmp) { - if(++num > max_items) { - HASH_DEL(q, curr); - free(curr->key); - free(curr); - } - } -} - -/* ******************************************************** */ - -void FrequentStringItems::print() { - FrequentStringKey_t *curr; - - m.lock(__FILE__, __LINE__); + sort(vec.begin(), vec.end(), sortByVal); - HASH_SORT(q, value_sort); - - for(curr=q; curr != NULL; curr = (FrequentStringKey_t*)curr->hh.next) { - ntop->getTrace()->traceEvent(TRACE_NORMAL, "%s = %u\n", curr->key, curr->value); + for(std::vector>::iterator it2 = vec.begin(); it2 != vec.end(); ++it2) { + if(++num < max_items) { + /* + u_int32_t id = it2.first; + std::string k = it2->second; + q.erase(k); + */ + } else + break; } - - m.unlock(__FILE__, __LINE__); } /* ******************************************************** */ char* FrequentStringItems::json() { - FrequentStringKey_t *curr; json_object *j; char *rsp; if((j = json_object_new_object()) == NULL) return(NULL); m.lock(__FILE__, __LINE__); - HASH_SORT(q, value_sort); - for(curr=q; curr != NULL; curr = (FrequentStringKey_t*)curr->hh.next) - json_object_object_add(j, curr->key, json_object_new_int64(curr->value)); + for(std::map::iterator it = q.begin(); it != q.end(); ++it) + json_object_object_add(j, it->first.c_str(), json_object_new_int64(it->second)); rsp = strdup(json_object_to_json_string(j)); json_object_put(j); diff --git a/src/NetworkInterface.cpp b/src/NetworkInterface.cpp index 22c7ca1da1..3cc6e7da36 100644 --- a/src/NetworkInterface.cpp +++ b/src/NetworkInterface.cpp @@ -241,7 +241,7 @@ void NetworkInterface::init() { is_loopback = is_traffic_mirrored = false, lbd_serialize_by_mac = false, discard_probing_traffic = false; flows_only_interface = false; - numSubInterfaces = 0, flowHashing = NULL, + numSubInterfaces = 0; pcap_datalink_type = 0, mtuWarningShown = false, purge_idle_flows_hosts = true, id = (u_int8_t)-1, last_remote_pps = 0, last_remote_bps = 0, @@ -344,6 +344,7 @@ void NetworkInterface::checkDisaggregationMode() { if(!strcmp(rsp, DISAGGREGATION_PROBE_IP)) flowHashingMode = flowhashing_probe_ip; else if(!strcmp(rsp, DISAGGREGATION_IFACE_ID)) flowHashingMode = flowhashing_iface_idx; else if(!strcmp(rsp, DISAGGREGATION_INGRESS_IFACE_ID)) flowHashingMode = flowhashing_ingress_iface_idx; + else if(!strcmp(rsp, DISAGGREGATION_INGRESS_PROBE_IP_AND_IFACE_ID)) flowHashingMode = flowhashing_probe_ip_and_ingress_iface_idx; else if(!strcmp(rsp, DISAGGREGATION_INGRESS_VRF_ID)) flowHashingMode = flowhashing_vrfid; else if(!strcmp(rsp, DISAGGREGATION_VLAN)) flowHashingMode = flowhashing_vlan; else if(strcmp(rsp, DISAGGREGATION_NONE)) @@ -557,18 +558,6 @@ NetworkInterface::~NetworkInterface() { delete it->second; external_alerts.clear(); - if(flowHashing) { - FlowHashing *current, *tmp; - - HASH_ITER(hh, flowHashing, current, tmp) { - /* Interfaces are deleted by the main termination function */ - HASH_DEL(flowHashing, current); - free(current); - } - - flowHashing = NULL; - } - #ifdef NTOPNG_PRO if(policer) delete(policer); #ifndef HAVE_NEDGE @@ -891,22 +880,10 @@ Flow* NetworkInterface::getFlow(Mac *srcMac, Mac *dstMac, /* **************************************************** */ /* NOTE: the interface is deleted when this method returns false */ -bool NetworkInterface::registerSubInterface(NetworkInterface *sub_iface, u_int32_t criteria) { - FlowHashing *h = NULL; - - h = (FlowHashing*)calloc(1, sizeof(FlowHashing)); - - if(h == NULL) { - ntop->getTrace()->traceEvent(TRACE_WARNING, "Not enough memory"); - delete sub_iface; - return false; - } - +bool NetworkInterface::registerSubInterface(NetworkInterface *sub_iface, u_int64_t criteria) { /* registerInterface deletes the interface on failure */ - if(!ntop->registerInterface(sub_iface)) { - free(h); + if(!ntop->registerInterface(sub_iface)) return false; - } sub_iface->setSubInterface(); @@ -918,10 +895,7 @@ bool NetworkInterface::registerSubInterface(NetworkInterface *sub_iface, u_int32 sub_iface->startPacketPolling(); /* Won't actually start a thread, just mark this interface as running */ - h->criteria = criteria; - h->iface = sub_iface; - - HASH_ADD_INT(flowHashing, criteria, h); + flowHashing[criteria] = sub_iface; /* Add it to the hash */ numSubInterfaces++; ntop->getRedis()->set(CONST_STR_RELOAD_LISTS, (const char * const)"1"); @@ -931,17 +905,13 @@ bool NetworkInterface::registerSubInterface(NetworkInterface *sub_iface, u_int32 /* **************************************************** */ -NetworkInterface* NetworkInterface::getDynInterface(u_int32_t criteria, bool parser_interface) { +NetworkInterface* NetworkInterface::getDynInterface(u_int64_t criteria, bool parser_interface) { NetworkInterface *sub_iface = NULL; #ifndef HAVE_NEDGE - FlowHashing *h = NULL; - - HASH_FIND_INT(flowHashing, &criteria, h); - - if(h != NULL) { - - sub_iface = h->iface; - + std::map::iterator subIface = flowHashing.find(criteria); + + if(subIface != flowHashing.end()) { + sub_iface = subIface->second; } else { /* Interface not found */ @@ -953,31 +923,43 @@ NetworkInterface* NetworkInterface::getDynInterface(u_int32_t criteria, bool par switch(flowHashingMode) { case flowhashing_vlan: vIface_type = CONST_INTERFACE_TYPE_VLAN; - snprintf(buf, sizeof(buf), "%s [VLAN Id: %u]", ifname, criteria); + snprintf(buf, sizeof(buf), "%s [VLAN Id: %u]", ifname, (unsigned int)criteria); // snprintf(buf, sizeof(buf), "VLAN Id %u", criteria); break; case flowhashing_probe_ip: vIface_type = CONST_INTERFACE_TYPE_FLOW; - snprintf(buf, sizeof(buf), "%s [Probe IP: %s]", ifname, Utils::intoaV4(criteria, buf1, sizeof(buf1))); - // snprintf(buf, sizeof(buf), "Probe IP %s", Utils::intoaV4(criteria, buf1, sizeof(buf1))); + snprintf(buf, sizeof(buf), "%s [Probe IP: %s]", ifname, Utils::intoaV4((unsigned int)criteria, buf1, sizeof(buf1))); + // snprintf(buf, sizeof(buf), "Probe IP %s", Utils::intoaV4((unsigned int)criteria, buf1, sizeof(buf1))); break; case flowhashing_iface_idx: case flowhashing_ingress_iface_idx: vIface_type = CONST_INTERFACE_TYPE_FLOW; - snprintf(buf, sizeof(buf), "%s [If Idx: %u]", ifname, criteria); - // snprintf(buf, sizeof(buf), "If Idx %u", criteria); + snprintf(buf, sizeof(buf), "%s [InIfIdx: %u]", ifname, (unsigned int)criteria); + // snprintf(buf, sizeof(buf), "If Idx %u", (unsigned int)criteria); + break; + + case flowhashing_probe_ip_and_ingress_iface_idx: + { + /* 64 bit value: upper 32 bit is nProbe IP, lower 32 bit ifIdx */ + u_int32_t nprobe_ip = (u_int32_t)(criteria >> 32); + u_int32_t if_id = (u_int32_t)(criteria & 0xFFFFFFFF); + + vIface_type = CONST_INTERFACE_TYPE_FLOW; + snprintf(buf, sizeof(buf), "%s [Probe IP: %s][InIfIdx: %u]", ifname, + Utils::intoaV4(nprobe_ip, buf1, sizeof(buf1)), if_id); + // snprintf(buf, sizeof(buf), "Probe IP %s", Utils::intoaV4((unsigned int)criteria, buf1, sizeof(buf1))); + } break; case flowhashing_vrfid: vIface_type = CONST_INTERFACE_TYPE_FLOW; - snprintf(buf, sizeof(buf), "%s [VRF Id: %u]", ifname, criteria); - // snprintf(buf, sizeof(buf), "VRF Id %u", criteria); + snprintf(buf, sizeof(buf), "%s [VRF Id: %u]", ifname, (unsigned int)criteria); + // snprintf(buf, sizeof(buf), "VRF Id %u", (unsigned int)criteria); break; default: - free(h); return(NULL); break; } @@ -996,9 +978,8 @@ NetworkInterface* NetworkInterface::getDynInterface(u_int32_t criteria, bool par /* NOTE: interface deleted by registerSubInterface */ sub_iface = NULL; } - } else { - ntop->getTrace()->traceEvent(TRACE_WARNING, "Failure allocating interface: not enough memory?"); - } + } else + ntop->getTrace()->traceEvent(TRACE_WARNING, "Failure allocating interface: not enough memory?"); } else { static bool too_many_interfaces_error = false; @@ -1585,14 +1566,8 @@ void NetworkInterface::purgeIdle(time_t when, bool force_idle) { ntop->getTrace()->traceEvent(TRACE_DEBUG, "Purged %u idle ASs, MAC, Countries, VLANs... on %s", o, ifname); - if(flowHashing) { - FlowHashing *current, *tmp; - - HASH_ITER(hh, flowHashing, current, tmp) { - if(current->iface) - current->iface->purgeIdle(when, force_idle); - } - } + for(std::map::iterator it = flowHashing.begin(); it != flowHashing.end(); ++it) + it->second->purgeIdle(when, force_idle); checkHostsToRestore(); diff --git a/src/ParserInterface.cpp b/src/ParserInterface.cpp index 787daceec8..abe5e0b338 100755 --- a/src/ParserInterface.cpp +++ b/src/ParserInterface.cpp @@ -94,26 +94,31 @@ void ParserInterface::processFlow(ParsedFlow *zflow) { switch(flowHashingMode) { case flowhashing_probe_ip: - vIface = getDynInterface((u_int32_t)zflow->device_ip, true); + vIface = getDynInterface((u_int64_t)zflow->device_ip, true); break; case flowhashing_iface_idx: - if(flowHashingIgnoredInterfaces.find((u_int32_t)zflow->outIndex) == flowHashingIgnoredInterfaces.end()) - vIfaceEgress = getDynInterface((u_int32_t)zflow->outIndex, true); + if(flowHashingIgnoredInterfaces.find((u_int64_t)zflow->outIndex) == flowHashingIgnoredInterfaces.end()) + vIfaceEgress = getDynInterface((u_int64_t)zflow->outIndex, true); /* No break HERE, want to get two interfaces, one for the ingress and one for the egress. */ case flowhashing_ingress_iface_idx: - if(flowHashingIgnoredInterfaces.find((u_int32_t)zflow->inIndex) == flowHashingIgnoredInterfaces.end()) - vIface = getDynInterface((u_int32_t)zflow->inIndex, true); + if(flowHashingIgnoredInterfaces.find((u_int64_t)zflow->inIndex) == flowHashingIgnoredInterfaces.end()) + vIface = getDynInterface((u_int64_t)zflow->inIndex, true); break; + case flowhashing_probe_ip_and_ingress_iface_idx: + // ntop->getTrace()->traceEvent(TRACE_NORMAL, "[IP: %u][inIndex: %u]", zflow->device_ip, zflow->inIndex); + vIface = getDynInterface((((u_int64_t)zflow->device_ip) << 32) + zflow->inIndex, true); + break; + case flowhashing_vrfid: - vIface = getDynInterface((u_int32_t)zflow->vrfId, true); + vIface = getDynInterface((u_int64_t)zflow->vrfId, true); break; case flowhashing_vlan: - vIface = getDynInterface((u_int32_t)zflow->vlan_id, true); + vIface = getDynInterface((u_int64_t)zflow->vlan_id, true); break; default: diff --git a/src/ZMQCollectorInterface.cpp b/src/ZMQCollectorInterface.cpp index 9951793168..4a70a3cc98 100644 --- a/src/ZMQCollectorInterface.cpp +++ b/src/ZMQCollectorInterface.cpp @@ -536,11 +536,8 @@ void ZMQCollectorInterface::lua(lua_State* vm) { void ZMQCollectorInterface::purgeIdle(time_t when, bool force_idle) { NetworkInterface::purgeIdle(when); - if(flowHashing) { - FlowHashing *current, *tmp; - HASH_ITER(hh, flowHashing, current, tmp) - static_cast(current->iface)->purgeIdle(when, force_idle); - } + for(std::map::iterator it = flowHashing.begin(); it != flowHashing.end(); ++it) + it->second->purgeIdle(when, force_idle); } #endif diff --git a/src/ZMQParserInterface.cpp b/src/ZMQParserInterface.cpp index e1225decb9..55974ebb4a 100755 --- a/src/ZMQParserInterface.cpp +++ b/src/ZMQParserInterface.cpp @@ -296,14 +296,10 @@ u_int8_t ZMQParserInterface::parseEvent(const char * const payload, int payload_ /* Process Flow */ setRemoteStats(&zrs); - if(flowHashing) { - FlowHashing *current, *tmp; - ZMQParserInterface *current_iface; - - HASH_ITER(hh, flowHashing, current, tmp) { - if((current_iface = dynamic_cast(current->iface))) - current_iface->setRemoteStats(&zrs); - } + for(std::map::iterator it = flowHashing.begin(); it != flowHashing.end(); ++it) { + ZMQParserInterface *z = (ZMQParserInterface*)it->second; + + z->setRemoteStats(&zrs); } /* Dispose memory */